Merge tag 'block-5.17-2022-01-28' of git://git.kernel.dk/linux-block
[linux-2.6-microblaze.git] / kernel / kprobes.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Kernel Probes (KProbes)
4  *
5  * Copyright (C) IBM Corporation, 2002, 2004
6  *
7  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8  *              Probes initial implementation (includes suggestions from
9  *              Rusty Russell).
10  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
11  *              hlists and exceptions notifier as suggested by Andi Kleen.
12  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
13  *              interface to access function arguments.
14  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
15  *              exceptions notifier to be first on the priority list.
16  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
17  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
18  *              <prasanna@in.ibm.com> added function-return probes.
19  */
20
21 #define pr_fmt(fmt) "kprobes: " fmt
22
23 #include <linux/kprobes.h>
24 #include <linux/hash.h>
25 #include <linux/init.h>
26 #include <linux/slab.h>
27 #include <linux/stddef.h>
28 #include <linux/export.h>
29 #include <linux/moduleloader.h>
30 #include <linux/kallsyms.h>
31 #include <linux/freezer.h>
32 #include <linux/seq_file.h>
33 #include <linux/debugfs.h>
34 #include <linux/sysctl.h>
35 #include <linux/kdebug.h>
36 #include <linux/memory.h>
37 #include <linux/ftrace.h>
38 #include <linux/cpu.h>
39 #include <linux/jump_label.h>
40 #include <linux/static_call.h>
41 #include <linux/perf_event.h>
42
43 #include <asm/sections.h>
44 #include <asm/cacheflush.h>
45 #include <asm/errno.h>
46 #include <linux/uaccess.h>
47
48 #define KPROBE_HASH_BITS 6
49 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
50
51 #if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL)
52 #define kprobe_sysctls_init() do { } while (0)
53 #endif
54
55 static int kprobes_initialized;
56 /* kprobe_table can be accessed by
57  * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held.
58  * Or
59  * - RCU hlist traversal under disabling preempt (breakpoint handlers)
60  */
61 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
62
63 /* NOTE: change this value only with 'kprobe_mutex' held */
64 static bool kprobes_all_disarmed;
65
66 /* This protects 'kprobe_table' and 'optimizing_list' */
67 static DEFINE_MUTEX(kprobe_mutex);
68 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
69
70 kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
71                                         unsigned int __unused)
72 {
73         return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
74 }
75
76 /*
77  * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where
78  * kprobes can not probe.
79  */
80 static LIST_HEAD(kprobe_blacklist);
81
82 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
83 /*
84  * 'kprobe::ainsn.insn' points to the copy of the instruction to be
85  * single-stepped. x86_64, POWER4 and above have no-exec support and
86  * stepping on the instruction on a vmalloced/kmalloced/data page
87  * is a recipe for disaster
88  */
89 struct kprobe_insn_page {
90         struct list_head list;
91         kprobe_opcode_t *insns;         /* Page of instruction slots */
92         struct kprobe_insn_cache *cache;
93         int nused;
94         int ngarbage;
95         char slot_used[];
96 };
97
98 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
99         (offsetof(struct kprobe_insn_page, slot_used) + \
100          (sizeof(char) * (slots)))
101
102 static int slots_per_page(struct kprobe_insn_cache *c)
103 {
104         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
105 }
106
107 enum kprobe_slot_state {
108         SLOT_CLEAN = 0,
109         SLOT_DIRTY = 1,
110         SLOT_USED = 2,
111 };
112
113 void __weak *alloc_insn_page(void)
114 {
115         /*
116          * Use module_alloc() so this page is within +/- 2GB of where the
117          * kernel image and loaded module images reside. This is required
118          * for most of the architectures.
119          * (e.g. x86-64 needs this to handle the %rip-relative fixups.)
120          */
121         return module_alloc(PAGE_SIZE);
122 }
123
124 static void free_insn_page(void *page)
125 {
126         module_memfree(page);
127 }
128
129 struct kprobe_insn_cache kprobe_insn_slots = {
130         .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
131         .alloc = alloc_insn_page,
132         .free = free_insn_page,
133         .sym = KPROBE_INSN_PAGE_SYM,
134         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135         .insn_size = MAX_INSN_SIZE,
136         .nr_garbage = 0,
137 };
138 static int collect_garbage_slots(struct kprobe_insn_cache *c);
139
140 /**
141  * __get_insn_slot() - Find a slot on an executable page for an instruction.
142  * We allocate an executable page if there's no room on existing ones.
143  */
144 kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
145 {
146         struct kprobe_insn_page *kip;
147         kprobe_opcode_t *slot = NULL;
148
149         /* Since the slot array is not protected by rcu, we need a mutex */
150         mutex_lock(&c->mutex);
151  retry:
152         rcu_read_lock();
153         list_for_each_entry_rcu(kip, &c->pages, list) {
154                 if (kip->nused < slots_per_page(c)) {
155                         int i;
156
157                         for (i = 0; i < slots_per_page(c); i++) {
158                                 if (kip->slot_used[i] == SLOT_CLEAN) {
159                                         kip->slot_used[i] = SLOT_USED;
160                                         kip->nused++;
161                                         slot = kip->insns + (i * c->insn_size);
162                                         rcu_read_unlock();
163                                         goto out;
164                                 }
165                         }
166                         /* kip->nused is broken. Fix it. */
167                         kip->nused = slots_per_page(c);
168                         WARN_ON(1);
169                 }
170         }
171         rcu_read_unlock();
172
173         /* If there are any garbage slots, collect it and try again. */
174         if (c->nr_garbage && collect_garbage_slots(c) == 0)
175                 goto retry;
176
177         /* All out of space.  Need to allocate a new page. */
178         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
179         if (!kip)
180                 goto out;
181
182         kip->insns = c->alloc();
183         if (!kip->insns) {
184                 kfree(kip);
185                 goto out;
186         }
187         INIT_LIST_HEAD(&kip->list);
188         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
189         kip->slot_used[0] = SLOT_USED;
190         kip->nused = 1;
191         kip->ngarbage = 0;
192         kip->cache = c;
193         list_add_rcu(&kip->list, &c->pages);
194         slot = kip->insns;
195
196         /* Record the perf ksymbol register event after adding the page */
197         perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
198                            PAGE_SIZE, false, c->sym);
199 out:
200         mutex_unlock(&c->mutex);
201         return slot;
202 }
203
204 /* Return true if all garbages are collected, otherwise false. */
205 static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
206 {
207         kip->slot_used[idx] = SLOT_CLEAN;
208         kip->nused--;
209         if (kip->nused == 0) {
210                 /*
211                  * Page is no longer in use.  Free it unless
212                  * it's the last one.  We keep the last one
213                  * so as not to have to set it up again the
214                  * next time somebody inserts a probe.
215                  */
216                 if (!list_is_singular(&kip->list)) {
217                         /*
218                          * Record perf ksymbol unregister event before removing
219                          * the page.
220                          */
221                         perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
222                                            (unsigned long)kip->insns, PAGE_SIZE, true,
223                                            kip->cache->sym);
224                         list_del_rcu(&kip->list);
225                         synchronize_rcu();
226                         kip->cache->free(kip->insns);
227                         kfree(kip);
228                 }
229                 return true;
230         }
231         return false;
232 }
233
234 static int collect_garbage_slots(struct kprobe_insn_cache *c)
235 {
236         struct kprobe_insn_page *kip, *next;
237
238         /* Ensure no-one is interrupted on the garbages */
239         synchronize_rcu();
240
241         list_for_each_entry_safe(kip, next, &c->pages, list) {
242                 int i;
243
244                 if (kip->ngarbage == 0)
245                         continue;
246                 kip->ngarbage = 0;      /* we will collect all garbages */
247                 for (i = 0; i < slots_per_page(c); i++) {
248                         if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
249                                 break;
250                 }
251         }
252         c->nr_garbage = 0;
253         return 0;
254 }
255
256 void __free_insn_slot(struct kprobe_insn_cache *c,
257                       kprobe_opcode_t *slot, int dirty)
258 {
259         struct kprobe_insn_page *kip;
260         long idx;
261
262         mutex_lock(&c->mutex);
263         rcu_read_lock();
264         list_for_each_entry_rcu(kip, &c->pages, list) {
265                 idx = ((long)slot - (long)kip->insns) /
266                         (c->insn_size * sizeof(kprobe_opcode_t));
267                 if (idx >= 0 && idx < slots_per_page(c))
268                         goto out;
269         }
270         /* Could not find this slot. */
271         WARN_ON(1);
272         kip = NULL;
273 out:
274         rcu_read_unlock();
275         /* Mark and sweep: this may sleep */
276         if (kip) {
277                 /* Check double free */
278                 WARN_ON(kip->slot_used[idx] != SLOT_USED);
279                 if (dirty) {
280                         kip->slot_used[idx] = SLOT_DIRTY;
281                         kip->ngarbage++;
282                         if (++c->nr_garbage > slots_per_page(c))
283                                 collect_garbage_slots(c);
284                 } else {
285                         collect_one_slot(kip, idx);
286                 }
287         }
288         mutex_unlock(&c->mutex);
289 }
290
291 /*
292  * Check given address is on the page of kprobe instruction slots.
293  * This will be used for checking whether the address on a stack
294  * is on a text area or not.
295  */
296 bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
297 {
298         struct kprobe_insn_page *kip;
299         bool ret = false;
300
301         rcu_read_lock();
302         list_for_each_entry_rcu(kip, &c->pages, list) {
303                 if (addr >= (unsigned long)kip->insns &&
304                     addr < (unsigned long)kip->insns + PAGE_SIZE) {
305                         ret = true;
306                         break;
307                 }
308         }
309         rcu_read_unlock();
310
311         return ret;
312 }
313
314 int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
315                              unsigned long *value, char *type, char *sym)
316 {
317         struct kprobe_insn_page *kip;
318         int ret = -ERANGE;
319
320         rcu_read_lock();
321         list_for_each_entry_rcu(kip, &c->pages, list) {
322                 if ((*symnum)--)
323                         continue;
324                 strscpy(sym, c->sym, KSYM_NAME_LEN);
325                 *type = 't';
326                 *value = (unsigned long)kip->insns;
327                 ret = 0;
328                 break;
329         }
330         rcu_read_unlock();
331
332         return ret;
333 }
334
335 #ifdef CONFIG_OPTPROBES
336 void __weak *alloc_optinsn_page(void)
337 {
338         return alloc_insn_page();
339 }
340
341 void __weak free_optinsn_page(void *page)
342 {
343         free_insn_page(page);
344 }
345
346 /* For optimized_kprobe buffer */
347 struct kprobe_insn_cache kprobe_optinsn_slots = {
348         .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
349         .alloc = alloc_optinsn_page,
350         .free = free_optinsn_page,
351         .sym = KPROBE_OPTINSN_PAGE_SYM,
352         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
353         /* .insn_size is initialized later */
354         .nr_garbage = 0,
355 };
356 #endif
357 #endif
358
359 /* We have preemption disabled.. so it is safe to use __ versions */
360 static inline void set_kprobe_instance(struct kprobe *kp)
361 {
362         __this_cpu_write(kprobe_instance, kp);
363 }
364
365 static inline void reset_kprobe_instance(void)
366 {
367         __this_cpu_write(kprobe_instance, NULL);
368 }
369
370 /*
371  * This routine is called either:
372  *      - under the 'kprobe_mutex' - during kprobe_[un]register().
373  *                              OR
374  *      - with preemption disabled - from architecture specific code.
375  */
376 struct kprobe *get_kprobe(void *addr)
377 {
378         struct hlist_head *head;
379         struct kprobe *p;
380
381         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
382         hlist_for_each_entry_rcu(p, head, hlist,
383                                  lockdep_is_held(&kprobe_mutex)) {
384                 if (p->addr == addr)
385                         return p;
386         }
387
388         return NULL;
389 }
390 NOKPROBE_SYMBOL(get_kprobe);
391
392 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
393
394 /* Return true if 'p' is an aggregator */
395 static inline bool kprobe_aggrprobe(struct kprobe *p)
396 {
397         return p->pre_handler == aggr_pre_handler;
398 }
399
400 /* Return true if 'p' is unused */
401 static inline bool kprobe_unused(struct kprobe *p)
402 {
403         return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
404                list_empty(&p->list);
405 }
406
407 /* Keep all fields in the kprobe consistent. */
408 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
409 {
410         memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
411         memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
412 }
413
414 #ifdef CONFIG_OPTPROBES
415 /* NOTE: This is protected by 'kprobe_mutex'. */
416 static bool kprobes_allow_optimization;
417
418 /*
419  * Call all 'kprobe::pre_handler' on the list, but ignores its return value.
420  * This must be called from arch-dep optimized caller.
421  */
422 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
423 {
424         struct kprobe *kp;
425
426         list_for_each_entry_rcu(kp, &p->list, list) {
427                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
428                         set_kprobe_instance(kp);
429                         kp->pre_handler(kp, regs);
430                 }
431                 reset_kprobe_instance();
432         }
433 }
434 NOKPROBE_SYMBOL(opt_pre_handler);
435
436 /* Free optimized instructions and optimized_kprobe */
437 static void free_aggr_kprobe(struct kprobe *p)
438 {
439         struct optimized_kprobe *op;
440
441         op = container_of(p, struct optimized_kprobe, kp);
442         arch_remove_optimized_kprobe(op);
443         arch_remove_kprobe(p);
444         kfree(op);
445 }
446
447 /* Return true if the kprobe is ready for optimization. */
448 static inline int kprobe_optready(struct kprobe *p)
449 {
450         struct optimized_kprobe *op;
451
452         if (kprobe_aggrprobe(p)) {
453                 op = container_of(p, struct optimized_kprobe, kp);
454                 return arch_prepared_optinsn(&op->optinsn);
455         }
456
457         return 0;
458 }
459
460 /* Return true if the kprobe is disarmed. Note: p must be on hash list */
461 static inline bool kprobe_disarmed(struct kprobe *p)
462 {
463         struct optimized_kprobe *op;
464
465         /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
466         if (!kprobe_aggrprobe(p))
467                 return kprobe_disabled(p);
468
469         op = container_of(p, struct optimized_kprobe, kp);
470
471         return kprobe_disabled(p) && list_empty(&op->list);
472 }
473
474 /* Return true if the probe is queued on (un)optimizing lists */
475 static bool kprobe_queued(struct kprobe *p)
476 {
477         struct optimized_kprobe *op;
478
479         if (kprobe_aggrprobe(p)) {
480                 op = container_of(p, struct optimized_kprobe, kp);
481                 if (!list_empty(&op->list))
482                         return true;
483         }
484         return false;
485 }
486
487 /*
488  * Return an optimized kprobe whose optimizing code replaces
489  * instructions including 'addr' (exclude breakpoint).
490  */
491 static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
492 {
493         int i;
494         struct kprobe *p = NULL;
495         struct optimized_kprobe *op;
496
497         /* Don't check i == 0, since that is a breakpoint case. */
498         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
499                 p = get_kprobe(addr - i);
500
501         if (p && kprobe_optready(p)) {
502                 op = container_of(p, struct optimized_kprobe, kp);
503                 if (arch_within_optimized_kprobe(op, addr))
504                         return p;
505         }
506
507         return NULL;
508 }
509
510 /* Optimization staging list, protected by 'kprobe_mutex' */
511 static LIST_HEAD(optimizing_list);
512 static LIST_HEAD(unoptimizing_list);
513 static LIST_HEAD(freeing_list);
514
515 static void kprobe_optimizer(struct work_struct *work);
516 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
517 #define OPTIMIZE_DELAY 5
518
519 /*
520  * Optimize (replace a breakpoint with a jump) kprobes listed on
521  * 'optimizing_list'.
522  */
523 static void do_optimize_kprobes(void)
524 {
525         lockdep_assert_held(&text_mutex);
526         /*
527          * The optimization/unoptimization refers 'online_cpus' via
528          * stop_machine() and cpu-hotplug modifies the 'online_cpus'.
529          * And same time, 'text_mutex' will be held in cpu-hotplug and here.
530          * This combination can cause a deadlock (cpu-hotplug tries to lock
531          * 'text_mutex' but stop_machine() can not be done because
532          * the 'online_cpus' has been changed)
533          * To avoid this deadlock, caller must have locked cpu-hotplug
534          * for preventing cpu-hotplug outside of 'text_mutex' locking.
535          */
536         lockdep_assert_cpus_held();
537
538         /* Optimization never be done when disarmed */
539         if (kprobes_all_disarmed || !kprobes_allow_optimization ||
540             list_empty(&optimizing_list))
541                 return;
542
543         arch_optimize_kprobes(&optimizing_list);
544 }
545
546 /*
547  * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
548  * if need) kprobes listed on 'unoptimizing_list'.
549  */
550 static void do_unoptimize_kprobes(void)
551 {
552         struct optimized_kprobe *op, *tmp;
553
554         lockdep_assert_held(&text_mutex);
555         /* See comment in do_optimize_kprobes() */
556         lockdep_assert_cpus_held();
557
558         /* Unoptimization must be done anytime */
559         if (list_empty(&unoptimizing_list))
560                 return;
561
562         arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
563         /* Loop on 'freeing_list' for disarming */
564         list_for_each_entry_safe(op, tmp, &freeing_list, list) {
565                 /* Switching from detour code to origin */
566                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
567                 /* Disarm probes if marked disabled */
568                 if (kprobe_disabled(&op->kp))
569                         arch_disarm_kprobe(&op->kp);
570                 if (kprobe_unused(&op->kp)) {
571                         /*
572                          * Remove unused probes from hash list. After waiting
573                          * for synchronization, these probes are reclaimed.
574                          * (reclaiming is done by do_free_cleaned_kprobes().)
575                          */
576                         hlist_del_rcu(&op->kp.hlist);
577                 } else
578                         list_del_init(&op->list);
579         }
580 }
581
582 /* Reclaim all kprobes on the 'freeing_list' */
583 static void do_free_cleaned_kprobes(void)
584 {
585         struct optimized_kprobe *op, *tmp;
586
587         list_for_each_entry_safe(op, tmp, &freeing_list, list) {
588                 list_del_init(&op->list);
589                 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
590                         /*
591                          * This must not happen, but if there is a kprobe
592                          * still in use, keep it on kprobes hash list.
593                          */
594                         continue;
595                 }
596                 free_aggr_kprobe(&op->kp);
597         }
598 }
599
600 /* Start optimizer after OPTIMIZE_DELAY passed */
601 static void kick_kprobe_optimizer(void)
602 {
603         schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
604 }
605
606 /* Kprobe jump optimizer */
607 static void kprobe_optimizer(struct work_struct *work)
608 {
609         mutex_lock(&kprobe_mutex);
610         cpus_read_lock();
611         mutex_lock(&text_mutex);
612
613         /*
614          * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
615          * kprobes before waiting for quiesence period.
616          */
617         do_unoptimize_kprobes();
618
619         /*
620          * Step 2: Wait for quiesence period to ensure all potentially
621          * preempted tasks to have normally scheduled. Because optprobe
622          * may modify multiple instructions, there is a chance that Nth
623          * instruction is preempted. In that case, such tasks can return
624          * to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
625          * Note that on non-preemptive kernel, this is transparently converted
626          * to synchronoze_sched() to wait for all interrupts to have completed.
627          */
628         synchronize_rcu_tasks();
629
630         /* Step 3: Optimize kprobes after quiesence period */
631         do_optimize_kprobes();
632
633         /* Step 4: Free cleaned kprobes after quiesence period */
634         do_free_cleaned_kprobes();
635
636         mutex_unlock(&text_mutex);
637         cpus_read_unlock();
638
639         /* Step 5: Kick optimizer again if needed */
640         if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
641                 kick_kprobe_optimizer();
642
643         mutex_unlock(&kprobe_mutex);
644 }
645
646 /* Wait for completing optimization and unoptimization */
647 void wait_for_kprobe_optimizer(void)
648 {
649         mutex_lock(&kprobe_mutex);
650
651         while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
652                 mutex_unlock(&kprobe_mutex);
653
654                 /* This will also make 'optimizing_work' execute immmediately */
655                 flush_delayed_work(&optimizing_work);
656                 /* 'optimizing_work' might not have been queued yet, relax */
657                 cpu_relax();
658
659                 mutex_lock(&kprobe_mutex);
660         }
661
662         mutex_unlock(&kprobe_mutex);
663 }
664
665 static bool optprobe_queued_unopt(struct optimized_kprobe *op)
666 {
667         struct optimized_kprobe *_op;
668
669         list_for_each_entry(_op, &unoptimizing_list, list) {
670                 if (op == _op)
671                         return true;
672         }
673
674         return false;
675 }
676
677 /* Optimize kprobe if p is ready to be optimized */
678 static void optimize_kprobe(struct kprobe *p)
679 {
680         struct optimized_kprobe *op;
681
682         /* Check if the kprobe is disabled or not ready for optimization. */
683         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
684             (kprobe_disabled(p) || kprobes_all_disarmed))
685                 return;
686
687         /* kprobes with 'post_handler' can not be optimized */
688         if (p->post_handler)
689                 return;
690
691         op = container_of(p, struct optimized_kprobe, kp);
692
693         /* Check there is no other kprobes at the optimized instructions */
694         if (arch_check_optimized_kprobe(op) < 0)
695                 return;
696
697         /* Check if it is already optimized. */
698         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
699                 if (optprobe_queued_unopt(op)) {
700                         /* This is under unoptimizing. Just dequeue the probe */
701                         list_del_init(&op->list);
702                 }
703                 return;
704         }
705         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
706
707         /*
708          * On the 'unoptimizing_list' and 'optimizing_list',
709          * 'op' must have OPTIMIZED flag
710          */
711         if (WARN_ON_ONCE(!list_empty(&op->list)))
712                 return;
713
714         list_add(&op->list, &optimizing_list);
715         kick_kprobe_optimizer();
716 }
717
718 /* Short cut to direct unoptimizing */
719 static void force_unoptimize_kprobe(struct optimized_kprobe *op)
720 {
721         lockdep_assert_cpus_held();
722         arch_unoptimize_kprobe(op);
723         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
724 }
725
726 /* Unoptimize a kprobe if p is optimized */
727 static void unoptimize_kprobe(struct kprobe *p, bool force)
728 {
729         struct optimized_kprobe *op;
730
731         if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
732                 return; /* This is not an optprobe nor optimized */
733
734         op = container_of(p, struct optimized_kprobe, kp);
735         if (!kprobe_optimized(p))
736                 return;
737
738         if (!list_empty(&op->list)) {
739                 if (optprobe_queued_unopt(op)) {
740                         /* Queued in unoptimizing queue */
741                         if (force) {
742                                 /*
743                                  * Forcibly unoptimize the kprobe here, and queue it
744                                  * in the freeing list for release afterwards.
745                                  */
746                                 force_unoptimize_kprobe(op);
747                                 list_move(&op->list, &freeing_list);
748                         }
749                 } else {
750                         /* Dequeue from the optimizing queue */
751                         list_del_init(&op->list);
752                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
753                 }
754                 return;
755         }
756
757         /* Optimized kprobe case */
758         if (force) {
759                 /* Forcibly update the code: this is a special case */
760                 force_unoptimize_kprobe(op);
761         } else {
762                 list_add(&op->list, &unoptimizing_list);
763                 kick_kprobe_optimizer();
764         }
765 }
766
767 /* Cancel unoptimizing for reusing */
768 static int reuse_unused_kprobe(struct kprobe *ap)
769 {
770         struct optimized_kprobe *op;
771
772         /*
773          * Unused kprobe MUST be on the way of delayed unoptimizing (means
774          * there is still a relative jump) and disabled.
775          */
776         op = container_of(ap, struct optimized_kprobe, kp);
777         WARN_ON_ONCE(list_empty(&op->list));
778         /* Enable the probe again */
779         ap->flags &= ~KPROBE_FLAG_DISABLED;
780         /* Optimize it again. (remove from 'op->list') */
781         if (!kprobe_optready(ap))
782                 return -EINVAL;
783
784         optimize_kprobe(ap);
785         return 0;
786 }
787
788 /* Remove optimized instructions */
789 static void kill_optimized_kprobe(struct kprobe *p)
790 {
791         struct optimized_kprobe *op;
792
793         op = container_of(p, struct optimized_kprobe, kp);
794         if (!list_empty(&op->list))
795                 /* Dequeue from the (un)optimization queue */
796                 list_del_init(&op->list);
797         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
798
799         if (kprobe_unused(p)) {
800                 /* Enqueue if it is unused */
801                 list_add(&op->list, &freeing_list);
802                 /*
803                  * Remove unused probes from the hash list. After waiting
804                  * for synchronization, this probe is reclaimed.
805                  * (reclaiming is done by do_free_cleaned_kprobes().)
806                  */
807                 hlist_del_rcu(&op->kp.hlist);
808         }
809
810         /* Don't touch the code, because it is already freed. */
811         arch_remove_optimized_kprobe(op);
812 }
813
814 static inline
815 void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
816 {
817         if (!kprobe_ftrace(p))
818                 arch_prepare_optimized_kprobe(op, p);
819 }
820
821 /* Try to prepare optimized instructions */
822 static void prepare_optimized_kprobe(struct kprobe *p)
823 {
824         struct optimized_kprobe *op;
825
826         op = container_of(p, struct optimized_kprobe, kp);
827         __prepare_optimized_kprobe(op, p);
828 }
829
830 /* Allocate new optimized_kprobe and try to prepare optimized instructions. */
831 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
832 {
833         struct optimized_kprobe *op;
834
835         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
836         if (!op)
837                 return NULL;
838
839         INIT_LIST_HEAD(&op->list);
840         op->kp.addr = p->addr;
841         __prepare_optimized_kprobe(op, p);
842
843         return &op->kp;
844 }
845
846 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
847
848 /*
849  * Prepare an optimized_kprobe and optimize it.
850  * NOTE: 'p' must be a normal registered kprobe.
851  */
852 static void try_to_optimize_kprobe(struct kprobe *p)
853 {
854         struct kprobe *ap;
855         struct optimized_kprobe *op;
856
857         /* Impossible to optimize ftrace-based kprobe. */
858         if (kprobe_ftrace(p))
859                 return;
860
861         /* For preparing optimization, jump_label_text_reserved() is called. */
862         cpus_read_lock();
863         jump_label_lock();
864         mutex_lock(&text_mutex);
865
866         ap = alloc_aggr_kprobe(p);
867         if (!ap)
868                 goto out;
869
870         op = container_of(ap, struct optimized_kprobe, kp);
871         if (!arch_prepared_optinsn(&op->optinsn)) {
872                 /* If failed to setup optimizing, fallback to kprobe. */
873                 arch_remove_optimized_kprobe(op);
874                 kfree(op);
875                 goto out;
876         }
877
878         init_aggr_kprobe(ap, p);
879         optimize_kprobe(ap);    /* This just kicks optimizer thread. */
880
881 out:
882         mutex_unlock(&text_mutex);
883         jump_label_unlock();
884         cpus_read_unlock();
885 }
886
887 static void optimize_all_kprobes(void)
888 {
889         struct hlist_head *head;
890         struct kprobe *p;
891         unsigned int i;
892
893         mutex_lock(&kprobe_mutex);
894         /* If optimization is already allowed, just return. */
895         if (kprobes_allow_optimization)
896                 goto out;
897
898         cpus_read_lock();
899         kprobes_allow_optimization = true;
900         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
901                 head = &kprobe_table[i];
902                 hlist_for_each_entry(p, head, hlist)
903                         if (!kprobe_disabled(p))
904                                 optimize_kprobe(p);
905         }
906         cpus_read_unlock();
907         pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
908 out:
909         mutex_unlock(&kprobe_mutex);
910 }
911
912 #ifdef CONFIG_SYSCTL
913 static void unoptimize_all_kprobes(void)
914 {
915         struct hlist_head *head;
916         struct kprobe *p;
917         unsigned int i;
918
919         mutex_lock(&kprobe_mutex);
920         /* If optimization is already prohibited, just return. */
921         if (!kprobes_allow_optimization) {
922                 mutex_unlock(&kprobe_mutex);
923                 return;
924         }
925
926         cpus_read_lock();
927         kprobes_allow_optimization = false;
928         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
929                 head = &kprobe_table[i];
930                 hlist_for_each_entry(p, head, hlist) {
931                         if (!kprobe_disabled(p))
932                                 unoptimize_kprobe(p, false);
933                 }
934         }
935         cpus_read_unlock();
936         mutex_unlock(&kprobe_mutex);
937
938         /* Wait for unoptimizing completion. */
939         wait_for_kprobe_optimizer();
940         pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
941 }
942
943 static DEFINE_MUTEX(kprobe_sysctl_mutex);
944 static int sysctl_kprobes_optimization;
945 static int proc_kprobes_optimization_handler(struct ctl_table *table,
946                                              int write, void *buffer,
947                                              size_t *length, loff_t *ppos)
948 {
949         int ret;
950
951         mutex_lock(&kprobe_sysctl_mutex);
952         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
953         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
954
955         if (sysctl_kprobes_optimization)
956                 optimize_all_kprobes();
957         else
958                 unoptimize_all_kprobes();
959         mutex_unlock(&kprobe_sysctl_mutex);
960
961         return ret;
962 }
963
964 static struct ctl_table kprobe_sysctls[] = {
965         {
966                 .procname       = "kprobes-optimization",
967                 .data           = &sysctl_kprobes_optimization,
968                 .maxlen         = sizeof(int),
969                 .mode           = 0644,
970                 .proc_handler   = proc_kprobes_optimization_handler,
971                 .extra1         = SYSCTL_ZERO,
972                 .extra2         = SYSCTL_ONE,
973         },
974         {}
975 };
976
977 static void __init kprobe_sysctls_init(void)
978 {
979         register_sysctl_init("debug", kprobe_sysctls);
980 }
981 #endif /* CONFIG_SYSCTL */
982
983 /* Put a breakpoint for a probe. */
984 static void __arm_kprobe(struct kprobe *p)
985 {
986         struct kprobe *_p;
987
988         lockdep_assert_held(&text_mutex);
989
990         /* Find the overlapping optimized kprobes. */
991         _p = get_optimized_kprobe(p->addr);
992         if (unlikely(_p))
993                 /* Fallback to unoptimized kprobe */
994                 unoptimize_kprobe(_p, true);
995
996         arch_arm_kprobe(p);
997         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
998 }
999
1000 /* Remove the breakpoint of a probe. */
1001 static void __disarm_kprobe(struct kprobe *p, bool reopt)
1002 {
1003         struct kprobe *_p;
1004
1005         lockdep_assert_held(&text_mutex);
1006
1007         /* Try to unoptimize */
1008         unoptimize_kprobe(p, kprobes_all_disarmed);
1009
1010         if (!kprobe_queued(p)) {
1011                 arch_disarm_kprobe(p);
1012                 /* If another kprobe was blocked, re-optimize it. */
1013                 _p = get_optimized_kprobe(p->addr);
1014                 if (unlikely(_p) && reopt)
1015                         optimize_kprobe(_p);
1016         }
1017         /*
1018          * TODO: Since unoptimization and real disarming will be done by
1019          * the worker thread, we can not check whether another probe are
1020          * unoptimized because of this probe here. It should be re-optimized
1021          * by the worker thread.
1022          */
1023 }
1024
1025 #else /* !CONFIG_OPTPROBES */
1026
1027 #define optimize_kprobe(p)                      do {} while (0)
1028 #define unoptimize_kprobe(p, f)                 do {} while (0)
1029 #define kill_optimized_kprobe(p)                do {} while (0)
1030 #define prepare_optimized_kprobe(p)             do {} while (0)
1031 #define try_to_optimize_kprobe(p)               do {} while (0)
1032 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
1033 #define __disarm_kprobe(p, o)                   arch_disarm_kprobe(p)
1034 #define kprobe_disarmed(p)                      kprobe_disabled(p)
1035 #define wait_for_kprobe_optimizer()             do {} while (0)
1036
1037 static int reuse_unused_kprobe(struct kprobe *ap)
1038 {
1039         /*
1040          * If the optimized kprobe is NOT supported, the aggr kprobe is
1041          * released at the same time that the last aggregated kprobe is
1042          * unregistered.
1043          * Thus there should be no chance to reuse unused kprobe.
1044          */
1045         WARN_ON_ONCE(1);
1046         return -EINVAL;
1047 }
1048
1049 static void free_aggr_kprobe(struct kprobe *p)
1050 {
1051         arch_remove_kprobe(p);
1052         kfree(p);
1053 }
1054
1055 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
1056 {
1057         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
1058 }
1059 #endif /* CONFIG_OPTPROBES */
1060
1061 #ifdef CONFIG_KPROBES_ON_FTRACE
1062 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1063         .func = kprobe_ftrace_handler,
1064         .flags = FTRACE_OPS_FL_SAVE_REGS,
1065 };
1066
1067 static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1068         .func = kprobe_ftrace_handler,
1069         .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
1070 };
1071
1072 static int kprobe_ipmodify_enabled;
1073 static int kprobe_ftrace_enabled;
1074
1075 static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1076                                int *cnt)
1077 {
1078         int ret = 0;
1079
1080         lockdep_assert_held(&kprobe_mutex);
1081
1082         ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1083         if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret))
1084                 return ret;
1085
1086         if (*cnt == 0) {
1087                 ret = register_ftrace_function(ops);
1088                 if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret))
1089                         goto err_ftrace;
1090         }
1091
1092         (*cnt)++;
1093         return ret;
1094
1095 err_ftrace:
1096         /*
1097          * At this point, sinec ops is not registered, we should be sefe from
1098          * registering empty filter.
1099          */
1100         ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1101         return ret;
1102 }
1103
1104 static int arm_kprobe_ftrace(struct kprobe *p)
1105 {
1106         bool ipmodify = (p->post_handler != NULL);
1107
1108         return __arm_kprobe_ftrace(p,
1109                 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1110                 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1111 }
1112
1113 static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1114                                   int *cnt)
1115 {
1116         int ret = 0;
1117
1118         lockdep_assert_held(&kprobe_mutex);
1119
1120         if (*cnt == 1) {
1121                 ret = unregister_ftrace_function(ops);
1122                 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1123                         return ret;
1124         }
1125
1126         (*cnt)--;
1127
1128         ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1129         WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1130                   p->addr, ret);
1131         return ret;
1132 }
1133
1134 static int disarm_kprobe_ftrace(struct kprobe *p)
1135 {
1136         bool ipmodify = (p->post_handler != NULL);
1137
1138         return __disarm_kprobe_ftrace(p,
1139                 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1140                 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1141 }
1142 #else   /* !CONFIG_KPROBES_ON_FTRACE */
1143 static inline int arm_kprobe_ftrace(struct kprobe *p)
1144 {
1145         return -ENODEV;
1146 }
1147
1148 static inline int disarm_kprobe_ftrace(struct kprobe *p)
1149 {
1150         return -ENODEV;
1151 }
1152 #endif
1153
1154 static int prepare_kprobe(struct kprobe *p)
1155 {
1156         /* Must ensure p->addr is really on ftrace */
1157         if (kprobe_ftrace(p))
1158                 return arch_prepare_kprobe_ftrace(p);
1159
1160         return arch_prepare_kprobe(p);
1161 }
1162
1163 static int arm_kprobe(struct kprobe *kp)
1164 {
1165         if (unlikely(kprobe_ftrace(kp)))
1166                 return arm_kprobe_ftrace(kp);
1167
1168         cpus_read_lock();
1169         mutex_lock(&text_mutex);
1170         __arm_kprobe(kp);
1171         mutex_unlock(&text_mutex);
1172         cpus_read_unlock();
1173
1174         return 0;
1175 }
1176
1177 static int disarm_kprobe(struct kprobe *kp, bool reopt)
1178 {
1179         if (unlikely(kprobe_ftrace(kp)))
1180                 return disarm_kprobe_ftrace(kp);
1181
1182         cpus_read_lock();
1183         mutex_lock(&text_mutex);
1184         __disarm_kprobe(kp, reopt);
1185         mutex_unlock(&text_mutex);
1186         cpus_read_unlock();
1187
1188         return 0;
1189 }
1190
1191 /*
1192  * Aggregate handlers for multiple kprobes support - these handlers
1193  * take care of invoking the individual kprobe handlers on p->list
1194  */
1195 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1196 {
1197         struct kprobe *kp;
1198
1199         list_for_each_entry_rcu(kp, &p->list, list) {
1200                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1201                         set_kprobe_instance(kp);
1202                         if (kp->pre_handler(kp, regs))
1203                                 return 1;
1204                 }
1205                 reset_kprobe_instance();
1206         }
1207         return 0;
1208 }
1209 NOKPROBE_SYMBOL(aggr_pre_handler);
1210
1211 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1212                               unsigned long flags)
1213 {
1214         struct kprobe *kp;
1215
1216         list_for_each_entry_rcu(kp, &p->list, list) {
1217                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1218                         set_kprobe_instance(kp);
1219                         kp->post_handler(kp, regs, flags);
1220                         reset_kprobe_instance();
1221                 }
1222         }
1223 }
1224 NOKPROBE_SYMBOL(aggr_post_handler);
1225
1226 /* Walks the list and increments 'nmissed' if 'p' has child probes. */
1227 void kprobes_inc_nmissed_count(struct kprobe *p)
1228 {
1229         struct kprobe *kp;
1230
1231         if (!kprobe_aggrprobe(p)) {
1232                 p->nmissed++;
1233         } else {
1234                 list_for_each_entry_rcu(kp, &p->list, list)
1235                         kp->nmissed++;
1236         }
1237 }
1238 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1239
1240 static void free_rp_inst_rcu(struct rcu_head *head)
1241 {
1242         struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
1243
1244         if (refcount_dec_and_test(&ri->rph->ref))
1245                 kfree(ri->rph);
1246         kfree(ri);
1247 }
1248 NOKPROBE_SYMBOL(free_rp_inst_rcu);
1249
1250 static void recycle_rp_inst(struct kretprobe_instance *ri)
1251 {
1252         struct kretprobe *rp = get_kretprobe(ri);
1253
1254         if (likely(rp))
1255                 freelist_add(&ri->freelist, &rp->freelist);
1256         else
1257                 call_rcu(&ri->rcu, free_rp_inst_rcu);
1258 }
1259 NOKPROBE_SYMBOL(recycle_rp_inst);
1260
1261 static struct kprobe kprobe_busy = {
1262         .addr = (void *) get_kprobe,
1263 };
1264
1265 void kprobe_busy_begin(void)
1266 {
1267         struct kprobe_ctlblk *kcb;
1268
1269         preempt_disable();
1270         __this_cpu_write(current_kprobe, &kprobe_busy);
1271         kcb = get_kprobe_ctlblk();
1272         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1273 }
1274
1275 void kprobe_busy_end(void)
1276 {
1277         __this_cpu_write(current_kprobe, NULL);
1278         preempt_enable();
1279 }
1280
1281 /*
1282  * This function is called from delayed_put_task_struct() when a task is
1283  * dead and cleaned up to recycle any kretprobe instances associated with
1284  * this task. These left over instances represent probed functions that
1285  * have been called but will never return.
1286  */
1287 void kprobe_flush_task(struct task_struct *tk)
1288 {
1289         struct kretprobe_instance *ri;
1290         struct llist_node *node;
1291
1292         /* Early boot, not yet initialized. */
1293         if (unlikely(!kprobes_initialized))
1294                 return;
1295
1296         kprobe_busy_begin();
1297
1298         node = __llist_del_all(&tk->kretprobe_instances);
1299         while (node) {
1300                 ri = container_of(node, struct kretprobe_instance, llist);
1301                 node = node->next;
1302
1303                 recycle_rp_inst(ri);
1304         }
1305
1306         kprobe_busy_end();
1307 }
1308 NOKPROBE_SYMBOL(kprobe_flush_task);
1309
1310 static inline void free_rp_inst(struct kretprobe *rp)
1311 {
1312         struct kretprobe_instance *ri;
1313         struct freelist_node *node;
1314         int count = 0;
1315
1316         node = rp->freelist.head;
1317         while (node) {
1318                 ri = container_of(node, struct kretprobe_instance, freelist);
1319                 node = node->next;
1320
1321                 kfree(ri);
1322                 count++;
1323         }
1324
1325         if (refcount_sub_and_test(count, &rp->rph->ref)) {
1326                 kfree(rp->rph);
1327                 rp->rph = NULL;
1328         }
1329 }
1330
1331 /* Add the new probe to 'ap->list'. */
1332 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1333 {
1334         if (p->post_handler)
1335                 unoptimize_kprobe(ap, true);    /* Fall back to normal kprobe */
1336
1337         list_add_rcu(&p->list, &ap->list);
1338         if (p->post_handler && !ap->post_handler)
1339                 ap->post_handler = aggr_post_handler;
1340
1341         return 0;
1342 }
1343
1344 /*
1345  * Fill in the required fields of the aggregator kprobe. Replace the
1346  * earlier kprobe in the hlist with the aggregator kprobe.
1347  */
1348 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1349 {
1350         /* Copy the insn slot of 'p' to 'ap'. */
1351         copy_kprobe(p, ap);
1352         flush_insn_slot(ap);
1353         ap->addr = p->addr;
1354         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1355         ap->pre_handler = aggr_pre_handler;
1356         /* We don't care the kprobe which has gone. */
1357         if (p->post_handler && !kprobe_gone(p))
1358                 ap->post_handler = aggr_post_handler;
1359
1360         INIT_LIST_HEAD(&ap->list);
1361         INIT_HLIST_NODE(&ap->hlist);
1362
1363         list_add_rcu(&p->list, &ap->list);
1364         hlist_replace_rcu(&p->hlist, &ap->hlist);
1365 }
1366
1367 /*
1368  * This registers the second or subsequent kprobe at the same address.
1369  */
1370 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1371 {
1372         int ret = 0;
1373         struct kprobe *ap = orig_p;
1374
1375         cpus_read_lock();
1376
1377         /* For preparing optimization, jump_label_text_reserved() is called */
1378         jump_label_lock();
1379         mutex_lock(&text_mutex);
1380
1381         if (!kprobe_aggrprobe(orig_p)) {
1382                 /* If 'orig_p' is not an 'aggr_kprobe', create new one. */
1383                 ap = alloc_aggr_kprobe(orig_p);
1384                 if (!ap) {
1385                         ret = -ENOMEM;
1386                         goto out;
1387                 }
1388                 init_aggr_kprobe(ap, orig_p);
1389         } else if (kprobe_unused(ap)) {
1390                 /* This probe is going to die. Rescue it */
1391                 ret = reuse_unused_kprobe(ap);
1392                 if (ret)
1393                         goto out;
1394         }
1395
1396         if (kprobe_gone(ap)) {
1397                 /*
1398                  * Attempting to insert new probe at the same location that
1399                  * had a probe in the module vaddr area which already
1400                  * freed. So, the instruction slot has already been
1401                  * released. We need a new slot for the new probe.
1402                  */
1403                 ret = arch_prepare_kprobe(ap);
1404                 if (ret)
1405                         /*
1406                          * Even if fail to allocate new slot, don't need to
1407                          * free the 'ap'. It will be used next time, or
1408                          * freed by unregister_kprobe().
1409                          */
1410                         goto out;
1411
1412                 /* Prepare optimized instructions if possible. */
1413                 prepare_optimized_kprobe(ap);
1414
1415                 /*
1416                  * Clear gone flag to prevent allocating new slot again, and
1417                  * set disabled flag because it is not armed yet.
1418                  */
1419                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1420                             | KPROBE_FLAG_DISABLED;
1421         }
1422
1423         /* Copy the insn slot of 'p' to 'ap'. */
1424         copy_kprobe(ap, p);
1425         ret = add_new_kprobe(ap, p);
1426
1427 out:
1428         mutex_unlock(&text_mutex);
1429         jump_label_unlock();
1430         cpus_read_unlock();
1431
1432         if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1433                 ap->flags &= ~KPROBE_FLAG_DISABLED;
1434                 if (!kprobes_all_disarmed) {
1435                         /* Arm the breakpoint again. */
1436                         ret = arm_kprobe(ap);
1437                         if (ret) {
1438                                 ap->flags |= KPROBE_FLAG_DISABLED;
1439                                 list_del_rcu(&p->list);
1440                                 synchronize_rcu();
1441                         }
1442                 }
1443         }
1444         return ret;
1445 }
1446
1447 bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1448 {
1449         /* The '__kprobes' functions and entry code must not be probed. */
1450         return addr >= (unsigned long)__kprobes_text_start &&
1451                addr < (unsigned long)__kprobes_text_end;
1452 }
1453
1454 static bool __within_kprobe_blacklist(unsigned long addr)
1455 {
1456         struct kprobe_blacklist_entry *ent;
1457
1458         if (arch_within_kprobe_blacklist(addr))
1459                 return true;
1460         /*
1461          * If 'kprobe_blacklist' is defined, check the address and
1462          * reject any probe registration in the prohibited area.
1463          */
1464         list_for_each_entry(ent, &kprobe_blacklist, list) {
1465                 if (addr >= ent->start_addr && addr < ent->end_addr)
1466                         return true;
1467         }
1468         return false;
1469 }
1470
1471 bool within_kprobe_blacklist(unsigned long addr)
1472 {
1473         char symname[KSYM_NAME_LEN], *p;
1474
1475         if (__within_kprobe_blacklist(addr))
1476                 return true;
1477
1478         /* Check if the address is on a suffixed-symbol */
1479         if (!lookup_symbol_name(addr, symname)) {
1480                 p = strchr(symname, '.');
1481                 if (!p)
1482                         return false;
1483                 *p = '\0';
1484                 addr = (unsigned long)kprobe_lookup_name(symname, 0);
1485                 if (addr)
1486                         return __within_kprobe_blacklist(addr);
1487         }
1488         return false;
1489 }
1490
1491 /*
1492  * If 'symbol_name' is specified, look it up and add the 'offset'
1493  * to it. This way, we can specify a relative address to a symbol.
1494  * This returns encoded errors if it fails to look up symbol or invalid
1495  * combination of parameters.
1496  */
1497 static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1498                         const char *symbol_name, unsigned int offset)
1499 {
1500         if ((symbol_name && addr) || (!symbol_name && !addr))
1501                 goto invalid;
1502
1503         if (symbol_name) {
1504                 addr = kprobe_lookup_name(symbol_name, offset);
1505                 if (!addr)
1506                         return ERR_PTR(-ENOENT);
1507         }
1508
1509         addr = (kprobe_opcode_t *)(((char *)addr) + offset);
1510         if (addr)
1511                 return addr;
1512
1513 invalid:
1514         return ERR_PTR(-EINVAL);
1515 }
1516
1517 static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1518 {
1519         return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1520 }
1521
1522 /*
1523  * Check the 'p' is valid and return the aggregator kprobe
1524  * at the same address.
1525  */
1526 static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1527 {
1528         struct kprobe *ap, *list_p;
1529
1530         lockdep_assert_held(&kprobe_mutex);
1531
1532         ap = get_kprobe(p->addr);
1533         if (unlikely(!ap))
1534                 return NULL;
1535
1536         if (p != ap) {
1537                 list_for_each_entry(list_p, &ap->list, list)
1538                         if (list_p == p)
1539                         /* kprobe p is a valid probe */
1540                                 goto valid;
1541                 return NULL;
1542         }
1543 valid:
1544         return ap;
1545 }
1546
1547 /*
1548  * Warn and return error if the kprobe is being re-registered since
1549  * there must be a software bug.
1550  */
1551 static inline int warn_kprobe_rereg(struct kprobe *p)
1552 {
1553         int ret = 0;
1554
1555         mutex_lock(&kprobe_mutex);
1556         if (WARN_ON_ONCE(__get_valid_kprobe(p)))
1557                 ret = -EINVAL;
1558         mutex_unlock(&kprobe_mutex);
1559
1560         return ret;
1561 }
1562
1563 static int check_ftrace_location(struct kprobe *p)
1564 {
1565         unsigned long ftrace_addr;
1566
1567         ftrace_addr = ftrace_location((unsigned long)p->addr);
1568         if (ftrace_addr) {
1569 #ifdef CONFIG_KPROBES_ON_FTRACE
1570                 /* Given address is not on the instruction boundary */
1571                 if ((unsigned long)p->addr != ftrace_addr)
1572                         return -EILSEQ;
1573                 p->flags |= KPROBE_FLAG_FTRACE;
1574 #else   /* !CONFIG_KPROBES_ON_FTRACE */
1575                 return -EINVAL;
1576 #endif
1577         }
1578         return 0;
1579 }
1580
1581 static int check_kprobe_address_safe(struct kprobe *p,
1582                                      struct module **probed_mod)
1583 {
1584         int ret;
1585
1586         ret = check_ftrace_location(p);
1587         if (ret)
1588                 return ret;
1589         jump_label_lock();
1590         preempt_disable();
1591
1592         /* Ensure it is not in reserved area nor out of text */
1593         if (!kernel_text_address((unsigned long) p->addr) ||
1594             within_kprobe_blacklist((unsigned long) p->addr) ||
1595             jump_label_text_reserved(p->addr, p->addr) ||
1596             static_call_text_reserved(p->addr, p->addr) ||
1597             find_bug((unsigned long)p->addr)) {
1598                 ret = -EINVAL;
1599                 goto out;
1600         }
1601
1602         /* Check if 'p' is probing a module. */
1603         *probed_mod = __module_text_address((unsigned long) p->addr);
1604         if (*probed_mod) {
1605                 /*
1606                  * We must hold a refcount of the probed module while updating
1607                  * its code to prohibit unexpected unloading.
1608                  */
1609                 if (unlikely(!try_module_get(*probed_mod))) {
1610                         ret = -ENOENT;
1611                         goto out;
1612                 }
1613
1614                 /*
1615                  * If the module freed '.init.text', we couldn't insert
1616                  * kprobes in there.
1617                  */
1618                 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1619                     (*probed_mod)->state != MODULE_STATE_COMING) {
1620                         module_put(*probed_mod);
1621                         *probed_mod = NULL;
1622                         ret = -ENOENT;
1623                 }
1624         }
1625 out:
1626         preempt_enable();
1627         jump_label_unlock();
1628
1629         return ret;
1630 }
1631
1632 int register_kprobe(struct kprobe *p)
1633 {
1634         int ret;
1635         struct kprobe *old_p;
1636         struct module *probed_mod;
1637         kprobe_opcode_t *addr;
1638
1639         /* Adjust probe address from symbol */
1640         addr = kprobe_addr(p);
1641         if (IS_ERR(addr))
1642                 return PTR_ERR(addr);
1643         p->addr = addr;
1644
1645         ret = warn_kprobe_rereg(p);
1646         if (ret)
1647                 return ret;
1648
1649         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1650         p->flags &= KPROBE_FLAG_DISABLED;
1651         p->nmissed = 0;
1652         INIT_LIST_HEAD(&p->list);
1653
1654         ret = check_kprobe_address_safe(p, &probed_mod);
1655         if (ret)
1656                 return ret;
1657
1658         mutex_lock(&kprobe_mutex);
1659
1660         old_p = get_kprobe(p->addr);
1661         if (old_p) {
1662                 /* Since this may unoptimize 'old_p', locking 'text_mutex'. */
1663                 ret = register_aggr_kprobe(old_p, p);
1664                 goto out;
1665         }
1666
1667         cpus_read_lock();
1668         /* Prevent text modification */
1669         mutex_lock(&text_mutex);
1670         ret = prepare_kprobe(p);
1671         mutex_unlock(&text_mutex);
1672         cpus_read_unlock();
1673         if (ret)
1674                 goto out;
1675
1676         INIT_HLIST_NODE(&p->hlist);
1677         hlist_add_head_rcu(&p->hlist,
1678                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1679
1680         if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1681                 ret = arm_kprobe(p);
1682                 if (ret) {
1683                         hlist_del_rcu(&p->hlist);
1684                         synchronize_rcu();
1685                         goto out;
1686                 }
1687         }
1688
1689         /* Try to optimize kprobe */
1690         try_to_optimize_kprobe(p);
1691 out:
1692         mutex_unlock(&kprobe_mutex);
1693
1694         if (probed_mod)
1695                 module_put(probed_mod);
1696
1697         return ret;
1698 }
1699 EXPORT_SYMBOL_GPL(register_kprobe);
1700
1701 /* Check if all probes on the 'ap' are disabled. */
1702 static bool aggr_kprobe_disabled(struct kprobe *ap)
1703 {
1704         struct kprobe *kp;
1705
1706         lockdep_assert_held(&kprobe_mutex);
1707
1708         list_for_each_entry(kp, &ap->list, list)
1709                 if (!kprobe_disabled(kp))
1710                         /*
1711                          * Since there is an active probe on the list,
1712                          * we can't disable this 'ap'.
1713                          */
1714                         return false;
1715
1716         return true;
1717 }
1718
1719 static struct kprobe *__disable_kprobe(struct kprobe *p)
1720 {
1721         struct kprobe *orig_p;
1722         int ret;
1723
1724         lockdep_assert_held(&kprobe_mutex);
1725
1726         /* Get an original kprobe for return */
1727         orig_p = __get_valid_kprobe(p);
1728         if (unlikely(orig_p == NULL))
1729                 return ERR_PTR(-EINVAL);
1730
1731         if (!kprobe_disabled(p)) {
1732                 /* Disable probe if it is a child probe */
1733                 if (p != orig_p)
1734                         p->flags |= KPROBE_FLAG_DISABLED;
1735
1736                 /* Try to disarm and disable this/parent probe */
1737                 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
1738                         /*
1739                          * If 'kprobes_all_disarmed' is set, 'orig_p'
1740                          * should have already been disarmed, so
1741                          * skip unneed disarming process.
1742                          */
1743                         if (!kprobes_all_disarmed) {
1744                                 ret = disarm_kprobe(orig_p, true);
1745                                 if (ret) {
1746                                         p->flags &= ~KPROBE_FLAG_DISABLED;
1747                                         return ERR_PTR(ret);
1748                                 }
1749                         }
1750                         orig_p->flags |= KPROBE_FLAG_DISABLED;
1751                 }
1752         }
1753
1754         return orig_p;
1755 }
1756
1757 /*
1758  * Unregister a kprobe without a scheduler synchronization.
1759  */
1760 static int __unregister_kprobe_top(struct kprobe *p)
1761 {
1762         struct kprobe *ap, *list_p;
1763
1764         /* Disable kprobe. This will disarm it if needed. */
1765         ap = __disable_kprobe(p);
1766         if (IS_ERR(ap))
1767                 return PTR_ERR(ap);
1768
1769         if (ap == p)
1770                 /*
1771                  * This probe is an independent(and non-optimized) kprobe
1772                  * (not an aggrprobe). Remove from the hash list.
1773                  */
1774                 goto disarmed;
1775
1776         /* Following process expects this probe is an aggrprobe */
1777         WARN_ON(!kprobe_aggrprobe(ap));
1778
1779         if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1780                 /*
1781                  * !disarmed could be happen if the probe is under delayed
1782                  * unoptimizing.
1783                  */
1784                 goto disarmed;
1785         else {
1786                 /* If disabling probe has special handlers, update aggrprobe */
1787                 if (p->post_handler && !kprobe_gone(p)) {
1788                         list_for_each_entry(list_p, &ap->list, list) {
1789                                 if ((list_p != p) && (list_p->post_handler))
1790                                         goto noclean;
1791                         }
1792                         ap->post_handler = NULL;
1793                 }
1794 noclean:
1795                 /*
1796                  * Remove from the aggrprobe: this path will do nothing in
1797                  * __unregister_kprobe_bottom().
1798                  */
1799                 list_del_rcu(&p->list);
1800                 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1801                         /*
1802                          * Try to optimize this probe again, because post
1803                          * handler may have been changed.
1804                          */
1805                         optimize_kprobe(ap);
1806         }
1807         return 0;
1808
1809 disarmed:
1810         hlist_del_rcu(&ap->hlist);
1811         return 0;
1812 }
1813
1814 static void __unregister_kprobe_bottom(struct kprobe *p)
1815 {
1816         struct kprobe *ap;
1817
1818         if (list_empty(&p->list))
1819                 /* This is an independent kprobe */
1820                 arch_remove_kprobe(p);
1821         else if (list_is_singular(&p->list)) {
1822                 /* This is the last child of an aggrprobe */
1823                 ap = list_entry(p->list.next, struct kprobe, list);
1824                 list_del(&p->list);
1825                 free_aggr_kprobe(ap);
1826         }
1827         /* Otherwise, do nothing. */
1828 }
1829
1830 int register_kprobes(struct kprobe **kps, int num)
1831 {
1832         int i, ret = 0;
1833
1834         if (num <= 0)
1835                 return -EINVAL;
1836         for (i = 0; i < num; i++) {
1837                 ret = register_kprobe(kps[i]);
1838                 if (ret < 0) {
1839                         if (i > 0)
1840                                 unregister_kprobes(kps, i);
1841                         break;
1842                 }
1843         }
1844         return ret;
1845 }
1846 EXPORT_SYMBOL_GPL(register_kprobes);
1847
1848 void unregister_kprobe(struct kprobe *p)
1849 {
1850         unregister_kprobes(&p, 1);
1851 }
1852 EXPORT_SYMBOL_GPL(unregister_kprobe);
1853
1854 void unregister_kprobes(struct kprobe **kps, int num)
1855 {
1856         int i;
1857
1858         if (num <= 0)
1859                 return;
1860         mutex_lock(&kprobe_mutex);
1861         for (i = 0; i < num; i++)
1862                 if (__unregister_kprobe_top(kps[i]) < 0)
1863                         kps[i]->addr = NULL;
1864         mutex_unlock(&kprobe_mutex);
1865
1866         synchronize_rcu();
1867         for (i = 0; i < num; i++)
1868                 if (kps[i]->addr)
1869                         __unregister_kprobe_bottom(kps[i]);
1870 }
1871 EXPORT_SYMBOL_GPL(unregister_kprobes);
1872
1873 int __weak kprobe_exceptions_notify(struct notifier_block *self,
1874                                         unsigned long val, void *data)
1875 {
1876         return NOTIFY_DONE;
1877 }
1878 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1879
1880 static struct notifier_block kprobe_exceptions_nb = {
1881         .notifier_call = kprobe_exceptions_notify,
1882         .priority = 0x7fffffff /* we need to be notified first */
1883 };
1884
1885 #ifdef CONFIG_KRETPROBES
1886
1887 /* This assumes the 'tsk' is the current task or the is not running. */
1888 static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk,
1889                                                   struct llist_node **cur)
1890 {
1891         struct kretprobe_instance *ri = NULL;
1892         struct llist_node *node = *cur;
1893
1894         if (!node)
1895                 node = tsk->kretprobe_instances.first;
1896         else
1897                 node = node->next;
1898
1899         while (node) {
1900                 ri = container_of(node, struct kretprobe_instance, llist);
1901                 if (ri->ret_addr != kretprobe_trampoline_addr()) {
1902                         *cur = node;
1903                         return ri->ret_addr;
1904                 }
1905                 node = node->next;
1906         }
1907         return NULL;
1908 }
1909 NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
1910
1911 /**
1912  * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe
1913  * @tsk: Target task
1914  * @fp: A frame pointer
1915  * @cur: a storage of the loop cursor llist_node pointer for next call
1916  *
1917  * Find the correct return address modified by a kretprobe on @tsk in unsigned
1918  * long type. If it finds the return address, this returns that address value,
1919  * or this returns 0.
1920  * The @tsk must be 'current' or a task which is not running. @fp is a hint
1921  * to get the currect return address - which is compared with the
1922  * kretprobe_instance::fp field. The @cur is a loop cursor for searching the
1923  * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
1924  * first call, but '@cur' itself must NOT NULL.
1925  */
1926 unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
1927                                       struct llist_node **cur)
1928 {
1929         struct kretprobe_instance *ri = NULL;
1930         kprobe_opcode_t *ret;
1931
1932         if (WARN_ON_ONCE(!cur))
1933                 return 0;
1934
1935         do {
1936                 ret = __kretprobe_find_ret_addr(tsk, cur);
1937                 if (!ret)
1938                         break;
1939                 ri = container_of(*cur, struct kretprobe_instance, llist);
1940         } while (ri->fp != fp);
1941
1942         return (unsigned long)ret;
1943 }
1944 NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
1945
1946 void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
1947                                         kprobe_opcode_t *correct_ret_addr)
1948 {
1949         /*
1950          * Do nothing by default. Please fill this to update the fake return
1951          * address on the stack with the correct one on each arch if possible.
1952          */
1953 }
1954
1955 unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
1956                                              void *frame_pointer)
1957 {
1958         kprobe_opcode_t *correct_ret_addr = NULL;
1959         struct kretprobe_instance *ri = NULL;
1960         struct llist_node *first, *node = NULL;
1961         struct kretprobe *rp;
1962
1963         /* Find correct address and all nodes for this frame. */
1964         correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
1965         if (!correct_ret_addr) {
1966                 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
1967                 BUG_ON(1);
1968         }
1969
1970         /*
1971          * Set the return address as the instruction pointer, because if the
1972          * user handler calls stack_trace_save_regs() with this 'regs',
1973          * the stack trace will start from the instruction pointer.
1974          */
1975         instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
1976
1977         /* Run the user handler of the nodes. */
1978         first = current->kretprobe_instances.first;
1979         while (first) {
1980                 ri = container_of(first, struct kretprobe_instance, llist);
1981
1982                 if (WARN_ON_ONCE(ri->fp != frame_pointer))
1983                         break;
1984
1985                 rp = get_kretprobe(ri);
1986                 if (rp && rp->handler) {
1987                         struct kprobe *prev = kprobe_running();
1988
1989                         __this_cpu_write(current_kprobe, &rp->kp);
1990                         ri->ret_addr = correct_ret_addr;
1991                         rp->handler(ri, regs);
1992                         __this_cpu_write(current_kprobe, prev);
1993                 }
1994                 if (first == node)
1995                         break;
1996
1997                 first = first->next;
1998         }
1999
2000         arch_kretprobe_fixup_return(regs, correct_ret_addr);
2001
2002         /* Unlink all nodes for this frame. */
2003         first = current->kretprobe_instances.first;
2004         current->kretprobe_instances.first = node->next;
2005         node->next = NULL;
2006
2007         /* Recycle free instances. */
2008         while (first) {
2009                 ri = container_of(first, struct kretprobe_instance, llist);
2010                 first = first->next;
2011
2012                 recycle_rp_inst(ri);
2013         }
2014
2015         return (unsigned long)correct_ret_addr;
2016 }
2017 NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2018
2019 /*
2020  * This kprobe pre_handler is registered with every kretprobe. When probe
2021  * hits it will set up the return probe.
2022  */
2023 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2024 {
2025         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2026         struct kretprobe_instance *ri;
2027         struct freelist_node *fn;
2028
2029         fn = freelist_try_get(&rp->freelist);
2030         if (!fn) {
2031                 rp->nmissed++;
2032                 return 0;
2033         }
2034
2035         ri = container_of(fn, struct kretprobe_instance, freelist);
2036
2037         if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2038                 freelist_add(&ri->freelist, &rp->freelist);
2039                 return 0;
2040         }
2041
2042         arch_prepare_kretprobe(ri, regs);
2043
2044         __llist_add(&ri->llist, &current->kretprobe_instances);
2045
2046         return 0;
2047 }
2048 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2049
2050 bool __weak arch_kprobe_on_func_entry(unsigned long offset)
2051 {
2052         return !offset;
2053 }
2054
2055 /**
2056  * kprobe_on_func_entry() -- check whether given address is function entry
2057  * @addr: Target address
2058  * @sym:  Target symbol name
2059  * @offset: The offset from the symbol or the address
2060  *
2061  * This checks whether the given @addr+@offset or @sym+@offset is on the
2062  * function entry address or not.
2063  * This returns 0 if it is the function entry, or -EINVAL if it is not.
2064  * And also it returns -ENOENT if it fails the symbol or address lookup.
2065  * Caller must pass @addr or @sym (either one must be NULL), or this
2066  * returns -EINVAL.
2067  */
2068 int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
2069 {
2070         kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
2071
2072         if (IS_ERR(kp_addr))
2073                 return PTR_ERR(kp_addr);
2074
2075         if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset))
2076                 return -ENOENT;
2077
2078         if (!arch_kprobe_on_func_entry(offset))
2079                 return -EINVAL;
2080
2081         return 0;
2082 }
2083
2084 int register_kretprobe(struct kretprobe *rp)
2085 {
2086         int ret;
2087         struct kretprobe_instance *inst;
2088         int i;
2089         void *addr;
2090
2091         ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset);
2092         if (ret)
2093                 return ret;
2094
2095         /* If only 'rp->kp.addr' is specified, check reregistering kprobes */
2096         if (rp->kp.addr && warn_kprobe_rereg(&rp->kp))
2097                 return -EINVAL;
2098
2099         if (kretprobe_blacklist_size) {
2100                 addr = kprobe_addr(&rp->kp);
2101                 if (IS_ERR(addr))
2102                         return PTR_ERR(addr);
2103
2104                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2105                         if (kretprobe_blacklist[i].addr == addr)
2106                                 return -EINVAL;
2107                 }
2108         }
2109
2110         if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2111                 return -E2BIG;
2112
2113         rp->kp.pre_handler = pre_handler_kretprobe;
2114         rp->kp.post_handler = NULL;
2115
2116         /* Pre-allocate memory for max kretprobe instances */
2117         if (rp->maxactive <= 0) {
2118 #ifdef CONFIG_PREEMPTION
2119                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2120 #else
2121                 rp->maxactive = num_possible_cpus();
2122 #endif
2123         }
2124         rp->freelist.head = NULL;
2125         rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
2126         if (!rp->rph)
2127                 return -ENOMEM;
2128
2129         rp->rph->rp = rp;
2130         for (i = 0; i < rp->maxactive; i++) {
2131                 inst = kzalloc(sizeof(struct kretprobe_instance) +
2132                                rp->data_size, GFP_KERNEL);
2133                 if (inst == NULL) {
2134                         refcount_set(&rp->rph->ref, i);
2135                         free_rp_inst(rp);
2136                         return -ENOMEM;
2137                 }
2138                 inst->rph = rp->rph;
2139                 freelist_add(&inst->freelist, &rp->freelist);
2140         }
2141         refcount_set(&rp->rph->ref, i);
2142
2143         rp->nmissed = 0;
2144         /* Establish function entry probe point */
2145         ret = register_kprobe(&rp->kp);
2146         if (ret != 0)
2147                 free_rp_inst(rp);
2148         return ret;
2149 }
2150 EXPORT_SYMBOL_GPL(register_kretprobe);
2151
2152 int register_kretprobes(struct kretprobe **rps, int num)
2153 {
2154         int ret = 0, i;
2155
2156         if (num <= 0)
2157                 return -EINVAL;
2158         for (i = 0; i < num; i++) {
2159                 ret = register_kretprobe(rps[i]);
2160                 if (ret < 0) {
2161                         if (i > 0)
2162                                 unregister_kretprobes(rps, i);
2163                         break;
2164                 }
2165         }
2166         return ret;
2167 }
2168 EXPORT_SYMBOL_GPL(register_kretprobes);
2169
2170 void unregister_kretprobe(struct kretprobe *rp)
2171 {
2172         unregister_kretprobes(&rp, 1);
2173 }
2174 EXPORT_SYMBOL_GPL(unregister_kretprobe);
2175
2176 void unregister_kretprobes(struct kretprobe **rps, int num)
2177 {
2178         int i;
2179
2180         if (num <= 0)
2181                 return;
2182         mutex_lock(&kprobe_mutex);
2183         for (i = 0; i < num; i++) {
2184                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
2185                         rps[i]->kp.addr = NULL;
2186                 rps[i]->rph->rp = NULL;
2187         }
2188         mutex_unlock(&kprobe_mutex);
2189
2190         synchronize_rcu();
2191         for (i = 0; i < num; i++) {
2192                 if (rps[i]->kp.addr) {
2193                         __unregister_kprobe_bottom(&rps[i]->kp);
2194                         free_rp_inst(rps[i]);
2195                 }
2196         }
2197 }
2198 EXPORT_SYMBOL_GPL(unregister_kretprobes);
2199
2200 #else /* CONFIG_KRETPROBES */
2201 int register_kretprobe(struct kretprobe *rp)
2202 {
2203         return -EOPNOTSUPP;
2204 }
2205 EXPORT_SYMBOL_GPL(register_kretprobe);
2206
2207 int register_kretprobes(struct kretprobe **rps, int num)
2208 {
2209         return -EOPNOTSUPP;
2210 }
2211 EXPORT_SYMBOL_GPL(register_kretprobes);
2212
2213 void unregister_kretprobe(struct kretprobe *rp)
2214 {
2215 }
2216 EXPORT_SYMBOL_GPL(unregister_kretprobe);
2217
2218 void unregister_kretprobes(struct kretprobe **rps, int num)
2219 {
2220 }
2221 EXPORT_SYMBOL_GPL(unregister_kretprobes);
2222
2223 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2224 {
2225         return 0;
2226 }
2227 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2228
2229 #endif /* CONFIG_KRETPROBES */
2230
2231 /* Set the kprobe gone and remove its instruction buffer. */
2232 static void kill_kprobe(struct kprobe *p)
2233 {
2234         struct kprobe *kp;
2235
2236         lockdep_assert_held(&kprobe_mutex);
2237
2238         p->flags |= KPROBE_FLAG_GONE;
2239         if (kprobe_aggrprobe(p)) {
2240                 /*
2241                  * If this is an aggr_kprobe, we have to list all the
2242                  * chained probes and mark them GONE.
2243                  */
2244                 list_for_each_entry(kp, &p->list, list)
2245                         kp->flags |= KPROBE_FLAG_GONE;
2246                 p->post_handler = NULL;
2247                 kill_optimized_kprobe(p);
2248         }
2249         /*
2250          * Here, we can remove insn_slot safely, because no thread calls
2251          * the original probed function (which will be freed soon) any more.
2252          */
2253         arch_remove_kprobe(p);
2254
2255         /*
2256          * The module is going away. We should disarm the kprobe which
2257          * is using ftrace, because ftrace framework is still available at
2258          * 'MODULE_STATE_GOING' notification.
2259          */
2260         if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2261                 disarm_kprobe_ftrace(p);
2262 }
2263
2264 /* Disable one kprobe */
2265 int disable_kprobe(struct kprobe *kp)
2266 {
2267         int ret = 0;
2268         struct kprobe *p;
2269
2270         mutex_lock(&kprobe_mutex);
2271
2272         /* Disable this kprobe */
2273         p = __disable_kprobe(kp);
2274         if (IS_ERR(p))
2275                 ret = PTR_ERR(p);
2276
2277         mutex_unlock(&kprobe_mutex);
2278         return ret;
2279 }
2280 EXPORT_SYMBOL_GPL(disable_kprobe);
2281
2282 /* Enable one kprobe */
2283 int enable_kprobe(struct kprobe *kp)
2284 {
2285         int ret = 0;
2286         struct kprobe *p;
2287
2288         mutex_lock(&kprobe_mutex);
2289
2290         /* Check whether specified probe is valid. */
2291         p = __get_valid_kprobe(kp);
2292         if (unlikely(p == NULL)) {
2293                 ret = -EINVAL;
2294                 goto out;
2295         }
2296
2297         if (kprobe_gone(kp)) {
2298                 /* This kprobe has gone, we couldn't enable it. */
2299                 ret = -EINVAL;
2300                 goto out;
2301         }
2302
2303         if (p != kp)
2304                 kp->flags &= ~KPROBE_FLAG_DISABLED;
2305
2306         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2307                 p->flags &= ~KPROBE_FLAG_DISABLED;
2308                 ret = arm_kprobe(p);
2309                 if (ret)
2310                         p->flags |= KPROBE_FLAG_DISABLED;
2311         }
2312 out:
2313         mutex_unlock(&kprobe_mutex);
2314         return ret;
2315 }
2316 EXPORT_SYMBOL_GPL(enable_kprobe);
2317
2318 /* Caller must NOT call this in usual path. This is only for critical case */
2319 void dump_kprobe(struct kprobe *kp)
2320 {
2321         pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2322                kp->symbol_name, kp->offset, kp->addr);
2323 }
2324 NOKPROBE_SYMBOL(dump_kprobe);
2325
2326 int kprobe_add_ksym_blacklist(unsigned long entry)
2327 {
2328         struct kprobe_blacklist_entry *ent;
2329         unsigned long offset = 0, size = 0;
2330
2331         if (!kernel_text_address(entry) ||
2332             !kallsyms_lookup_size_offset(entry, &size, &offset))
2333                 return -EINVAL;
2334
2335         ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2336         if (!ent)
2337                 return -ENOMEM;
2338         ent->start_addr = entry;
2339         ent->end_addr = entry + size;
2340         INIT_LIST_HEAD(&ent->list);
2341         list_add_tail(&ent->list, &kprobe_blacklist);
2342
2343         return (int)size;
2344 }
2345
2346 /* Add all symbols in given area into kprobe blacklist */
2347 int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2348 {
2349         unsigned long entry;
2350         int ret = 0;
2351
2352         for (entry = start; entry < end; entry += ret) {
2353                 ret = kprobe_add_ksym_blacklist(entry);
2354                 if (ret < 0)
2355                         return ret;
2356                 if (ret == 0)   /* In case of alias symbol */
2357                         ret = 1;
2358         }
2359         return 0;
2360 }
2361
2362 /* Remove all symbols in given area from kprobe blacklist */
2363 static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2364 {
2365         struct kprobe_blacklist_entry *ent, *n;
2366
2367         list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2368                 if (ent->start_addr < start || ent->start_addr >= end)
2369                         continue;
2370                 list_del(&ent->list);
2371                 kfree(ent);
2372         }
2373 }
2374
2375 static void kprobe_remove_ksym_blacklist(unsigned long entry)
2376 {
2377         kprobe_remove_area_blacklist(entry, entry + 1);
2378 }
2379
2380 int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
2381                                    char *type, char *sym)
2382 {
2383         return -ERANGE;
2384 }
2385
2386 int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2387                        char *sym)
2388 {
2389 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2390         if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2391                 return 0;
2392 #ifdef CONFIG_OPTPROBES
2393         if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2394                 return 0;
2395 #endif
2396 #endif
2397         if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2398                 return 0;
2399         return -ERANGE;
2400 }
2401
2402 int __init __weak arch_populate_kprobe_blacklist(void)
2403 {
2404         return 0;
2405 }
2406
2407 /*
2408  * Lookup and populate the kprobe_blacklist.
2409  *
2410  * Unlike the kretprobe blacklist, we'll need to determine
2411  * the range of addresses that belong to the said functions,
2412  * since a kprobe need not necessarily be at the beginning
2413  * of a function.
2414  */
2415 static int __init populate_kprobe_blacklist(unsigned long *start,
2416                                              unsigned long *end)
2417 {
2418         unsigned long entry;
2419         unsigned long *iter;
2420         int ret;
2421
2422         for (iter = start; iter < end; iter++) {
2423                 entry = (unsigned long)dereference_symbol_descriptor((void *)*iter);
2424                 ret = kprobe_add_ksym_blacklist(entry);
2425                 if (ret == -EINVAL)
2426                         continue;
2427                 if (ret < 0)
2428                         return ret;
2429         }
2430
2431         /* Symbols in '__kprobes_text' are blacklisted */
2432         ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2433                                         (unsigned long)__kprobes_text_end);
2434         if (ret)
2435                 return ret;
2436
2437         /* Symbols in 'noinstr' section are blacklisted */
2438         ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2439                                         (unsigned long)__noinstr_text_end);
2440
2441         return ret ? : arch_populate_kprobe_blacklist();
2442 }
2443
2444 static void add_module_kprobe_blacklist(struct module *mod)
2445 {
2446         unsigned long start, end;
2447         int i;
2448
2449         if (mod->kprobe_blacklist) {
2450                 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2451                         kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2452         }
2453
2454         start = (unsigned long)mod->kprobes_text_start;
2455         if (start) {
2456                 end = start + mod->kprobes_text_size;
2457                 kprobe_add_area_blacklist(start, end);
2458         }
2459
2460         start = (unsigned long)mod->noinstr_text_start;
2461         if (start) {
2462                 end = start + mod->noinstr_text_size;
2463                 kprobe_add_area_blacklist(start, end);
2464         }
2465 }
2466
2467 static void remove_module_kprobe_blacklist(struct module *mod)
2468 {
2469         unsigned long start, end;
2470         int i;
2471
2472         if (mod->kprobe_blacklist) {
2473                 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2474                         kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2475         }
2476
2477         start = (unsigned long)mod->kprobes_text_start;
2478         if (start) {
2479                 end = start + mod->kprobes_text_size;
2480                 kprobe_remove_area_blacklist(start, end);
2481         }
2482
2483         start = (unsigned long)mod->noinstr_text_start;
2484         if (start) {
2485                 end = start + mod->noinstr_text_size;
2486                 kprobe_remove_area_blacklist(start, end);
2487         }
2488 }
2489
2490 /* Module notifier call back, checking kprobes on the module */
2491 static int kprobes_module_callback(struct notifier_block *nb,
2492                                    unsigned long val, void *data)
2493 {
2494         struct module *mod = data;
2495         struct hlist_head *head;
2496         struct kprobe *p;
2497         unsigned int i;
2498         int checkcore = (val == MODULE_STATE_GOING);
2499
2500         if (val == MODULE_STATE_COMING) {
2501                 mutex_lock(&kprobe_mutex);
2502                 add_module_kprobe_blacklist(mod);
2503                 mutex_unlock(&kprobe_mutex);
2504         }
2505         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2506                 return NOTIFY_DONE;
2507
2508         /*
2509          * When 'MODULE_STATE_GOING' was notified, both of module '.text' and
2510          * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was
2511          * notified, only '.init.text' section would be freed. We need to
2512          * disable kprobes which have been inserted in the sections.
2513          */
2514         mutex_lock(&kprobe_mutex);
2515         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2516                 head = &kprobe_table[i];
2517                 hlist_for_each_entry(p, head, hlist)
2518                         if (within_module_init((unsigned long)p->addr, mod) ||
2519                             (checkcore &&
2520                              within_module_core((unsigned long)p->addr, mod))) {
2521                                 /*
2522                                  * The vaddr this probe is installed will soon
2523                                  * be vfreed buy not synced to disk. Hence,
2524                                  * disarming the breakpoint isn't needed.
2525                                  *
2526                                  * Note, this will also move any optimized probes
2527                                  * that are pending to be removed from their
2528                                  * corresponding lists to the 'freeing_list' and
2529                                  * will not be touched by the delayed
2530                                  * kprobe_optimizer() work handler.
2531                                  */
2532                                 kill_kprobe(p);
2533                         }
2534         }
2535         if (val == MODULE_STATE_GOING)
2536                 remove_module_kprobe_blacklist(mod);
2537         mutex_unlock(&kprobe_mutex);
2538         return NOTIFY_DONE;
2539 }
2540
2541 static struct notifier_block kprobe_module_nb = {
2542         .notifier_call = kprobes_module_callback,
2543         .priority = 0
2544 };
2545
2546 void kprobe_free_init_mem(void)
2547 {
2548         void *start = (void *)(&__init_begin);
2549         void *end = (void *)(&__init_end);
2550         struct hlist_head *head;
2551         struct kprobe *p;
2552         int i;
2553
2554         mutex_lock(&kprobe_mutex);
2555
2556         /* Kill all kprobes on initmem because the target code has been freed. */
2557         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2558                 head = &kprobe_table[i];
2559                 hlist_for_each_entry(p, head, hlist) {
2560                         if (start <= (void *)p->addr && (void *)p->addr < end)
2561                                 kill_kprobe(p);
2562                 }
2563         }
2564
2565         mutex_unlock(&kprobe_mutex);
2566 }
2567
2568 static int __init init_kprobes(void)
2569 {
2570         int i, err = 0;
2571
2572         /* FIXME allocate the probe table, currently defined statically */
2573         /* initialize all list heads */
2574         for (i = 0; i < KPROBE_TABLE_SIZE; i++)
2575                 INIT_HLIST_HEAD(&kprobe_table[i]);
2576
2577         err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2578                                         __stop_kprobe_blacklist);
2579         if (err)
2580                 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
2581
2582         if (kretprobe_blacklist_size) {
2583                 /* lookup the function address from its name */
2584                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2585                         kretprobe_blacklist[i].addr =
2586                                 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
2587                         if (!kretprobe_blacklist[i].addr)
2588                                 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
2589                                        kretprobe_blacklist[i].name);
2590                 }
2591         }
2592
2593         /* By default, kprobes are armed */
2594         kprobes_all_disarmed = false;
2595
2596 #if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2597         /* Init 'kprobe_optinsn_slots' for allocation */
2598         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2599 #endif
2600
2601         err = arch_init_kprobes();
2602         if (!err)
2603                 err = register_die_notifier(&kprobe_exceptions_nb);
2604         if (!err)
2605                 err = register_module_notifier(&kprobe_module_nb);
2606
2607         kprobes_initialized = (err == 0);
2608         kprobe_sysctls_init();
2609         return err;
2610 }
2611 early_initcall(init_kprobes);
2612
2613 #if defined(CONFIG_OPTPROBES)
2614 static int __init init_optprobes(void)
2615 {
2616         /*
2617          * Enable kprobe optimization - this kicks the optimizer which
2618          * depends on synchronize_rcu_tasks() and ksoftirqd, that is
2619          * not spawned in early initcall. So delay the optimization.
2620          */
2621         optimize_all_kprobes();
2622
2623         return 0;
2624 }
2625 subsys_initcall(init_optprobes);
2626 #endif
2627
2628 #ifdef CONFIG_DEBUG_FS
2629 static void report_probe(struct seq_file *pi, struct kprobe *p,
2630                 const char *sym, int offset, char *modname, struct kprobe *pp)
2631 {
2632         char *kprobe_type;
2633         void *addr = p->addr;
2634
2635         if (p->pre_handler == pre_handler_kretprobe)
2636                 kprobe_type = "r";
2637         else
2638                 kprobe_type = "k";
2639
2640         if (!kallsyms_show_value(pi->file->f_cred))
2641                 addr = NULL;
2642
2643         if (sym)
2644                 seq_printf(pi, "%px  %s  %s+0x%x  %s ",
2645                         addr, kprobe_type, sym, offset,
2646                         (modname ? modname : " "));
2647         else    /* try to use %pS */
2648                 seq_printf(pi, "%px  %s  %pS ",
2649                         addr, kprobe_type, p->addr);
2650
2651         if (!pp)
2652                 pp = p;
2653         seq_printf(pi, "%s%s%s%s\n",
2654                 (kprobe_gone(p) ? "[GONE]" : ""),
2655                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
2656                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2657                 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
2658 }
2659
2660 static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2661 {
2662         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2663 }
2664
2665 static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2666 {
2667         (*pos)++;
2668         if (*pos >= KPROBE_TABLE_SIZE)
2669                 return NULL;
2670         return pos;
2671 }
2672
2673 static void kprobe_seq_stop(struct seq_file *f, void *v)
2674 {
2675         /* Nothing to do */
2676 }
2677
2678 static int show_kprobe_addr(struct seq_file *pi, void *v)
2679 {
2680         struct hlist_head *head;
2681         struct kprobe *p, *kp;
2682         const char *sym = NULL;
2683         unsigned int i = *(loff_t *) v;
2684         unsigned long offset = 0;
2685         char *modname, namebuf[KSYM_NAME_LEN];
2686
2687         head = &kprobe_table[i];
2688         preempt_disable();
2689         hlist_for_each_entry_rcu(p, head, hlist) {
2690                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
2691                                         &offset, &modname, namebuf);
2692                 if (kprobe_aggrprobe(p)) {
2693                         list_for_each_entry_rcu(kp, &p->list, list)
2694                                 report_probe(pi, kp, sym, offset, modname, p);
2695                 } else
2696                         report_probe(pi, p, sym, offset, modname, NULL);
2697         }
2698         preempt_enable();
2699         return 0;
2700 }
2701
2702 static const struct seq_operations kprobes_sops = {
2703         .start = kprobe_seq_start,
2704         .next  = kprobe_seq_next,
2705         .stop  = kprobe_seq_stop,
2706         .show  = show_kprobe_addr
2707 };
2708
2709 DEFINE_SEQ_ATTRIBUTE(kprobes);
2710
2711 /* kprobes/blacklist -- shows which functions can not be probed */
2712 static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2713 {
2714         mutex_lock(&kprobe_mutex);
2715         return seq_list_start(&kprobe_blacklist, *pos);
2716 }
2717
2718 static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2719 {
2720         return seq_list_next(v, &kprobe_blacklist, pos);
2721 }
2722
2723 static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2724 {
2725         struct kprobe_blacklist_entry *ent =
2726                 list_entry(v, struct kprobe_blacklist_entry, list);
2727
2728         /*
2729          * If '/proc/kallsyms' is not showing kernel address, we won't
2730          * show them here either.
2731          */
2732         if (!kallsyms_show_value(m->file->f_cred))
2733                 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2734                            (void *)ent->start_addr);
2735         else
2736                 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2737                            (void *)ent->end_addr, (void *)ent->start_addr);
2738         return 0;
2739 }
2740
2741 static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v)
2742 {
2743         mutex_unlock(&kprobe_mutex);
2744 }
2745
2746 static const struct seq_operations kprobe_blacklist_sops = {
2747         .start = kprobe_blacklist_seq_start,
2748         .next  = kprobe_blacklist_seq_next,
2749         .stop  = kprobe_blacklist_seq_stop,
2750         .show  = kprobe_blacklist_seq_show,
2751 };
2752 DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
2753
2754 static int arm_all_kprobes(void)
2755 {
2756         struct hlist_head *head;
2757         struct kprobe *p;
2758         unsigned int i, total = 0, errors = 0;
2759         int err, ret = 0;
2760
2761         mutex_lock(&kprobe_mutex);
2762
2763         /* If kprobes are armed, just return */
2764         if (!kprobes_all_disarmed)
2765                 goto already_enabled;
2766
2767         /*
2768          * optimize_kprobe() called by arm_kprobe() checks
2769          * kprobes_all_disarmed, so set kprobes_all_disarmed before
2770          * arm_kprobe.
2771          */
2772         kprobes_all_disarmed = false;
2773         /* Arming kprobes doesn't optimize kprobe itself */
2774         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2775                 head = &kprobe_table[i];
2776                 /* Arm all kprobes on a best-effort basis */
2777                 hlist_for_each_entry(p, head, hlist) {
2778                         if (!kprobe_disabled(p)) {
2779                                 err = arm_kprobe(p);
2780                                 if (err)  {
2781                                         errors++;
2782                                         ret = err;
2783                                 }
2784                                 total++;
2785                         }
2786                 }
2787         }
2788
2789         if (errors)
2790                 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
2791                         errors, total);
2792         else
2793                 pr_info("Kprobes globally enabled\n");
2794
2795 already_enabled:
2796         mutex_unlock(&kprobe_mutex);
2797         return ret;
2798 }
2799
2800 static int disarm_all_kprobes(void)
2801 {
2802         struct hlist_head *head;
2803         struct kprobe *p;
2804         unsigned int i, total = 0, errors = 0;
2805         int err, ret = 0;
2806
2807         mutex_lock(&kprobe_mutex);
2808
2809         /* If kprobes are already disarmed, just return */
2810         if (kprobes_all_disarmed) {
2811                 mutex_unlock(&kprobe_mutex);
2812                 return 0;
2813         }
2814
2815         kprobes_all_disarmed = true;
2816
2817         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2818                 head = &kprobe_table[i];
2819                 /* Disarm all kprobes on a best-effort basis */
2820                 hlist_for_each_entry(p, head, hlist) {
2821                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2822                                 err = disarm_kprobe(p, false);
2823                                 if (err) {
2824                                         errors++;
2825                                         ret = err;
2826                                 }
2827                                 total++;
2828                         }
2829                 }
2830         }
2831
2832         if (errors)
2833                 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
2834                         errors, total);
2835         else
2836                 pr_info("Kprobes globally disabled\n");
2837
2838         mutex_unlock(&kprobe_mutex);
2839
2840         /* Wait for disarming all kprobes by optimizer */
2841         wait_for_kprobe_optimizer();
2842
2843         return ret;
2844 }
2845
2846 /*
2847  * XXX: The debugfs bool file interface doesn't allow for callbacks
2848  * when the bool state is switched. We can reuse that facility when
2849  * available
2850  */
2851 static ssize_t read_enabled_file_bool(struct file *file,
2852                char __user *user_buf, size_t count, loff_t *ppos)
2853 {
2854         char buf[3];
2855
2856         if (!kprobes_all_disarmed)
2857                 buf[0] = '1';
2858         else
2859                 buf[0] = '0';
2860         buf[1] = '\n';
2861         buf[2] = 0x00;
2862         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2863 }
2864
2865 static ssize_t write_enabled_file_bool(struct file *file,
2866                const char __user *user_buf, size_t count, loff_t *ppos)
2867 {
2868         bool enable;
2869         int ret;
2870
2871         ret = kstrtobool_from_user(user_buf, count, &enable);
2872         if (ret)
2873                 return ret;
2874
2875         ret = enable ? arm_all_kprobes() : disarm_all_kprobes();
2876         if (ret)
2877                 return ret;
2878
2879         return count;
2880 }
2881
2882 static const struct file_operations fops_kp = {
2883         .read =         read_enabled_file_bool,
2884         .write =        write_enabled_file_bool,
2885         .llseek =       default_llseek,
2886 };
2887
2888 static int __init debugfs_kprobe_init(void)
2889 {
2890         struct dentry *dir;
2891
2892         dir = debugfs_create_dir("kprobes", NULL);
2893
2894         debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
2895
2896         debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
2897
2898         debugfs_create_file("blacklist", 0400, dir, NULL,
2899                             &kprobe_blacklist_fops);
2900
2901         return 0;
2902 }
2903
2904 late_initcall(debugfs_kprobe_init);
2905 #endif /* CONFIG_DEBUG_FS */