kernel/trace/trace_kprobe.c

   1 /*
   2  * Kprobes-based tracing events
   3  *
   4  * Created by Masami Hiramatsu <mhiramat@redhat.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  */
  19 #define pr_fmt(fmt)     "trace_kprobe: " fmt
  20
  21 #include <linux/module.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/rculist.h>
  24 #include <linux/error-injection.h>
  25
  26 #include "trace_probe.h"
  27
  28 #define KPROBE_EVENT_SYSTEM "kprobes"
  29 #define KRETPROBE_MAXACTIVE_MAX 4096
  30
  31 /**
  32  * Kprobe event core functions
  33  */
  34 struct trace_kprobe {
  35         struct list_head        list;
  36         struct kretprobe        rp;     /* Use rp.kp for kprobe use */
  37         unsigned long __percpu *nhit;
  38         const char              *symbol;        /* symbol name */
  39         struct trace_probe      tp;
  40 };
  41
  42 #define SIZEOF_TRACE_KPROBE(n)                          \
  43         (offsetof(struct trace_kprobe, tp.args) +       \
  44         (sizeof(struct probe_arg) * (n)))
  45
  46 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
  47 {
  48         return tk->rp.handler != NULL;
  49 }
  50
  51 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
  52 {
  53         return tk->symbol ? tk->symbol : "unknown";
  54 }
  55
  56 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
  57 {
  58         return tk->rp.kp.offset;
  59 }
  60
  61 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
  62 {
  63         return !!(kprobe_gone(&tk->rp.kp));
  64 }
  65
  66 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
  67                                                  struct module *mod)
  68 {
  69         int len = strlen(mod->name);
  70         const char *name = trace_kprobe_symbol(tk);
  71         return strncmp(mod->name, name, len) == 0 && name[len] == ':';
  72 }
  73
  74 static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
  75 {
  76         return !!strchr(trace_kprobe_symbol(tk), ':');
  77 }
  78
  79 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
  80 {
  81         unsigned long nhit = 0;
  82         int cpu;
  83
  84         for_each_possible_cpu(cpu)
  85                 nhit += *per_cpu_ptr(tk->nhit, cpu);
  86
  87         return nhit;
  88 }
  89
  90 bool trace_kprobe_on_func_entry(struct trace_event_call *call)
  91 {
  92         struct trace_kprobe *tk = (struct trace_kprobe *)call->data;
  93
  94         return kprobe_on_func_entry(tk->rp.kp.addr,
  95                         tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
  96                         tk->rp.kp.addr ? 0 : tk->rp.kp.offset);
  97 }
  98
  99 bool trace_kprobe_error_injectable(struct trace_event_call *call)
 100 {
 101         struct trace_kprobe *tk = (struct trace_kprobe *)call->data;
 102         unsigned long addr;
 103
 104         if (tk->symbol) {
 105                 addr = (unsigned long)
 106                         kallsyms_lookup_name(trace_kprobe_symbol(tk));
 107                 addr += tk->rp.kp.offset;
 108         } else {
 109                 addr = (unsigned long)tk->rp.kp.addr;
 110         }
 111         return within_error_injection_list(addr);
 112 }
 113
 114 static int register_kprobe_event(struct trace_kprobe *tk);
 115 static int unregister_kprobe_event(struct trace_kprobe *tk);
 116
 117 static DEFINE_MUTEX(probe_lock);
 118 static LIST_HEAD(probe_list);
 119
 120 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
 121 static int kretprobe_dispatcher(struct kretprobe_instance *ri,
 122                                 struct pt_regs *regs);
 123
 124 /* Memory fetching by symbol */
 125 struct symbol_cache {
 126         char            *symbol;
 127         long            offset;
 128         unsigned long   addr;
 129 };
 130
 131 unsigned long update_symbol_cache(struct symbol_cache *sc)
 132 {
 133         sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
 134
 135         if (sc->addr)
 136                 sc->addr += sc->offset;
 137
 138         return sc->addr;
 139 }
 140
 141 void free_symbol_cache(struct symbol_cache *sc)
 142 {
 143         kfree(sc->symbol);
 144         kfree(sc);
 145 }
 146
 147 struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
 148 {
 149         struct symbol_cache *sc;
 150
 151         if (!sym || strlen(sym) == 0)
 152                 return NULL;
 153
 154         sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
 155         if (!sc)
 156                 return NULL;
 157
 158         sc->symbol = kstrdup(sym, GFP_KERNEL);
 159         if (!sc->symbol) {
 160                 kfree(sc);
 161                 return NULL;
 162         }
 163         sc->offset = offset;
 164         update_symbol_cache(sc);
 165
 166         return sc;
 167 }
 168
 169 /*
 170  * Kprobes-specific fetch functions
 171  */
 172 #define DEFINE_FETCH_stack(type)                                        \
 173 static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,          \
 174                                           void *offset, void *dest)     \
 175 {                                                                       \
 176         *(type *)dest = (type)regs_get_kernel_stack_nth(regs,           \
 177                                 (unsigned int)((unsigned long)offset)); \
 178 }                                                                       \
 179 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
 180
 181 DEFINE_BASIC_FETCH_FUNCS(stack)
 182 /* No string on the stack entry */
 183 #define fetch_stack_string      NULL
 184 #define fetch_stack_string_size NULL
 185
 186 #define DEFINE_FETCH_memory(type)                                       \
 187 static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,         \
 188                                           void *addr, void *dest)       \
 189 {                                                                       \
 190         type retval;                                                    \
 191         if (probe_kernel_address(addr, retval))                         \
 192                 *(type *)dest = 0;                                      \
 193         else                                                            \
 194                 *(type *)dest = retval;                                 \
 195 }                                                                       \
 196 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
 197
 198 DEFINE_BASIC_FETCH_FUNCS(memory)
 199 /*
 200  * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
 201  * length and relative data location.
 202  */
 203 static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
 204                                             void *addr, void *dest)
 205 {
 206         int maxlen = get_rloc_len(*(u32 *)dest);
 207         u8 *dst = get_rloc_data(dest);
 208         long ret;
 209
 210         if (!maxlen)
 211                 return;
 212
 213         /*
 214          * Try to get string again, since the string can be changed while
 215          * probing.
 216          */
 217         ret = strncpy_from_unsafe(dst, addr, maxlen);
 218
 219         if (ret < 0) {  /* Failed to fetch string */
 220                 dst[0] = '\0';
 221                 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
 222         } else {
 223                 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
 224         }
 225 }
 226 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
 227
 228 /* Return the length of string -- including null terminal byte */
 229 static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
 230                                                  void *addr, void *dest)
 231 {
 232         mm_segment_t old_fs;
 233         int ret, len = 0;
 234         u8 c;
 235
 236         old_fs = get_fs();
 237         set_fs(KERNEL_DS);
 238         pagefault_disable();
 239
 240         do {
 241                 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
 242                 len++;
 243         } while (c && ret == 0 && len < MAX_STRING_SIZE);
 244
 245         pagefault_enable();
 246         set_fs(old_fs);
 247
 248         if (ret < 0)    /* Failed to check the length */
 249                 *(u32 *)dest = 0;
 250         else
 251                 *(u32 *)dest = len;
 252 }
 253 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
 254
 255 #define DEFINE_FETCH_symbol(type)                                       \
 256 void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
 257 {                                                                       \
 258         struct symbol_cache *sc = data;                                 \
 259         if (sc->addr)                                                   \
 260                 fetch_memory_##type(regs, (void *)sc->addr, dest);      \
 261         else                                                            \
 262                 *(type *)dest = 0;                                      \
 263 }                                                                       \
 264 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
 265
 266 DEFINE_BASIC_FETCH_FUNCS(symbol)
 267 DEFINE_FETCH_symbol(string)
 268 DEFINE_FETCH_symbol(string_size)
 269
 270 /* kprobes don't support file_offset fetch methods */
 271 #define fetch_file_offset_u8            NULL
 272 #define fetch_file_offset_u16           NULL
 273 #define fetch_file_offset_u32           NULL
 274 #define fetch_file_offset_u64           NULL
 275 #define fetch_file_offset_string        NULL
 276 #define fetch_file_offset_string_size   NULL
 277
 278 /* Fetch type information table */
 279 static const struct fetch_type kprobes_fetch_type_table[] = {
 280         /* Special types */
 281         [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
 282                                         sizeof(u32), 1, "__data_loc char[]"),
 283         [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
 284                                         string_size, sizeof(u32), 0, "u32"),
 285         /* Basic types */
 286         ASSIGN_FETCH_TYPE(u8,  u8,  0),
 287         ASSIGN_FETCH_TYPE(u16, u16, 0),
 288         ASSIGN_FETCH_TYPE(u32, u32, 0),
 289         ASSIGN_FETCH_TYPE(u64, u64, 0),
 290         ASSIGN_FETCH_TYPE(s8,  u8,  1),
 291         ASSIGN_FETCH_TYPE(s16, u16, 1),
 292         ASSIGN_FETCH_TYPE(s32, u32, 1),
 293         ASSIGN_FETCH_TYPE(s64, u64, 1),
 294         ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
 295         ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
 296         ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
 297         ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
 298
 299         ASSIGN_FETCH_TYPE_END
 300 };
 301
 302 /*
 303  * Allocate new trace_probe and initialize it (including kprobes).
 304  */
 305 static struct trace_kprobe *alloc_trace_kprobe(const char *group,
 306                                              const char *event,
 307                                              void *addr,
 308                                              const char *symbol,
 309                                              unsigned long offs,
 310                                              int maxactive,
 311                                              int nargs, bool is_return)
 312 {
 313         struct trace_kprobe *tk;
 314         int ret = -ENOMEM;
 315
 316         tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
 317         if (!tk)
 318                 return ERR_PTR(ret);
 319
 320         tk->nhit = alloc_percpu(unsigned long);
 321         if (!tk->nhit)
 322                 goto error;
 323
 324         if (symbol) {
 325                 tk->symbol = kstrdup(symbol, GFP_KERNEL);
 326                 if (!tk->symbol)
 327                         goto error;
 328                 tk->rp.kp.symbol_name = tk->symbol;
 329                 tk->rp.kp.offset = offs;
 330         } else
 331                 tk->rp.kp.addr = addr;
 332
 333         if (is_return)
 334                 tk->rp.handler = kretprobe_dispatcher;
 335         else
 336                 tk->rp.kp.pre_handler = kprobe_dispatcher;
 337
 338         tk->rp.maxactive = maxactive;
 339
 340         if (!event || !is_good_name(event)) {
 341                 ret = -EINVAL;
 342                 goto error;
 343         }
 344
 345         tk->tp.call.class = &tk->tp.class;
 346         tk->tp.call.name = kstrdup(event, GFP_KERNEL);
 347         if (!tk->tp.call.name)
 348                 goto error;
 349
 350         if (!group || !is_good_name(group)) {
 351                 ret = -EINVAL;
 352                 goto error;
 353         }
 354
 355         tk->tp.class.system = kstrdup(group, GFP_KERNEL);
 356         if (!tk->tp.class.system)
 357                 goto error;
 358
 359         INIT_LIST_HEAD(&tk->list);
 360         INIT_LIST_HEAD(&tk->tp.files);
 361         return tk;
 362 error:
 363         kfree(tk->tp.call.name);
 364         kfree(tk->symbol);
 365         free_percpu(tk->nhit);
 366         kfree(tk);
 367         return ERR_PTR(ret);
 368 }
 369
 370 static void free_trace_kprobe(struct trace_kprobe *tk)
 371 {
 372         int i;
 373
 374         for (i = 0; i < tk->tp.nr_args; i++)
 375                 traceprobe_free_probe_arg(&tk->tp.args[i]);
 376
 377         kfree(tk->tp.call.class->system);
 378         kfree(tk->tp.call.name);
 379         kfree(tk->symbol);
 380         free_percpu(tk->nhit);
 381         kfree(tk);
 382 }
 383
 384 static struct trace_kprobe *find_trace_kprobe(const char *event,
 385                                               const char *group)
 386 {
 387         struct trace_kprobe *tk;
 388
 389         list_for_each_entry(tk, &probe_list, list)
 390                 if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
 391                     strcmp(tk->tp.call.class->system, group) == 0)
 392                         return tk;
 393         return NULL;
 394 }
 395
 396 /*
 397  * Enable trace_probe
 398  * if the file is NULL, enable "perf" handler, or enable "trace" handler.
 399  */
 400 static int
 401 enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
 402 {
 403         int ret = 0;
 404
 405         if (file) {
 406                 struct event_file_link *link;
 407
 408                 link = kmalloc(sizeof(*link), GFP_KERNEL);
 409                 if (!link) {
 410                         ret = -ENOMEM;
 411                         goto out;
 412                 }
 413
 414                 link->file = file;
 415                 list_add_tail_rcu(&link->list, &tk->tp.files);
 416
 417                 tk->tp.flags |= TP_FLAG_TRACE;
 418         } else
 419                 tk->tp.flags |= TP_FLAG_PROFILE;
 420
 421         if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
 422                 if (trace_kprobe_is_return(tk))
 423                         ret = enable_kretprobe(&tk->rp);
 424                 else
 425                         ret = enable_kprobe(&tk->rp.kp);
 426         }
 427  out:
 428         return ret;
 429 }
 430
 431 /*
 432  * Disable trace_probe
 433  * if the file is NULL, disable "perf" handler, or disable "trace" handler.
 434  */
 435 static int
 436 disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
 437 {
 438         struct event_file_link *link = NULL;
 439         int wait = 0;
 440         int ret = 0;
 441
 442         if (file) {
 443                 link = find_event_file_link(&tk->tp, file);
 444                 if (!link) {
 445                         ret = -EINVAL;
 446                         goto out;
 447                 }
 448
 449                 list_del_rcu(&link->list);
 450                 wait = 1;
 451                 if (!list_empty(&tk->tp.files))
 452                         goto out;
 453
 454                 tk->tp.flags &= ~TP_FLAG_TRACE;
 455         } else
 456                 tk->tp.flags &= ~TP_FLAG_PROFILE;
 457
 458         if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
 459                 if (trace_kprobe_is_return(tk))
 460                         disable_kretprobe(&tk->rp);
 461                 else
 462                         disable_kprobe(&tk->rp.kp);
 463                 wait = 1;
 464         }
 465
 466         /*
 467          * if tk is not added to any list, it must be a local trace_kprobe
 468          * created with perf_event_open. We don't need to wait for these
 469          * trace_kprobes
 470          */
 471         if (list_empty(&tk->list))
 472                 wait = 0;
 473  out:
 474         if (wait) {
 475                 /*
 476                  * Synchronize with kprobe_trace_func/kretprobe_trace_func
 477                  * to ensure disabled (all running handlers are finished).
 478                  * This is not only for kfree(), but also the caller,
 479                  * trace_remove_event_call() supposes it for releasing
 480                  * event_call related objects, which will be accessed in
 481                  * the kprobe_trace_func/kretprobe_trace_func.
 482                  */
 483                 synchronize_sched();
 484                 kfree(link);    /* Ignored if link == NULL */
 485         }
 486
 487         return ret;
 488 }
 489
 490 /* Internal register function - just handle k*probes and flags */
 491 static int __register_trace_kprobe(struct trace_kprobe *tk)
 492 {
 493         int i, ret;
 494
 495         if (trace_probe_is_registered(&tk->tp))
 496                 return -EINVAL;
 497
 498         for (i = 0; i < tk->tp.nr_args; i++)
 499                 traceprobe_update_arg(&tk->tp.args[i]);
 500
 501         /* Set/clear disabled flag according to tp->flag */
 502         if (trace_probe_is_enabled(&tk->tp))
 503                 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
 504         else
 505                 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
 506
 507         if (trace_kprobe_is_return(tk))
 508                 ret = register_kretprobe(&tk->rp);
 509         else
 510                 ret = register_kprobe(&tk->rp.kp);
 511
 512         if (ret == 0)
 513                 tk->tp.flags |= TP_FLAG_REGISTERED;
 514         else {
 515                 pr_warn("Could not insert probe at %s+%lu: %d\n",
 516                         trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);
 517                 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {
 518                         pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
 519                         ret = 0;
 520                 } else if (ret == -EILSEQ) {
 521                         pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
 522                                 tk->rp.kp.addr);
 523                         ret = -EINVAL;
 524                 }
 525         }
 526
 527         return ret;
 528 }
 529
 530 /* Internal unregister function - just handle k*probes and flags */
 531 static void __unregister_trace_kprobe(struct trace_kprobe *tk)
 532 {
 533         if (trace_probe_is_registered(&tk->tp)) {
 534                 if (trace_kprobe_is_return(tk))
 535                         unregister_kretprobe(&tk->rp);
 536                 else
 537                         unregister_kprobe(&tk->rp.kp);
 538                 tk->tp.flags &= ~TP_FLAG_REGISTERED;
 539                 /* Cleanup kprobe for reuse */
 540                 if (tk->rp.kp.symbol_name)
 541                         tk->rp.kp.addr = NULL;
 542         }
 543 }
 544
 545 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
 546 static int unregister_trace_kprobe(struct trace_kprobe *tk)
 547 {
 548         /* Enabled event can not be unregistered */
 549         if (trace_probe_is_enabled(&tk->tp))
 550                 return -EBUSY;
 551
 552         /* Will fail if probe is being used by ftrace or perf */
 553         if (unregister_kprobe_event(tk))
 554                 return -EBUSY;
 555
 556         __unregister_trace_kprobe(tk);
 557         list_del(&tk->list);
 558
 559         return 0;
 560 }
 561
 562 /* Register a trace_probe and probe_event */
 563 static int register_trace_kprobe(struct trace_kprobe *tk)
 564 {
 565         struct trace_kprobe *old_tk;
 566         int ret;
 567
 568         mutex_lock(&probe_lock);
 569
 570         /* Delete old (same name) event if exist */
 571         old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),
 572                         tk->tp.call.class->system);
 573         if (old_tk) {
 574                 ret = unregister_trace_kprobe(old_tk);
 575                 if (ret < 0)
 576                         goto end;
 577                 free_trace_kprobe(old_tk);
 578         }
 579
 580         /* Register new event */
 581         ret = register_kprobe_event(tk);
 582         if (ret) {
 583                 pr_warn("Failed to register probe event(%d)\n", ret);
 584                 goto end;
 585         }
 586
 587         /* Register k*probe */
 588         ret = __register_trace_kprobe(tk);
 589         if (ret < 0)
 590                 unregister_kprobe_event(tk);
 591         else
 592                 list_add_tail(&tk->list, &probe_list);
 593
 594 end:
 595         mutex_unlock(&probe_lock);
 596         return ret;
 597 }
 598
 599 /* Module notifier call back, checking event on the module */
 600 static int trace_kprobe_module_callback(struct notifier_block *nb,
 601                                        unsigned long val, void *data)
 602 {
 603         struct module *mod = data;
 604         struct trace_kprobe *tk;
 605         int ret;
 606
 607         if (val != MODULE_STATE_COMING)
 608                 return NOTIFY_DONE;
 609
 610         /* Update probes on coming module */
 611         mutex_lock(&probe_lock);
 612         list_for_each_entry(tk, &probe_list, list) {
 613                 if (trace_kprobe_within_module(tk, mod)) {
 614                         /* Don't need to check busy - this should have gone. */
 615                         __unregister_trace_kprobe(tk);
 616                         ret = __register_trace_kprobe(tk);
 617                         if (ret)
 618                                 pr_warn("Failed to re-register probe %s on %s: %d\n",
 619                                         trace_event_name(&tk->tp.call),
 620                                         mod->name, ret);
 621                 }
 622         }
 623         mutex_unlock(&probe_lock);
 624
 625         return NOTIFY_DONE;
 626 }
 627
 628 static struct notifier_block trace_kprobe_module_nb = {
 629         .notifier_call = trace_kprobe_module_callback,
 630         .priority = 1   /* Invoked after kprobe module callback */
 631 };
 632
 633 /* Convert certain expected symbols into '_' when generating event names */
 634 static inline void sanitize_event_name(char *name)
 635 {
 636         while (*name++ != '\0')
 637                 if (*name == ':' || *name == '.')
 638                         *name = '_';
 639 }
 640
 641 static int create_trace_kprobe(int argc, char **argv)
 642 {
 643         /*
 644          * Argument syntax:
 645          *  - Add kprobe:
 646          *      p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
 647          *  - Add kretprobe:
 648          *      r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
 649          * Fetch args:
 650          *  $retval     : fetch return value
 651          *  $stack      : fetch stack address
 652          *  $stackN     : fetch Nth of stack (N:0-)
 653          *  $comm       : fetch current task comm
 654          *  @ADDR       : fetch memory at ADDR (ADDR should be in kernel)
 655          *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
 656          *  %REG        : fetch register REG
 657          * Dereferencing memory fetch:
 658          *  +|-offs(ARG) : fetch memory at ARG +|- offs address.
 659          * Alias name of args:
 660          *  NAME=FETCHARG : set NAME as alias of FETCHARG.
 661          * Type of args:
 662          *  FETCHARG:TYPE : use TYPE instead of unsigned long.
 663          */
 664         struct trace_kprobe *tk;
 665         int i, ret = 0;
 666         bool is_return = false, is_delete = false;
 667         char *symbol = NULL, *event = NULL, *group = NULL;
 668         int maxactive = 0;
 669         char *arg;
 670         unsigned long offset = 0;
 671         void *addr = NULL;
 672         char buf[MAX_EVENT_NAME_LEN];
 673
 674         /* argc must be >= 1 */
 675         if (argv[0][0] == 'p')
 676                 is_return = false;
 677         else if (argv[0][0] == 'r')
 678                 is_return = true;
 679         else if (argv[0][0] == '-')
 680                 is_delete = true;
 681         else {
 682                 pr_info("Probe definition must be started with 'p', 'r' or"
 683                         " '-'.\n");
 684                 return -EINVAL;
 685         }
 686
 687         event = strchr(&argv[0][1], ':');
 688         if (event) {
 689                 event[0] = '\0';
 690                 event++;
 691         }
 692         if (is_return && isdigit(argv[0][1])) {
 693                 ret = kstrtouint(&argv[0][1], 0, &maxactive);
 694                 if (ret) {
 695                         pr_info("Failed to parse maxactive.\n");
 696                         return ret;
 697                 }
 698                 /* kretprobes instances are iterated over via a list. The
 699                  * maximum should stay reasonable.
 700                  */
 701                 if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
 702                         pr_info("Maxactive is too big (%d > %d).\n",
 703                                 maxactive, KRETPROBE_MAXACTIVE_MAX);
 704                         return -E2BIG;
 705                 }
 706         }
 707
 708         if (event) {
 709                 if (strchr(event, '/')) {
 710                         group = event;
 711                         event = strchr(group, '/') + 1;
 712                         event[-1] = '\0';
 713                         if (strlen(group) == 0) {
 714                                 pr_info("Group name is not specified\n");
 715                                 return -EINVAL;
 716                         }
 717                 }
 718                 if (strlen(event) == 0) {
 719                         pr_info("Event name is not specified\n");
 720                         return -EINVAL;
 721                 }
 722         }
 723         if (!group)
 724                 group = KPROBE_EVENT_SYSTEM;
 725
 726         if (is_delete) {
 727                 if (!event) {
 728                         pr_info("Delete command needs an event name.\n");
 729                         return -EINVAL;
 730                 }
 731                 mutex_lock(&probe_lock);
 732                 tk = find_trace_kprobe(event, group);
 733                 if (!tk) {
 734                         mutex_unlock(&probe_lock);
 735                         pr_info("Event %s/%s doesn't exist.\n", group, event);
 736                         return -ENOENT;
 737                 }
 738                 /* delete an event */
 739                 ret = unregister_trace_kprobe(tk);
 740                 if (ret == 0)
 741                         free_trace_kprobe(tk);
 742                 mutex_unlock(&probe_lock);
 743                 return ret;
 744         }
 745
 746         if (argc < 2) {
 747                 pr_info("Probe point is not specified.\n");
 748                 return -EINVAL;
 749         }
 750
 751         /* try to parse an address. if that fails, try to read the
 752          * input as a symbol. */
 753         if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
 754                 /* a symbol specified */
 755                 symbol = argv[1];
 756                 /* TODO: support .init module functions */
 757                 ret = traceprobe_split_symbol_offset(symbol, &offset);
 758                 if (ret) {
 759                         pr_info("Failed to parse either an address or a symbol.\n");
 760                         return ret;
 761                 }
 762                 if (offset && is_return &&
 763                     !kprobe_on_func_entry(NULL, symbol, offset)) {
 764                         pr_info("Given offset is not valid for return probe.\n");
 765                         return -EINVAL;
 766                 }
 767         }
 768         argc -= 2; argv += 2;
 769
 770         /* setup a probe */
 771         if (!event) {
 772                 /* Make a new event name */
 773                 if (symbol)
 774                         snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
 775                                  is_return ? 'r' : 'p', symbol, offset);
 776                 else
 777                         snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
 778                                  is_return ? 'r' : 'p', addr);
 779                 sanitize_event_name(buf);
 780                 event = buf;
 781         }
 782         tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
 783                                argc, is_return);
 784         if (IS_ERR(tk)) {
 785                 pr_info("Failed to allocate trace_probe.(%d)\n",
 786                         (int)PTR_ERR(tk));
 787                 return PTR_ERR(tk);
 788         }
 789
 790         /* parse arguments */
 791         ret = 0;
 792         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
 793                 struct probe_arg *parg = &tk->tp.args[i];
 794
 795                 /* Increment count for freeing args in error case */
 796                 tk->tp.nr_args++;
 797
 798                 /* Parse argument name */
 799                 arg = strchr(argv[i], '=');
 800                 if (arg) {
 801                         *arg++ = '\0';
 802                         parg->name = kstrdup(argv[i], GFP_KERNEL);
 803                 } else {
 804                         arg = argv[i];
 805                         /* If argument name is omitted, set "argN" */
 806                         snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
 807                         parg->name = kstrdup(buf, GFP_KERNEL);
 808                 }
 809
 810                 if (!parg->name) {
 811                         pr_info("Failed to allocate argument[%d] name.\n", i);
 812                         ret = -ENOMEM;
 813                         goto error;
 814                 }
 815
 816                 if (!is_good_name(parg->name)) {
 817                         pr_info("Invalid argument[%d] name: %s\n",
 818                                 i, parg->name);
 819                         ret = -EINVAL;
 820                         goto error;
 821                 }
 822
 823                 if (traceprobe_conflict_field_name(parg->name,
 824                                                         tk->tp.args, i)) {
 825                         pr_info("Argument[%d] name '%s' conflicts with "
 826                                 "another field.\n", i, argv[i]);
 827                         ret = -EINVAL;
 828                         goto error;
 829                 }
 830
 831                 /* Parse fetch argument */
 832                 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
 833                                                 is_return, true,
 834                                                 kprobes_fetch_type_table);
 835                 if (ret) {
 836                         pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
 837                         goto error;
 838                 }
 839         }
 840
 841         ret = register_trace_kprobe(tk);
 842         if (ret)
 843                 goto error;
 844         return 0;
 845
 846 error:
 847         free_trace_kprobe(tk);
 848         return ret;
 849 }
 850
 851 static int release_all_trace_kprobes(void)
 852 {
 853         struct trace_kprobe *tk;
 854         int ret = 0;
 855
 856         mutex_lock(&probe_lock);
 857         /* Ensure no probe is in use. */
 858         list_for_each_entry(tk, &probe_list, list)
 859                 if (trace_probe_is_enabled(&tk->tp)) {
 860                         ret = -EBUSY;
 861                         goto end;
 862                 }
 863         /* TODO: Use batch unregistration */
 864         while (!list_empty(&probe_list)) {
 865                 tk = list_entry(probe_list.next, struct trace_kprobe, list);
 866                 ret = unregister_trace_kprobe(tk);
 867                 if (ret)
 868                         goto end;
 869                 free_trace_kprobe(tk);
 870         }
 871
 872 end:
 873         mutex_unlock(&probe_lock);
 874
 875         return ret;
 876 }
 877
 878 /* Probes listing interfaces */
 879 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
 880 {
 881         mutex_lock(&probe_lock);
 882         return seq_list_start(&probe_list, *pos);
 883 }
 884
 885 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
 886 {
 887         return seq_list_next(v, &probe_list, pos);
 888 }
 889
 890 static void probes_seq_stop(struct seq_file *m, void *v)
 891 {
 892         mutex_unlock(&probe_lock);
 893 }
 894
 895 static int probes_seq_show(struct seq_file *m, void *v)
 896 {
 897         struct trace_kprobe *tk = v;
 898         int i;
 899
 900         seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
 901         seq_printf(m, ":%s/%s", tk->tp.call.class->system,
 902                         trace_event_name(&tk->tp.call));
 903
 904         if (!tk->symbol)
 905                 seq_printf(m, " 0x%p", tk->rp.kp.addr);
 906         else if (tk->rp.kp.offset)
 907                 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
 908                            tk->rp.kp.offset);
 909         else
 910                 seq_printf(m, " %s", trace_kprobe_symbol(tk));
 911
 912         for (i = 0; i < tk->tp.nr_args; i++)
 913                 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
 914         seq_putc(m, '\n');
 915
 916         return 0;
 917 }
 918
 919 static const struct seq_operations probes_seq_op = {
 920         .start  = probes_seq_start,
 921         .next   = probes_seq_next,
 922         .stop   = probes_seq_stop,
 923         .show   = probes_seq_show
 924 };
 925
 926 static int probes_open(struct inode *inode, struct file *file)
 927 {
 928         int ret;
 929
 930         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
 931                 ret = release_all_trace_kprobes();
 932                 if (ret < 0)
 933                         return ret;
 934         }
 935
 936         return seq_open(file, &probes_seq_op);
 937 }
 938
 939 static ssize_t probes_write(struct file *file, const char __user *buffer,
 940                             size_t count, loff_t *ppos)
 941 {
 942         return trace_parse_run_command(file, buffer, count, ppos,
 943                                        create_trace_kprobe);
 944 }
 945
 946 static const struct file_operations kprobe_events_ops = {
 947         .owner          = THIS_MODULE,
 948         .open           = probes_open,
 949         .read           = seq_read,
 950         .llseek         = seq_lseek,
 951         .release        = seq_release,
 952         .write          = probes_write,
 953 };
 954
 955 /* Probes profiling interfaces */
 956 static int probes_profile_seq_show(struct seq_file *m, void *v)
 957 {
 958         struct trace_kprobe *tk = v;
 959
 960         seq_printf(m, "  %-44s %15lu %15lu\n",
 961                    trace_event_name(&tk->tp.call),
 962                    trace_kprobe_nhit(tk),
 963                    tk->rp.kp.nmissed);
 964
 965         return 0;
 966 }
 967
 968 static const struct seq_operations profile_seq_op = {
 969         .start  = probes_seq_start,
 970         .next   = probes_seq_next,
 971         .stop   = probes_seq_stop,
 972         .show   = probes_profile_seq_show
 973 };
 974
 975 static int profile_open(struct inode *inode, struct file *file)
 976 {
 977         return seq_open(file, &profile_seq_op);
 978 }
 979
 980 static const struct file_operations kprobe_profile_ops = {
 981         .owner          = THIS_MODULE,
 982         .open           = profile_open,
 983         .read           = seq_read,
 984         .llseek         = seq_lseek,
 985         .release        = seq_release,
 986 };
 987
 988 /* Kprobe handler */
 989 static nokprobe_inline void
 990 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
 991                     struct trace_event_file *trace_file)
 992 {
 993         struct kprobe_trace_entry_head *entry;
 994         struct ring_buffer_event *event;
 995         struct ring_buffer *buffer;
 996         int size, dsize, pc;
 997         unsigned long irq_flags;
 998         struct trace_event_call *call = &tk->tp.call;
 999
1000         WARN_ON(call != trace_file->event_call);
1001
1002         if (trace_trigger_soft_disabled(trace_file))
1003                 return;
1004
1005         local_save_flags(irq_flags);
1006         pc = preempt_count();
1007
1008         dsize = __get_data_size(&tk->tp, regs);
1009         size = sizeof(*entry) + tk->tp.size + dsize;
1010
1011         event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1012                                                 call->event.type,
1013                                                 size, irq_flags, pc);
1014         if (!event)
1015                 return;
1016
1017         entry = ring_buffer_event_data(event);
1018         entry->ip = (unsigned long)tk->rp.kp.addr;
1019         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1020
1021         event_trigger_unlock_commit_regs(trace_file, buffer, event,
1022                                          entry, irq_flags, pc, regs);
1023 }
1024
1025 static void
1026 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1027 {
1028         struct event_file_link *link;
1029
1030         list_for_each_entry_rcu(link, &tk->tp.files, list)
1031                 __kprobe_trace_func(tk, regs, link->file);
1032 }
1033 NOKPROBE_SYMBOL(kprobe_trace_func);
1034
1035 /* Kretprobe handler */
1036 static nokprobe_inline void
1037 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1038                        struct pt_regs *regs,
1039                        struct trace_event_file *trace_file)
1040 {
1041         struct kretprobe_trace_entry_head *entry;
1042         struct ring_buffer_event *event;
1043         struct ring_buffer *buffer;
1044         int size, pc, dsize;
1045         unsigned long irq_flags;
1046         struct trace_event_call *call = &tk->tp.call;
1047
1048         WARN_ON(call != trace_file->event_call);
1049
1050         if (trace_trigger_soft_disabled(trace_file))
1051                 return;
1052
1053         local_save_flags(irq_flags);
1054         pc = preempt_count();
1055
1056         dsize = __get_data_size(&tk->tp, regs);
1057         size = sizeof(*entry) + tk->tp.size + dsize;
1058
1059         event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1060                                                 call->event.type,
1061                                                 size, irq_flags, pc);
1062         if (!event)
1063                 return;
1064
1065         entry = ring_buffer_event_data(event);
1066         entry->func = (unsigned long)tk->rp.kp.addr;
1067         entry->ret_ip = (unsigned long)ri->ret_addr;
1068         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1069
1070         event_trigger_unlock_commit_regs(trace_file, buffer, event,
1071                                          entry, irq_flags, pc, regs);
1072 }
1073
1074 static void
1075 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1076                      struct pt_regs *regs)
1077 {
1078         struct event_file_link *link;
1079
1080         list_for_each_entry_rcu(link, &tk->tp.files, list)
1081                 __kretprobe_trace_func(tk, ri, regs, link->file);
1082 }
1083 NOKPROBE_SYMBOL(kretprobe_trace_func);
1084
1085 /* Event entry printers */
1086 static enum print_line_t
1087 print_kprobe_event(struct trace_iterator *iter, int flags,
1088                    struct trace_event *event)
1089 {
1090         struct kprobe_trace_entry_head *field;
1091         struct trace_seq *s = &iter->seq;
1092         struct trace_probe *tp;
1093         u8 *data;
1094         int i;
1095
1096         field = (struct kprobe_trace_entry_head *)iter->ent;
1097         tp = container_of(event, struct trace_probe, call.event);
1098
1099         trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
1100
1101         if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1102                 goto out;
1103
1104         trace_seq_putc(s, ')');
1105
1106         data = (u8 *)&field[1];
1107         for (i = 0; i < tp->nr_args; i++)
1108                 if (!tp->args[i].type->print(s, tp->args[i].name,
1109                                              data + tp->args[i].offset, field))
1110                         goto out;
1111
1112         trace_seq_putc(s, '\n');
1113  out:
1114         return trace_handle_return(s);
1115 }
1116
1117 static enum print_line_t
1118 print_kretprobe_event(struct trace_iterator *iter, int flags,
1119                       struct trace_event *event)
1120 {
1121         struct kretprobe_trace_entry_head *field;
1122         struct trace_seq *s = &iter->seq;
1123         struct trace_probe *tp;
1124         u8 *data;
1125         int i;
1126
1127         field = (struct kretprobe_trace_entry_head *)iter->ent;
1128         tp = container_of(event, struct trace_probe, call.event);
1129
1130         trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
1131
1132         if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1133                 goto out;
1134
1135         trace_seq_puts(s, " <- ");
1136
1137         if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1138                 goto out;
1139
1140         trace_seq_putc(s, ')');
1141
1142         data = (u8 *)&field[1];
1143         for (i = 0; i < tp->nr_args; i++)
1144                 if (!tp->args[i].type->print(s, tp->args[i].name,
1145                                              data + tp->args[i].offset, field))
1146                         goto out;
1147
1148         trace_seq_putc(s, '\n');
1149
1150  out:
1151         return trace_handle_return(s);
1152 }
1153
1154
1155 static int kprobe_event_define_fields(struct trace_event_call *event_call)
1156 {
1157         int ret, i;
1158         struct kprobe_trace_entry_head field;
1159         struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1160
1161         DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1162         /* Set argument names as fields */
1163         for (i = 0; i < tk->tp.nr_args; i++) {
1164                 struct probe_arg *parg = &tk->tp.args[i];
1165
1166                 ret = trace_define_field(event_call, parg->type->fmttype,
1167                                          parg->name,
1168                                          sizeof(field) + parg->offset,
1169                                          parg->type->size,
1170                                          parg->type->is_signed,
1171                                          FILTER_OTHER);
1172                 if (ret)
1173                         return ret;
1174         }
1175         return 0;
1176 }
1177
1178 static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1179 {
1180         int ret, i;
1181         struct kretprobe_trace_entry_head field;
1182         struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1183
1184         DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1185         DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1186         /* Set argument names as fields */
1187         for (i = 0; i < tk->tp.nr_args; i++) {
1188                 struct probe_arg *parg = &tk->tp.args[i];
1189
1190                 ret = trace_define_field(event_call, parg->type->fmttype,
1191                                          parg->name,
1192                                          sizeof(field) + parg->offset,
1193                                          parg->type->size,
1194                                          parg->type->is_signed,
1195                                          FILTER_OTHER);
1196                 if (ret)
1197                         return ret;
1198         }
1199         return 0;
1200 }
1201
1202 #ifdef CONFIG_PERF_EVENTS
1203
1204 /* Kprobe profile handler */
1205 static int
1206 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1207 {
1208         struct trace_event_call *call = &tk->tp.call;
1209         struct kprobe_trace_entry_head *entry;
1210         struct hlist_head *head;
1211         int size, __size, dsize;
1212         int rctx;
1213
1214         if (bpf_prog_array_valid(call)) {
1215                 unsigned long orig_ip = instruction_pointer(regs);
1216                 int ret;
1217
1218                 ret = trace_call_bpf(call, regs);
1219
1220                 /*
1221                  * We need to check and see if we modified the pc of the
1222                  * pt_regs, and if so clear the kprobe and return 1 so that we
1223                  * don't do the single stepping.
1224                  * The ftrace kprobe handler leaves it up to us to re-enable
1225                  * preemption here before returning if we've modified the ip.
1226                  */
1227                 if (orig_ip != instruction_pointer(regs)) {
1228                         reset_current_kprobe();
1229                         preempt_enable_no_resched();
1230                         return 1;
1231                 }
1232                 if (!ret)
1233                         return 0;
1234         }
1235
1236         head = this_cpu_ptr(call->perf_events);
1237         if (hlist_empty(head))
1238                 return 0;
1239
1240         dsize = __get_data_size(&tk->tp, regs);
1241         __size = sizeof(*entry) + tk->tp.size + dsize;
1242         size = ALIGN(__size + sizeof(u32), sizeof(u64));
1243         size -= sizeof(u32);
1244
1245         entry = perf_trace_buf_alloc(size, NULL, &rctx);
1246         if (!entry)
1247                 return 0;
1248
1249         entry->ip = (unsigned long)tk->rp.kp.addr;
1250         memset(&entry[1], 0, dsize);
1251         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1252         perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1253                               head, NULL);
1254         return 0;
1255 }
1256 NOKPROBE_SYMBOL(kprobe_perf_func);
1257
1258 /* Kretprobe profile handler */
1259 static void
1260 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1261                     struct pt_regs *regs)
1262 {
1263         struct trace_event_call *call = &tk->tp.call;
1264         struct kretprobe_trace_entry_head *entry;
1265         struct hlist_head *head;
1266         int size, __size, dsize;
1267         int rctx;
1268
1269         if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1270                 return;
1271
1272         head = this_cpu_ptr(call->perf_events);
1273         if (hlist_empty(head))
1274                 return;
1275
1276         dsize = __get_data_size(&tk->tp, regs);
1277         __size = sizeof(*entry) + tk->tp.size + dsize;
1278         size = ALIGN(__size + sizeof(u32), sizeof(u64));
1279         size -= sizeof(u32);
1280
1281         entry = perf_trace_buf_alloc(size, NULL, &rctx);
1282         if (!entry)
1283                 return;
1284
1285         entry->func = (unsigned long)tk->rp.kp.addr;
1286         entry->ret_ip = (unsigned long)ri->ret_addr;
1287         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1288         perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1289                               head, NULL);
1290 }
1291 NOKPROBE_SYMBOL(kretprobe_perf_func);
1292 #endif  /* CONFIG_PERF_EVENTS */
1293
1294 /*
1295  * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1296  *
1297  * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1298  * lockless, but we can't race with this __init function.
1299  */
1300 static int kprobe_register(struct trace_event_call *event,
1301                            enum trace_reg type, void *data)
1302 {
1303         struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
1304         struct trace_event_file *file = data;
1305
1306         switch (type) {
1307         case TRACE_REG_REGISTER:
1308                 return enable_trace_kprobe(tk, file);
1309         case TRACE_REG_UNREGISTER:
1310                 return disable_trace_kprobe(tk, file);
1311
1312 #ifdef CONFIG_PERF_EVENTS
1313         case TRACE_REG_PERF_REGISTER:
1314                 return enable_trace_kprobe(tk, NULL);
1315         case TRACE_REG_PERF_UNREGISTER:
1316                 return disable_trace_kprobe(tk, NULL);
1317         case TRACE_REG_PERF_OPEN:
1318         case TRACE_REG_PERF_CLOSE:
1319         case TRACE_REG_PERF_ADD:
1320         case TRACE_REG_PERF_DEL:
1321                 return 0;
1322 #endif
1323         }
1324         return 0;
1325 }
1326
1327 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1328 {
1329         struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1330         int ret = 0;
1331
1332         raw_cpu_inc(*tk->nhit);
1333
1334         if (tk->tp.flags & TP_FLAG_TRACE)
1335                 kprobe_trace_func(tk, regs);
1336 #ifdef CONFIG_PERF_EVENTS
1337         if (tk->tp.flags & TP_FLAG_PROFILE)
1338                 ret = kprobe_perf_func(tk, regs);
1339 #endif
1340         return ret;
1341 }
1342 NOKPROBE_SYMBOL(kprobe_dispatcher);
1343
1344 static int
1345 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1346 {
1347         struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1348
1349         raw_cpu_inc(*tk->nhit);
1350
1351         if (tk->tp.flags & TP_FLAG_TRACE)
1352                 kretprobe_trace_func(tk, ri, regs);
1353 #ifdef CONFIG_PERF_EVENTS
1354         if (tk->tp.flags & TP_FLAG_PROFILE)
1355                 kretprobe_perf_func(tk, ri, regs);
1356 #endif
1357         return 0;       /* We don't tweek kernel, so just return 0 */
1358 }
1359 NOKPROBE_SYMBOL(kretprobe_dispatcher);
1360
1361 static struct trace_event_functions kretprobe_funcs = {
1362         .trace          = print_kretprobe_event
1363 };
1364
1365 static struct trace_event_functions kprobe_funcs = {
1366         .trace          = print_kprobe_event
1367 };
1368
1369 static inline void init_trace_event_call(struct trace_kprobe *tk,
1370                                          struct trace_event_call *call)
1371 {
1372         INIT_LIST_HEAD(&call->class->fields);
1373         if (trace_kprobe_is_return(tk)) {
1374                 call->event.funcs = &kretprobe_funcs;
1375                 call->class->define_fields = kretprobe_event_define_fields;
1376         } else {
1377                 call->event.funcs = &kprobe_funcs;
1378                 call->class->define_fields = kprobe_event_define_fields;
1379         }
1380
1381         call->flags = TRACE_EVENT_FL_KPROBE;
1382         call->class->reg = kprobe_register;
1383         call->data = tk;
1384 }
1385
1386 static int register_kprobe_event(struct trace_kprobe *tk)
1387 {
1388         struct trace_event_call *call = &tk->tp.call;
1389         int ret = 0;
1390
1391         init_trace_event_call(tk, call);
1392
1393         if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1394                 return -ENOMEM;
1395         ret = register_trace_event(&call->event);
1396         if (!ret) {
1397                 kfree(call->print_fmt);
1398                 return -ENODEV;
1399         }
1400         ret = trace_add_event_call(call);
1401         if (ret) {
1402                 pr_info("Failed to register kprobe event: %s\n",
1403                         trace_event_name(call));
1404                 kfree(call->print_fmt);
1405                 unregister_trace_event(&call->event);
1406         }
1407         return ret;
1408 }
1409
1410 static int unregister_kprobe_event(struct trace_kprobe *tk)
1411 {
1412         int ret;
1413
1414         /* tp->event is unregistered in trace_remove_event_call() */
1415         ret = trace_remove_event_call(&tk->tp.call);
1416         if (!ret)
1417                 kfree(tk->tp.call.print_fmt);
1418         return ret;
1419 }
1420
1421 #ifdef CONFIG_PERF_EVENTS
1422 /* create a trace_kprobe, but don't add it to global lists */
1423 struct trace_event_call *
1424 create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1425                           bool is_return)
1426 {
1427         struct trace_kprobe *tk;
1428         int ret;
1429         char *event;
1430
1431         /*
1432          * local trace_kprobes are not added to probe_list, so they are never
1433          * searched in find_trace_kprobe(). Therefore, there is no concern of
1434          * duplicated name here.
1435          */
1436         event = func ? func : "DUMMY_EVENT";
1437
1438         tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1439                                 offs, 0 /* maxactive */, 0 /* nargs */,
1440                                 is_return);
1441
1442         if (IS_ERR(tk)) {
1443                 pr_info("Failed to allocate trace_probe.(%d)\n",
1444                         (int)PTR_ERR(tk));
1445                 return ERR_CAST(tk);
1446         }
1447
1448         init_trace_event_call(tk, &tk->tp.call);
1449
1450         if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1451                 ret = -ENOMEM;
1452                 goto error;
1453         }
1454
1455         ret = __register_trace_kprobe(tk);
1456         if (ret < 0)
1457                 goto error;
1458
1459         return &tk->tp.call;
1460 error:
1461         free_trace_kprobe(tk);
1462         return ERR_PTR(ret);
1463 }
1464
1465 void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1466 {
1467         struct trace_kprobe *tk;
1468
1469         tk = container_of(event_call, struct trace_kprobe, tp.call);
1470
1471         if (trace_probe_is_enabled(&tk->tp)) {
1472                 WARN_ON(1);
1473                 return;
1474         }
1475
1476         __unregister_trace_kprobe(tk);
1477         free_trace_kprobe(tk);
1478 }
1479 #endif /* CONFIG_PERF_EVENTS */
1480
1481 /* Make a tracefs interface for controlling probe points */
1482 static __init int init_kprobe_trace(void)
1483 {
1484         struct dentry *d_tracer;
1485         struct dentry *entry;
1486
1487         if (register_module_notifier(&trace_kprobe_module_nb))
1488                 return -EINVAL;
1489
1490         d_tracer = tracing_init_dentry();
1491         if (IS_ERR(d_tracer))
1492                 return 0;
1493
1494         entry = tracefs_create_file("kprobe_events", 0644, d_tracer,
1495                                     NULL, &kprobe_events_ops);
1496
1497         /* Event list interface */
1498         if (!entry)
1499                 pr_warn("Could not create tracefs 'kprobe_events' entry\n");
1500
1501         /* Profile interface */
1502         entry = tracefs_create_file("kprobe_profile", 0444, d_tracer,
1503                                     NULL, &kprobe_profile_ops);
1504
1505         if (!entry)
1506                 pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
1507         return 0;
1508 }
1509 fs_initcall(init_kprobe_trace);
1510
1511
1512 #ifdef CONFIG_FTRACE_STARTUP_TEST
1513 /*
1514  * The "__used" keeps gcc from removing the function symbol
1515  * from the kallsyms table. 'noinline' makes sure that there
1516  * isn't an inlined version used by the test method below
1517  */
1518 static __used __init noinline int
1519 kprobe_trace_selftest_target(int a1, int a2, int a3, int a4, int a5, int a6)
1520 {
1521         return a1 + a2 + a3 + a4 + a5 + a6;
1522 }
1523
1524 static __init struct trace_event_file *
1525 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1526 {
1527         struct trace_event_file *file;
1528
1529         list_for_each_entry(file, &tr->events, list)
1530                 if (file->event_call == &tk->tp.call)
1531                         return file;
1532
1533         return NULL;
1534 }
1535
1536 /*
1537  * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1538  * stage, we can do this lockless.
1539  */
1540 static __init int kprobe_trace_self_tests_init(void)
1541 {
1542         int ret, warn = 0;
1543         int (*target)(int, int, int, int, int, int);
1544         struct trace_kprobe *tk;
1545         struct trace_event_file *file;
1546
1547         if (tracing_is_disabled())
1548                 return -ENODEV;
1549
1550         target = kprobe_trace_selftest_target;
1551
1552         pr_info("Testing kprobe tracing: ");
1553
1554         ret = trace_run_command("p:testprobe kprobe_trace_selftest_target "
1555                                 "$stack $stack0 +0($stack)",
1556                                 create_trace_kprobe);
1557         if (WARN_ON_ONCE(ret)) {
1558                 pr_warn("error on probing function entry.\n");
1559                 warn++;
1560         } else {
1561                 /* Enable trace point */
1562                 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1563                 if (WARN_ON_ONCE(tk == NULL)) {
1564                         pr_warn("error on getting new probe.\n");
1565                         warn++;
1566                 } else {
1567                         file = find_trace_probe_file(tk, top_trace_array());
1568                         if (WARN_ON_ONCE(file == NULL)) {
1569                                 pr_warn("error on getting probe file.\n");
1570                                 warn++;
1571                         } else
1572                                 enable_trace_kprobe(tk, file);
1573                 }
1574         }
1575
1576         ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target "
1577                                 "$retval", create_trace_kprobe);
1578         if (WARN_ON_ONCE(ret)) {
1579                 pr_warn("error on probing function return.\n");
1580                 warn++;
1581         } else {
1582                 /* Enable trace point */
1583                 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1584                 if (WARN_ON_ONCE(tk == NULL)) {
1585                         pr_warn("error on getting 2nd new probe.\n");
1586                         warn++;
1587                 } else {
1588                         file = find_trace_probe_file(tk, top_trace_array());
1589                         if (WARN_ON_ONCE(file == NULL)) {
1590                                 pr_warn("error on getting probe file.\n");
1591                                 warn++;
1592                         } else
1593                                 enable_trace_kprobe(tk, file);
1594                 }
1595         }
1596
1597         if (warn)
1598                 goto end;
1599
1600         ret = target(1, 2, 3, 4, 5, 6);
1601
1602         /*
1603          * Not expecting an error here, the check is only to prevent the
1604          * optimizer from removing the call to target() as otherwise there
1605          * are no side-effects and the call is never performed.
1606          */
1607         if (ret != 21)
1608                 warn++;
1609
1610         /* Disable trace points before removing it */
1611         tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1612         if (WARN_ON_ONCE(tk == NULL)) {
1613                 pr_warn("error on getting test probe.\n");
1614                 warn++;
1615         } else {
1616                 if (trace_kprobe_nhit(tk) != 1) {
1617                         pr_warn("incorrect number of testprobe hits\n");
1618                         warn++;
1619                 }
1620
1621                 file = find_trace_probe_file(tk, top_trace_array());
1622                 if (WARN_ON_ONCE(file == NULL)) {
1623                         pr_warn("error on getting probe file.\n");
1624                         warn++;
1625                 } else
1626                         disable_trace_kprobe(tk, file);
1627         }
1628
1629         tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1630         if (WARN_ON_ONCE(tk == NULL)) {
1631                 pr_warn("error on getting 2nd test probe.\n");
1632                 warn++;
1633         } else {
1634                 if (trace_kprobe_nhit(tk) != 1) {
1635                         pr_warn("incorrect number of testprobe2 hits\n");
1636                         warn++;
1637                 }
1638
1639                 file = find_trace_probe_file(tk, top_trace_array());
1640                 if (WARN_ON_ONCE(file == NULL)) {
1641                         pr_warn("error on getting probe file.\n");
1642                         warn++;
1643                 } else
1644                         disable_trace_kprobe(tk, file);
1645         }
1646
1647         ret = trace_run_command("-:testprobe", create_trace_kprobe);
1648         if (WARN_ON_ONCE(ret)) {
1649                 pr_warn("error on deleting a probe.\n");
1650                 warn++;
1651         }
1652
1653         ret = trace_run_command("-:testprobe2", create_trace_kprobe);
1654         if (WARN_ON_ONCE(ret)) {
1655                 pr_warn("error on deleting a probe.\n");
1656                 warn++;
1657         }
1658
1659 end:
1660         release_all_trace_kprobes();
1661         /*
1662          * Wait for the optimizer work to finish. Otherwise it might fiddle
1663          * with probes in already freed __init text.
1664          */
1665         wait_for_kprobe_optimizer();
1666         if (warn)
1667                 pr_cont("NG: Some tests are failed. Please check them.\n");
1668         else
1669                 pr_cont("OK\n");
1670         return 0;
1671 }
1672
1673 late_initcall(kprobe_trace_self_tests_init);
1674
1675 #endif