kernel/trace/trace_eprobe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * event probes
   4  *
   5  * Part of this code was copied from kernel/trace/trace_kprobe.c written by
   6  * Masami Hiramatsu <mhiramat@kernel.org>
   7  *
   8  * Copyright (C) 2021, VMware Inc, Steven Rostedt <rostedt@goodmis.org>
   9  * Copyright (C) 2021, VMware Inc, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
  10  *
  11  */
  12 #include <linux/module.h>
  13 #include <linux/mutex.h>
  14 #include <linux/ftrace.h>
  15
  16 #include "trace_dynevent.h"
  17 #include "trace_probe.h"
  18 #include "trace_probe_tmpl.h"
  19
  20 #define EPROBE_EVENT_SYSTEM "eprobes"
  21
  22 struct trace_eprobe {
  23         /* tracepoint system */
  24         const char *event_system;
  25
  26         /* tracepoint event */
  27         const char *event_name;
  28
  29         struct trace_event_call *event;
  30
  31         struct dyn_event        devent;
  32         struct trace_probe      tp;
  33 };
  34
  35 struct eprobe_data {
  36         struct trace_event_file *file;
  37         struct trace_eprobe     *ep;
  38 };
  39
  40 static int __trace_eprobe_create(int argc, const char *argv[]);
  41
  42 static void trace_event_probe_cleanup(struct trace_eprobe *ep)
  43 {
  44         if (!ep)
  45                 return;
  46         trace_probe_cleanup(&ep->tp);
  47         kfree(ep->event_name);
  48         kfree(ep->event_system);
  49         if (ep->event)
  50                 trace_event_put_ref(ep->event);
  51         kfree(ep);
  52 }
  53
  54 static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev)
  55 {
  56         return container_of(ev, struct trace_eprobe, devent);
  57 }
  58
  59 static int eprobe_dyn_event_create(const char *raw_command)
  60 {
  61         return trace_probe_create(raw_command, __trace_eprobe_create);
  62 }
  63
  64 static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
  65 {
  66         struct trace_eprobe *ep = to_trace_eprobe(ev);
  67         int i;
  68
  69         seq_printf(m, "e:%s/%s", trace_probe_group_name(&ep->tp),
  70                                 trace_probe_name(&ep->tp));
  71         seq_printf(m, " %s.%s", ep->event_system, ep->event_name);
  72
  73         for (i = 0; i < ep->tp.nr_args; i++)
  74                 seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
  75         seq_putc(m, '\n');
  76
  77         return 0;
  78 }
  79
  80 static int unregister_trace_eprobe(struct trace_eprobe *ep)
  81 {
  82         /* If other probes are on the event, just unregister eprobe */
  83         if (trace_probe_has_sibling(&ep->tp))
  84                 goto unreg;
  85
  86         /* Enabled event can not be unregistered */
  87         if (trace_probe_is_enabled(&ep->tp))
  88                 return -EBUSY;
  89
  90         /* Will fail if probe is being used by ftrace or perf */
  91         if (trace_probe_unregister_event_call(&ep->tp))
  92                 return -EBUSY;
  93
  94 unreg:
  95         dyn_event_remove(&ep->devent);
  96         trace_probe_unlink(&ep->tp);
  97
  98         return 0;
  99 }
 100
 101 static int eprobe_dyn_event_release(struct dyn_event *ev)
 102 {
 103         struct trace_eprobe *ep = to_trace_eprobe(ev);
 104         int ret = unregister_trace_eprobe(ep);
 105
 106         if (!ret)
 107                 trace_event_probe_cleanup(ep);
 108         return ret;
 109 }
 110
 111 static bool eprobe_dyn_event_is_busy(struct dyn_event *ev)
 112 {
 113         struct trace_eprobe *ep = to_trace_eprobe(ev);
 114
 115         return trace_probe_is_enabled(&ep->tp);
 116 }
 117
 118 static bool eprobe_dyn_event_match(const char *system, const char *event,
 119                         int argc, const char **argv, struct dyn_event *ev)
 120 {
 121         struct trace_eprobe *ep = to_trace_eprobe(ev);
 122         const char *slash;
 123
 124         /*
 125          * We match the following:
 126          *  event only                  - match all eprobes with event name
 127          *  system and event only       - match all system/event probes
 128          *
 129          * The below has the above satisfied with more arguments:
 130          *
 131          *  attached system/event       - If the arg has the system and event
 132          *                                the probe is attached to, match
 133          *                                probes with the attachment.
 134          *
 135          *  If any more args are given, then it requires a full match.
 136          */
 137
 138         /*
 139          * If system exists, but this probe is not part of that system
 140          * do not match.
 141          */
 142         if (system && strcmp(trace_probe_group_name(&ep->tp), system) != 0)
 143                 return false;
 144
 145         /* Must match the event name */
 146         if (strcmp(trace_probe_name(&ep->tp), event) != 0)
 147                 return false;
 148
 149         /* No arguments match all */
 150         if (argc < 1)
 151                 return true;
 152
 153         /* First argument is the system/event the probe is attached to */
 154
 155         slash = strchr(argv[0], '/');
 156         if (!slash)
 157                 slash = strchr(argv[0], '.');
 158         if (!slash)
 159                 return false;
 160
 161         if (strncmp(ep->event_system, argv[0], slash - argv[0]))
 162                 return false;
 163         if (strcmp(ep->event_name, slash + 1))
 164                 return false;
 165
 166         argc--;
 167         argv++;
 168
 169         /* If there are no other args, then match */
 170         if (argc < 1)
 171                 return true;
 172
 173         return trace_probe_match_command_args(&ep->tp, argc, argv);
 174 }
 175
 176 static struct dyn_event_operations eprobe_dyn_event_ops = {
 177         .create = eprobe_dyn_event_create,
 178         .show = eprobe_dyn_event_show,
 179         .is_busy = eprobe_dyn_event_is_busy,
 180         .free = eprobe_dyn_event_release,
 181         .match = eprobe_dyn_event_match,
 182 };
 183
 184 static struct trace_eprobe *alloc_event_probe(const char *group,
 185                                               const char *this_event,
 186                                               struct trace_event_call *event,
 187                                               int nargs)
 188 {
 189         struct trace_eprobe *ep;
 190         const char *event_name;
 191         const char *sys_name;
 192         int ret = -ENOMEM;
 193
 194         if (!event)
 195                 return ERR_PTR(-ENODEV);
 196
 197         sys_name = event->class->system;
 198         event_name = trace_event_name(event);
 199
 200         ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL);
 201         if (!ep) {
 202                 trace_event_put_ref(event);
 203                 goto error;
 204         }
 205         ep->event = event;
 206         ep->event_name = kstrdup(event_name, GFP_KERNEL);
 207         if (!ep->event_name)
 208                 goto error;
 209         ep->event_system = kstrdup(sys_name, GFP_KERNEL);
 210         if (!ep->event_system)
 211                 goto error;
 212
 213         ret = trace_probe_init(&ep->tp, this_event, group, false);
 214         if (ret < 0)
 215                 goto error;
 216
 217         dyn_event_init(&ep->devent, &eprobe_dyn_event_ops);
 218         return ep;
 219 error:
 220         trace_event_probe_cleanup(ep);
 221         return ERR_PTR(ret);
 222 }
 223
 224 static int trace_eprobe_tp_arg_update(struct trace_eprobe *ep, int i)
 225 {
 226         struct probe_arg *parg = &ep->tp.args[i];
 227         struct ftrace_event_field *field;
 228         struct list_head *head;
 229
 230         head = trace_get_fields(ep->event);
 231         list_for_each_entry(field, head, link) {
 232                 if (!strcmp(parg->code->data, field->name)) {
 233                         kfree(parg->code->data);
 234                         parg->code->data = field;
 235                         return 0;
 236                 }
 237         }
 238         kfree(parg->code->data);
 239         parg->code->data = NULL;
 240         return -ENOENT;
 241 }
 242
 243 static int eprobe_event_define_fields(struct trace_event_call *event_call)
 244 {
 245         int ret;
 246         struct eprobe_trace_entry_head field;
 247         struct trace_probe *tp;
 248
 249         tp = trace_probe_primary_from_call(event_call);
 250         if (WARN_ON_ONCE(!tp))
 251                 return -ENOENT;
 252
 253         DEFINE_FIELD(unsigned int, type, FIELD_STRING_TYPE, 0);
 254
 255         return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
 256 }
 257
 258 static struct trace_event_fields eprobe_fields_array[] = {
 259         { .type = TRACE_FUNCTION_TYPE,
 260           .define_fields = eprobe_event_define_fields },
 261         {}
 262 };
 263
 264 /* Event entry printers */
 265 static enum print_line_t
 266 print_eprobe_event(struct trace_iterator *iter, int flags,
 267                    struct trace_event *event)
 268 {
 269         struct eprobe_trace_entry_head *field;
 270         struct trace_event_call *pevent;
 271         struct trace_event *probed_event;
 272         struct trace_seq *s = &iter->seq;
 273         struct trace_probe *tp;
 274
 275         field = (struct eprobe_trace_entry_head *)iter->ent;
 276         tp = trace_probe_primary_from_call(
 277                 container_of(event, struct trace_event_call, event));
 278         if (WARN_ON_ONCE(!tp))
 279                 goto out;
 280
 281         trace_seq_printf(s, "%s: (", trace_probe_name(tp));
 282
 283         probed_event = ftrace_find_event(field->type);
 284         if (probed_event) {
 285                 pevent = container_of(probed_event, struct trace_event_call, event);
 286                 trace_seq_printf(s, "%s.%s", pevent->class->system,
 287                                  trace_event_name(pevent));
 288         } else {
 289                 trace_seq_printf(s, "%u", field->type);
 290         }
 291
 292         trace_seq_putc(s, ')');
 293
 294         if (print_probe_args(s, tp->args, tp->nr_args,
 295                              (u8 *)&field[1], field) < 0)
 296                 goto out;
 297
 298         trace_seq_putc(s, '\n');
 299  out:
 300         return trace_handle_return(s);
 301 }
 302
 303 static unsigned long get_event_field(struct fetch_insn *code, void *rec)
 304 {
 305         struct ftrace_event_field *field = code->data;
 306         unsigned long val;
 307         void *addr;
 308
 309         addr = rec + field->offset;
 310
 311         switch (field->size) {
 312         case 1:
 313                 if (field->is_signed)
 314                         val = *(char *)addr;
 315                 else
 316                         val = *(unsigned char *)addr;
 317                 break;
 318         case 2:
 319                 if (field->is_signed)
 320                         val = *(short *)addr;
 321                 else
 322                         val = *(unsigned short *)addr;
 323                 break;
 324         case 4:
 325                 if (field->is_signed)
 326                         val = *(int *)addr;
 327                 else
 328                         val = *(unsigned int *)addr;
 329                 break;
 330         default:
 331                 if (field->is_signed)
 332                         val = *(long *)addr;
 333                 else
 334                         val = *(unsigned long *)addr;
 335                 break;
 336         }
 337         return val;
 338 }
 339
 340 static int get_eprobe_size(struct trace_probe *tp, void *rec)
 341 {
 342         struct probe_arg *arg;
 343         int i, len, ret = 0;
 344
 345         for (i = 0; i < tp->nr_args; i++) {
 346                 arg = tp->args + i;
 347                 if (unlikely(arg->dynamic)) {
 348                         unsigned long val;
 349
 350                         val = get_event_field(arg->code, rec);
 351                         len = process_fetch_insn_bottom(arg->code + 1, val, NULL, NULL);
 352                         if (len > 0)
 353                                 ret += len;
 354                 }
 355         }
 356
 357         return ret;
 358 }
 359
 360 /* Kprobe specific fetch functions */
 361
 362 /* Note that we don't verify it, since the code does not come from user space */
 363 static int
 364 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
 365                    void *base)
 366 {
 367         unsigned long val;
 368
 369         val = get_event_field(code, rec);
 370         return process_fetch_insn_bottom(code + 1, val, dest, base);
 371 }
 372 NOKPROBE_SYMBOL(process_fetch_insn)
 373
 374 /* Return the length of string -- including null terminal byte */
 375 static nokprobe_inline int
 376 fetch_store_strlen_user(unsigned long addr)
 377 {
 378         const void __user *uaddr =  (__force const void __user *)addr;
 379
 380         return strnlen_user_nofault(uaddr, MAX_STRING_SIZE);
 381 }
 382
 383 /* Return the length of string -- including null terminal byte */
 384 static nokprobe_inline int
 385 fetch_store_strlen(unsigned long addr)
 386 {
 387         int ret, len = 0;
 388         u8 c;
 389
 390 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 391         if (addr < TASK_SIZE)
 392                 return fetch_store_strlen_user(addr);
 393 #endif
 394
 395         do {
 396                 ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1);
 397                 len++;
 398         } while (c && ret == 0 && len < MAX_STRING_SIZE);
 399
 400         return (ret < 0) ? ret : len;
 401 }
 402
 403 /*
 404  * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf
 405  * with max length and relative data location.
 406  */
 407 static nokprobe_inline int
 408 fetch_store_string_user(unsigned long addr, void *dest, void *base)
 409 {
 410         const void __user *uaddr =  (__force const void __user *)addr;
 411         int maxlen = get_loc_len(*(u32 *)dest);
 412         void *__dest;
 413         long ret;
 414
 415         if (unlikely(!maxlen))
 416                 return -ENOMEM;
 417
 418         __dest = get_loc_data(dest, base);
 419
 420         ret = strncpy_from_user_nofault(__dest, uaddr, maxlen);
 421         if (ret >= 0)
 422                 *(u32 *)dest = make_data_loc(ret, __dest - base);
 423
 424         return ret;
 425 }
 426
 427 /*
 428  * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
 429  * length and relative data location.
 430  */
 431 static nokprobe_inline int
 432 fetch_store_string(unsigned long addr, void *dest, void *base)
 433 {
 434         int maxlen = get_loc_len(*(u32 *)dest);
 435         void *__dest;
 436         long ret;
 437
 438 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 439         if ((unsigned long)addr < TASK_SIZE)
 440                 return fetch_store_string_user(addr, dest, base);
 441 #endif
 442
 443         if (unlikely(!maxlen))
 444                 return -ENOMEM;
 445
 446         __dest = get_loc_data(dest, base);
 447
 448         /*
 449          * Try to get string again, since the string can be changed while
 450          * probing.
 451          */
 452         ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen);
 453         if (ret >= 0)
 454                 *(u32 *)dest = make_data_loc(ret, __dest - base);
 455
 456         return ret;
 457 }
 458
 459 static nokprobe_inline int
 460 probe_mem_read_user(void *dest, void *src, size_t size)
 461 {
 462         const void __user *uaddr =  (__force const void __user *)src;
 463
 464         return copy_from_user_nofault(dest, uaddr, size);
 465 }
 466
 467 static nokprobe_inline int
 468 probe_mem_read(void *dest, void *src, size_t size)
 469 {
 470 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 471         if ((unsigned long)src < TASK_SIZE)
 472                 return probe_mem_read_user(dest, src, size);
 473 #endif
 474         return copy_from_kernel_nofault(dest, src, size);
 475 }
 476
 477 /* eprobe handler */
 478 static inline void
 479 __eprobe_trace_func(struct eprobe_data *edata, void *rec)
 480 {
 481         struct eprobe_trace_entry_head *entry;
 482         struct trace_event_call *call = trace_probe_event_call(&edata->ep->tp);
 483         struct trace_event_buffer fbuffer;
 484         int dsize;
 485
 486         if (WARN_ON_ONCE(call != edata->file->event_call))
 487                 return;
 488
 489         if (trace_trigger_soft_disabled(edata->file))
 490                 return;
 491
 492         fbuffer.trace_ctx = tracing_gen_ctx();
 493         fbuffer.trace_file = edata->file;
 494
 495         dsize = get_eprobe_size(&edata->ep->tp, rec);
 496         fbuffer.regs = NULL;
 497
 498         fbuffer.event =
 499                 trace_event_buffer_lock_reserve(&fbuffer.buffer, edata->file,
 500                                         call->event.type,
 501                                         sizeof(*entry) + edata->ep->tp.size + dsize,
 502                                         fbuffer.trace_ctx);
 503         if (!fbuffer.event)
 504                 return;
 505
 506         entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
 507         if (edata->ep->event)
 508                 entry->type = edata->ep->event->event.type;
 509         else
 510                 entry->type = 0;
 511         store_trace_args(&entry[1], &edata->ep->tp, rec, sizeof(*entry), dsize);
 512
 513         trace_event_buffer_commit(&fbuffer);
 514 }
 515
 516 /*
 517  * The event probe implementation uses event triggers to get access to
 518  * the event it is attached to, but is not an actual trigger. The below
 519  * functions are just stubs to fulfill what is needed to use the trigger
 520  * infrastructure.
 521  */
 522 static int eprobe_trigger_init(struct event_trigger_ops *ops,
 523                                struct event_trigger_data *data)
 524 {
 525         return 0;
 526 }
 527
 528 static void eprobe_trigger_free(struct event_trigger_ops *ops,
 529                                 struct event_trigger_data *data)
 530 {
 531
 532 }
 533
 534 static int eprobe_trigger_print(struct seq_file *m,
 535                                 struct event_trigger_ops *ops,
 536                                 struct event_trigger_data *data)
 537 {
 538         /* Do not print eprobe event triggers */
 539         return 0;
 540 }
 541
 542 static void eprobe_trigger_func(struct event_trigger_data *data,
 543                                 struct trace_buffer *buffer, void *rec,
 544                                 struct ring_buffer_event *rbe)
 545 {
 546         struct eprobe_data *edata = data->private_data;
 547
 548         __eprobe_trace_func(edata, rec);
 549 }
 550
 551 static struct event_trigger_ops eprobe_trigger_ops = {
 552         .func                   = eprobe_trigger_func,
 553         .print                  = eprobe_trigger_print,
 554         .init                   = eprobe_trigger_init,
 555         .free                   = eprobe_trigger_free,
 556 };
 557
 558 static int eprobe_trigger_cmd_func(struct event_command *cmd_ops,
 559                                    struct trace_event_file *file,
 560                                    char *glob, char *cmd, char *param)
 561 {
 562         return -1;
 563 }
 564
 565 static int eprobe_trigger_reg_func(char *glob, struct event_trigger_ops *ops,
 566                                  struct event_trigger_data *data,
 567                                  struct trace_event_file *file)
 568 {
 569         return -1;
 570 }
 571
 572 static void eprobe_trigger_unreg_func(char *glob, struct event_trigger_ops *ops,
 573                                     struct event_trigger_data *data,
 574                                     struct trace_event_file *file)
 575 {
 576
 577 }
 578
 579 static struct event_trigger_ops *eprobe_trigger_get_ops(char *cmd,
 580                                                         char *param)
 581 {
 582         return &eprobe_trigger_ops;
 583 }
 584
 585 static struct event_command event_trigger_cmd = {
 586         .name                   = "eprobe",
 587         .trigger_type           = ETT_EVENT_EPROBE,
 588         .flags                  = EVENT_CMD_FL_NEEDS_REC,
 589         .func                   = eprobe_trigger_cmd_func,
 590         .reg                    = eprobe_trigger_reg_func,
 591         .unreg                  = eprobe_trigger_unreg_func,
 592         .unreg_all              = NULL,
 593         .get_trigger_ops        = eprobe_trigger_get_ops,
 594         .set_filter             = NULL,
 595 };
 596
 597 static struct event_trigger_data *
 598 new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file)
 599 {
 600         struct event_trigger_data *trigger;
 601         struct eprobe_data *edata;
 602
 603         edata = kzalloc(sizeof(*edata), GFP_KERNEL);
 604         trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
 605         if (!trigger || !edata) {
 606                 kfree(edata);
 607                 kfree(trigger);
 608                 return ERR_PTR(-ENOMEM);
 609         }
 610
 611         trigger->flags = EVENT_TRIGGER_FL_PROBE;
 612         trigger->count = -1;
 613         trigger->ops = &eprobe_trigger_ops;
 614
 615         /*
 616          * EVENT PROBE triggers are not registered as commands with
 617          * register_event_command(), as they are not controlled by the user
 618          * from the trigger file
 619          */
 620         trigger->cmd_ops = &event_trigger_cmd;
 621
 622         INIT_LIST_HEAD(&trigger->list);
 623         RCU_INIT_POINTER(trigger->filter, NULL);
 624
 625         edata->file = file;
 626         edata->ep = ep;
 627         trigger->private_data = edata;
 628
 629         return trigger;
 630 }
 631
 632 static int enable_eprobe(struct trace_eprobe *ep,
 633                          struct trace_event_file *eprobe_file)
 634 {
 635         struct event_trigger_data *trigger;
 636         struct trace_event_file *file;
 637         struct trace_array *tr = eprobe_file->tr;
 638
 639         file = find_event_file(tr, ep->event_system, ep->event_name);
 640         if (!file)
 641                 return -ENOENT;
 642         trigger = new_eprobe_trigger(ep, eprobe_file);
 643         if (IS_ERR(trigger))
 644                 return PTR_ERR(trigger);
 645
 646         list_add_tail_rcu(&trigger->list, &file->triggers);
 647
 648         trace_event_trigger_enable_disable(file, 1);
 649         update_cond_flag(file);
 650
 651         return 0;
 652 }
 653
 654 static struct trace_event_functions eprobe_funcs = {
 655         .trace          = print_eprobe_event
 656 };
 657
 658 static int disable_eprobe(struct trace_eprobe *ep,
 659                           struct trace_array *tr)
 660 {
 661         struct event_trigger_data *trigger;
 662         struct trace_event_file *file;
 663         struct eprobe_data *edata;
 664
 665         file = find_event_file(tr, ep->event_system, ep->event_name);
 666         if (!file)
 667                 return -ENOENT;
 668
 669         list_for_each_entry(trigger, &file->triggers, list) {
 670                 if (!(trigger->flags & EVENT_TRIGGER_FL_PROBE))
 671                         continue;
 672                 edata = trigger->private_data;
 673                 if (edata->ep == ep)
 674                         break;
 675         }
 676         if (list_entry_is_head(trigger, &file->triggers, list))
 677                 return -ENODEV;
 678
 679         list_del_rcu(&trigger->list);
 680
 681         trace_event_trigger_enable_disable(file, 0);
 682         update_cond_flag(file);
 683
 684         /* Make sure nothing is using the edata or trigger */
 685         tracepoint_synchronize_unregister();
 686
 687         kfree(edata);
 688         kfree(trigger);
 689
 690         return 0;
 691 }
 692
 693 static int enable_trace_eprobe(struct trace_event_call *call,
 694                                struct trace_event_file *file)
 695 {
 696         struct trace_probe *pos, *tp;
 697         struct trace_eprobe *ep;
 698         bool enabled;
 699         int ret = 0;
 700
 701         tp = trace_probe_primary_from_call(call);
 702         if (WARN_ON_ONCE(!tp))
 703                 return -ENODEV;
 704         enabled = trace_probe_is_enabled(tp);
 705
 706         /* This also changes "enabled" state */
 707         if (file) {
 708                 ret = trace_probe_add_file(tp, file);
 709                 if (ret)
 710                         return ret;
 711         } else
 712                 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
 713
 714         if (enabled)
 715                 return 0;
 716
 717         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
 718                 ep = container_of(pos, struct trace_eprobe, tp);
 719                 ret = enable_eprobe(ep, file);
 720                 if (ret)
 721                         break;
 722                 enabled = true;
 723         }
 724
 725         if (ret) {
 726                 /* Failed to enable one of them. Roll back all */
 727                 if (enabled)
 728                         disable_eprobe(ep, file->tr);
 729                 if (file)
 730                         trace_probe_remove_file(tp, file);
 731                 else
 732                         trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 733         }
 734
 735         return ret;
 736 }
 737
 738 static int disable_trace_eprobe(struct trace_event_call *call,
 739                                 struct trace_event_file *file)
 740 {
 741         struct trace_probe *pos, *tp;
 742         struct trace_eprobe *ep;
 743
 744         tp = trace_probe_primary_from_call(call);
 745         if (WARN_ON_ONCE(!tp))
 746                 return -ENODEV;
 747
 748         if (file) {
 749                 if (!trace_probe_get_file_link(tp, file))
 750                         return -ENOENT;
 751                 if (!trace_probe_has_single_file(tp))
 752                         goto out;
 753                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
 754         } else
 755                 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 756
 757         if (!trace_probe_is_enabled(tp)) {
 758                 list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
 759                         ep = container_of(pos, struct trace_eprobe, tp);
 760                         disable_eprobe(ep, file->tr);
 761                 }
 762         }
 763
 764  out:
 765         if (file)
 766                 /*
 767                  * Synchronization is done in below function. For perf event,
 768                  * file == NULL and perf_trace_event_unreg() calls
 769                  * tracepoint_synchronize_unregister() to ensure synchronize
 770                  * event. We don't need to care about it.
 771                  */
 772                 trace_probe_remove_file(tp, file);
 773
 774         return 0;
 775 }
 776
 777 static int eprobe_register(struct trace_event_call *event,
 778                            enum trace_reg type, void *data)
 779 {
 780         struct trace_event_file *file = data;
 781
 782         switch (type) {
 783         case TRACE_REG_REGISTER:
 784                 return enable_trace_eprobe(event, file);
 785         case TRACE_REG_UNREGISTER:
 786                 return disable_trace_eprobe(event, file);
 787 #ifdef CONFIG_PERF_EVENTS
 788         case TRACE_REG_PERF_REGISTER:
 789         case TRACE_REG_PERF_UNREGISTER:
 790         case TRACE_REG_PERF_OPEN:
 791         case TRACE_REG_PERF_CLOSE:
 792         case TRACE_REG_PERF_ADD:
 793         case TRACE_REG_PERF_DEL:
 794                 return 0;
 795 #endif
 796         }
 797         return 0;
 798 }
 799
 800 static inline void init_trace_eprobe_call(struct trace_eprobe *ep)
 801 {
 802         struct trace_event_call *call = trace_probe_event_call(&ep->tp);
 803
 804         call->flags = TRACE_EVENT_FL_EPROBE;
 805         call->event.funcs = &eprobe_funcs;
 806         call->class->fields_array = eprobe_fields_array;
 807         call->class->reg = eprobe_register;
 808 }
 809
 810 static struct trace_event_call *
 811 find_and_get_event(const char *system, const char *event_name)
 812 {
 813         struct trace_event_call *tp_event;
 814         const char *name;
 815
 816         list_for_each_entry(tp_event, &ftrace_events, list) {
 817                 /* Skip other probes and ftrace events */
 818                 if (tp_event->flags &
 819                     (TRACE_EVENT_FL_IGNORE_ENABLE |
 820                      TRACE_EVENT_FL_KPROBE |
 821                      TRACE_EVENT_FL_UPROBE |
 822                      TRACE_EVENT_FL_EPROBE))
 823                         continue;
 824                 if (!tp_event->class->system ||
 825                     strcmp(system, tp_event->class->system))
 826                         continue;
 827                 name = trace_event_name(tp_event);
 828                 if (!name || strcmp(event_name, name))
 829                         continue;
 830                 if (!trace_event_try_get_ref(tp_event)) {
 831                         return NULL;
 832                         break;
 833                 }
 834                 return tp_event;
 835                 break;
 836         }
 837         return NULL;
 838 }
 839
 840 static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i)
 841 {
 842         unsigned int flags = TPARG_FL_KERNEL | TPARG_FL_TPOINT;
 843         int ret;
 844
 845         ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], flags);
 846         if (ret)
 847                 return ret;
 848
 849         if (ep->tp.args[i].code->op == FETCH_OP_TP_ARG)
 850                 ret = trace_eprobe_tp_arg_update(ep, i);
 851
 852         return ret;
 853 }
 854
 855 static int __trace_eprobe_create(int argc, const char *argv[])
 856 {
 857         /*
 858          * Argument syntax:
 859          *      e[:[GRP/]ENAME] SYSTEM.EVENT [FETCHARGS]
 860          * Fetch args:
 861          *  <name>=$<field>[:TYPE]
 862          */
 863         const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
 864         const char *sys_event = NULL, *sys_name = NULL;
 865         struct trace_event_call *event_call;
 866         struct trace_eprobe *ep = NULL;
 867         char buf1[MAX_EVENT_NAME_LEN];
 868         char buf2[MAX_EVENT_NAME_LEN];
 869         int ret = 0;
 870         int i;
 871
 872         if (argc < 2 || argv[0][0] != 'e')
 873                 return -ECANCELED;
 874
 875         trace_probe_log_init("event_probe", argc, argv);
 876
 877         event = strchr(&argv[0][1], ':');
 878         if (event) {
 879                 event++;
 880                 ret = traceprobe_parse_event_name(&event, &group, buf1,
 881                                                   event - argv[0]);
 882                 if (ret)
 883                         goto parse_error;
 884         } else {
 885                 strscpy(buf1, argv[1], MAX_EVENT_NAME_LEN);
 886                 sanitize_event_name(buf1);
 887                 event = buf1;
 888         }
 889         if (!is_good_name(event) || !is_good_name(group))
 890                 goto parse_error;
 891
 892         sys_event = argv[1];
 893         ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2,
 894                                           sys_event - argv[1]);
 895         if (ret || !sys_name)
 896                 goto parse_error;
 897         if (!is_good_name(sys_event) || !is_good_name(sys_name))
 898                 goto parse_error;
 899
 900         mutex_lock(&event_mutex);
 901         event_call = find_and_get_event(sys_name, sys_event);
 902         ep = alloc_event_probe(group, event, event_call, argc - 2);
 903         mutex_unlock(&event_mutex);
 904
 905         if (IS_ERR(ep)) {
 906                 ret = PTR_ERR(ep);
 907                 /* This must return -ENOMEM, else there is a bug */
 908                 WARN_ON_ONCE(ret != -ENOMEM);
 909                 ep = NULL;
 910                 goto error;
 911         }
 912
 913         argc -= 2; argv += 2;
 914         /* parse arguments */
 915         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
 916                 trace_probe_log_set_index(i + 2);
 917                 ret = trace_eprobe_tp_update_arg(ep, argv, i);
 918                 if (ret)
 919                         goto error;
 920         }
 921         ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT);
 922         if (ret < 0)
 923                 goto error;
 924         init_trace_eprobe_call(ep);
 925         mutex_lock(&event_mutex);
 926         ret = trace_probe_register_event_call(&ep->tp);
 927         if (ret) {
 928                 if (ret == -EEXIST) {
 929                         trace_probe_log_set_index(0);
 930                         trace_probe_log_err(0, EVENT_EXIST);
 931                 }
 932                 mutex_unlock(&event_mutex);
 933                 goto error;
 934         }
 935         ret = dyn_event_add(&ep->devent, &ep->tp.event->call);
 936         mutex_unlock(&event_mutex);
 937         return ret;
 938 parse_error:
 939         ret = -EINVAL;
 940 error:
 941         trace_event_probe_cleanup(ep);
 942         return ret;
 943 }
 944
 945 /*
 946  * Register dynevent at core_initcall. This allows kernel to setup eprobe
 947  * events in postcore_initcall without tracefs.
 948  */
 949 static __init int trace_events_eprobe_init_early(void)
 950 {
 951         int err = 0;
 952
 953         err = dyn_event_register(&eprobe_dyn_event_ops);
 954         if (err)
 955                 pr_warn("Could not register eprobe_dyn_event_ops\n");
 956
 957         return err;
 958 }
 959 core_initcall(trace_events_eprobe_init_early);