kernel/trace/trace_events.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * event tracer
   4  *
   5  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
   6  *
   7  *  - Added format output of fields of the trace point.
   8  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
   9  *
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/workqueue.h>
  15 #include <linux/security.h>
  16 #include <linux/spinlock.h>
  17 #include <linux/kthread.h>
  18 #include <linux/tracefs.h>
  19 #include <linux/uaccess.h>
  20 #include <linux/module.h>
  21 #include <linux/ctype.h>
  22 #include <linux/sort.h>
  23 #include <linux/slab.h>
  24 #include <linux/delay.h>
  25
  26 #include <trace/events/sched.h>
  27 #include <trace/syscall.h>
  28
  29 #include <asm/setup.h>
  30
  31 #include "trace_output.h"
  32
  33 #undef TRACE_SYSTEM
  34 #define TRACE_SYSTEM "TRACE_SYSTEM"
  35
  36 DEFINE_MUTEX(event_mutex);
  37
  38 LIST_HEAD(ftrace_events);
  39 static LIST_HEAD(ftrace_generic_fields);
  40 static LIST_HEAD(ftrace_common_fields);
  41
  42 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
  43
  44 static struct kmem_cache *field_cachep;
  45 static struct kmem_cache *file_cachep;
  46
  47 static inline int system_refcount(struct event_subsystem *system)
  48 {
  49         return system->ref_count;
  50 }
  51
  52 static int system_refcount_inc(struct event_subsystem *system)
  53 {
  54         return system->ref_count++;
  55 }
  56
  57 static int system_refcount_dec(struct event_subsystem *system)
  58 {
  59         return --system->ref_count;
  60 }
  61
  62 /* Double loops, do not use break, only goto's work */
  63 #define do_for_each_event_file(tr, file)                        \
  64         list_for_each_entry(tr, &ftrace_trace_arrays, list) {   \
  65                 list_for_each_entry(file, &tr->events, list)
  66
  67 #define do_for_each_event_file_safe(tr, file)                   \
  68         list_for_each_entry(tr, &ftrace_trace_arrays, list) {   \
  69                 struct trace_event_file *___n;                          \
  70                 list_for_each_entry_safe(file, ___n, &tr->events, list)
  71
  72 #define while_for_each_event_file()             \
  73         }
  74
  75 static struct ftrace_event_field *
  76 __find_event_field(struct list_head *head, char *name)
  77 {
  78         struct ftrace_event_field *field;
  79
  80         list_for_each_entry(field, head, link) {
  81                 if (!strcmp(field->name, name))
  82                         return field;
  83         }
  84
  85         return NULL;
  86 }
  87
  88 struct ftrace_event_field *
  89 trace_find_event_field(struct trace_event_call *call, char *name)
  90 {
  91         struct ftrace_event_field *field;
  92         struct list_head *head;
  93
  94         head = trace_get_fields(call);
  95         field = __find_event_field(head, name);
  96         if (field)
  97                 return field;
  98
  99         field = __find_event_field(&ftrace_generic_fields, name);
 100         if (field)
 101                 return field;
 102
 103         return __find_event_field(&ftrace_common_fields, name);
 104 }
 105
 106 static int __trace_define_field(struct list_head *head, const char *type,
 107                                 const char *name, int offset, int size,
 108                                 int is_signed, int filter_type)
 109 {
 110         struct ftrace_event_field *field;
 111
 112         field = kmem_cache_alloc(field_cachep, GFP_TRACE);
 113         if (!field)
 114                 return -ENOMEM;
 115
 116         field->name = name;
 117         field->type = type;
 118
 119         if (filter_type == FILTER_OTHER)
 120                 field->filter_type = filter_assign_type(type);
 121         else
 122                 field->filter_type = filter_type;
 123
 124         field->offset = offset;
 125         field->size = size;
 126         field->is_signed = is_signed;
 127
 128         list_add(&field->link, head);
 129
 130         return 0;
 131 }
 132
 133 int trace_define_field(struct trace_event_call *call, const char *type,
 134                        const char *name, int offset, int size, int is_signed,
 135                        int filter_type)
 136 {
 137         struct list_head *head;
 138
 139         if (WARN_ON(!call->class))
 140                 return 0;
 141
 142         head = trace_get_fields(call);
 143         return __trace_define_field(head, type, name, offset, size,
 144                                     is_signed, filter_type);
 145 }
 146 EXPORT_SYMBOL_GPL(trace_define_field);
 147
 148 #define __generic_field(type, item, filter_type)                        \
 149         ret = __trace_define_field(&ftrace_generic_fields, #type,       \
 150                                    #item, 0, 0, is_signed_type(type),   \
 151                                    filter_type);                        \
 152         if (ret)                                                        \
 153                 return ret;
 154
 155 #define __common_field(type, item)                                      \
 156         ret = __trace_define_field(&ftrace_common_fields, #type,        \
 157                                    "common_" #item,                     \
 158                                    offsetof(typeof(ent), item),         \
 159                                    sizeof(ent.item),                    \
 160                                    is_signed_type(type), FILTER_OTHER); \
 161         if (ret)                                                        \
 162                 return ret;
 163
 164 static int trace_define_generic_fields(void)
 165 {
 166         int ret;
 167
 168         __generic_field(int, CPU, FILTER_CPU);
 169         __generic_field(int, cpu, FILTER_CPU);
 170         __generic_field(char *, COMM, FILTER_COMM);
 171         __generic_field(char *, comm, FILTER_COMM);
 172
 173         return ret;
 174 }
 175
 176 static int trace_define_common_fields(void)
 177 {
 178         int ret;
 179         struct trace_entry ent;
 180
 181         __common_field(unsigned short, type);
 182         __common_field(unsigned char, flags);
 183         __common_field(unsigned char, preempt_count);
 184         __common_field(int, pid);
 185
 186         return ret;
 187 }
 188
 189 static void trace_destroy_fields(struct trace_event_call *call)
 190 {
 191         struct ftrace_event_field *field, *next;
 192         struct list_head *head;
 193
 194         head = trace_get_fields(call);
 195         list_for_each_entry_safe(field, next, head, link) {
 196                 list_del(&field->link);
 197                 kmem_cache_free(field_cachep, field);
 198         }
 199 }
 200
 201 /*
 202  * run-time version of trace_event_get_offsets_<call>() that returns the last
 203  * accessible offset of trace fields excluding __dynamic_array bytes
 204  */
 205 int trace_event_get_offsets(struct trace_event_call *call)
 206 {
 207         struct ftrace_event_field *tail;
 208         struct list_head *head;
 209
 210         head = trace_get_fields(call);
 211         /*
 212          * head->next points to the last field with the largest offset,
 213          * since it was added last by trace_define_field()
 214          */
 215         tail = list_first_entry(head, struct ftrace_event_field, link);
 216         return tail->offset + tail->size;
 217 }
 218
 219 int trace_event_raw_init(struct trace_event_call *call)
 220 {
 221         int id;
 222
 223         id = register_trace_event(&call->event);
 224         if (!id)
 225                 return -ENODEV;
 226
 227         return 0;
 228 }
 229 EXPORT_SYMBOL_GPL(trace_event_raw_init);
 230
 231 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
 232 {
 233         struct trace_array *tr = trace_file->tr;
 234         struct trace_array_cpu *data;
 235         struct trace_pid_list *pid_list;
 236
 237         pid_list = rcu_dereference_raw(tr->filtered_pids);
 238         if (!pid_list)
 239                 return false;
 240
 241         data = this_cpu_ptr(tr->trace_buffer.data);
 242
 243         return data->ignore_pid;
 244 }
 245 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
 246
 247 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
 248                                  struct trace_event_file *trace_file,
 249                                  unsigned long len)
 250 {
 251         struct trace_event_call *event_call = trace_file->event_call;
 252
 253         if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
 254             trace_event_ignore_this_pid(trace_file))
 255                 return NULL;
 256
 257         local_save_flags(fbuffer->flags);
 258         fbuffer->pc = preempt_count();
 259         /*
 260          * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
 261          * preemption (adding one to the preempt_count). Since we are
 262          * interested in the preempt_count at the time the tracepoint was
 263          * hit, we need to subtract one to offset the increment.
 264          */
 265         if (IS_ENABLED(CONFIG_PREEMPTION))
 266                 fbuffer->pc--;
 267         fbuffer->trace_file = trace_file;
 268
 269         fbuffer->event =
 270                 trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
 271                                                 event_call->event.type, len,
 272                                                 fbuffer->flags, fbuffer->pc);
 273         if (!fbuffer->event)
 274                 return NULL;
 275
 276         fbuffer->entry = ring_buffer_event_data(fbuffer->event);
 277         return fbuffer->entry;
 278 }
 279 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
 280
 281 int trace_event_reg(struct trace_event_call *call,
 282                     enum trace_reg type, void *data)
 283 {
 284         struct trace_event_file *file = data;
 285
 286         WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
 287         switch (type) {
 288         case TRACE_REG_REGISTER:
 289                 return tracepoint_probe_register(call->tp,
 290                                                  call->class->probe,
 291                                                  file);
 292         case TRACE_REG_UNREGISTER:
 293                 tracepoint_probe_unregister(call->tp,
 294                                             call->class->probe,
 295                                             file);
 296                 return 0;
 297
 298 #ifdef CONFIG_PERF_EVENTS
 299         case TRACE_REG_PERF_REGISTER:
 300                 return tracepoint_probe_register(call->tp,
 301                                                  call->class->perf_probe,
 302                                                  call);
 303         case TRACE_REG_PERF_UNREGISTER:
 304                 tracepoint_probe_unregister(call->tp,
 305                                             call->class->perf_probe,
 306                                             call);
 307                 return 0;
 308         case TRACE_REG_PERF_OPEN:
 309         case TRACE_REG_PERF_CLOSE:
 310         case TRACE_REG_PERF_ADD:
 311         case TRACE_REG_PERF_DEL:
 312                 return 0;
 313 #endif
 314         }
 315         return 0;
 316 }
 317 EXPORT_SYMBOL_GPL(trace_event_reg);
 318
 319 void trace_event_enable_cmd_record(bool enable)
 320 {
 321         struct trace_event_file *file;
 322         struct trace_array *tr;
 323
 324         lockdep_assert_held(&event_mutex);
 325
 326         do_for_each_event_file(tr, file) {
 327
 328                 if (!(file->flags & EVENT_FILE_FL_ENABLED))
 329                         continue;
 330
 331                 if (enable) {
 332                         tracing_start_cmdline_record();
 333                         set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 334                 } else {
 335                         tracing_stop_cmdline_record();
 336                         clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 337                 }
 338         } while_for_each_event_file();
 339 }
 340
 341 void trace_event_enable_tgid_record(bool enable)
 342 {
 343         struct trace_event_file *file;
 344         struct trace_array *tr;
 345
 346         lockdep_assert_held(&event_mutex);
 347
 348         do_for_each_event_file(tr, file) {
 349                 if (!(file->flags & EVENT_FILE_FL_ENABLED))
 350                         continue;
 351
 352                 if (enable) {
 353                         tracing_start_tgid_record();
 354                         set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 355                 } else {
 356                         tracing_stop_tgid_record();
 357                         clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
 358                                   &file->flags);
 359                 }
 360         } while_for_each_event_file();
 361 }
 362
 363 static int __ftrace_event_enable_disable(struct trace_event_file *file,
 364                                          int enable, int soft_disable)
 365 {
 366         struct trace_event_call *call = file->event_call;
 367         struct trace_array *tr = file->tr;
 368         unsigned long file_flags = file->flags;
 369         int ret = 0;
 370         int disable;
 371
 372         switch (enable) {
 373         case 0:
 374                 /*
 375                  * When soft_disable is set and enable is cleared, the sm_ref
 376                  * reference counter is decremented. If it reaches 0, we want
 377                  * to clear the SOFT_DISABLED flag but leave the event in the
 378                  * state that it was. That is, if the event was enabled and
 379                  * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
 380                  * is set we do not want the event to be enabled before we
 381                  * clear the bit.
 382                  *
 383                  * When soft_disable is not set but the SOFT_MODE flag is,
 384                  * we do nothing. Do not disable the tracepoint, otherwise
 385                  * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
 386                  */
 387                 if (soft_disable) {
 388                         if (atomic_dec_return(&file->sm_ref) > 0)
 389                                 break;
 390                         disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
 391                         clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
 392                 } else
 393                         disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
 394
 395                 if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
 396                         clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 397                         if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
 398                                 tracing_stop_cmdline_record();
 399                                 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 400                         }
 401
 402                         if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
 403                                 tracing_stop_tgid_record();
 404                                 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 405                         }
 406
 407                         call->class->reg(call, TRACE_REG_UNREGISTER, file);
 408                 }
 409                 /* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
 410                 if (file->flags & EVENT_FILE_FL_SOFT_MODE)
 411                         set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 412                 else
 413                         clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 414                 break;
 415         case 1:
 416                 /*
 417                  * When soft_disable is set and enable is set, we want to
 418                  * register the tracepoint for the event, but leave the event
 419                  * as is. That means, if the event was already enabled, we do
 420                  * nothing (but set SOFT_MODE). If the event is disabled, we
 421                  * set SOFT_DISABLED before enabling the event tracepoint, so
 422                  * it still seems to be disabled.
 423                  */
 424                 if (!soft_disable)
 425                         clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 426                 else {
 427                         if (atomic_inc_return(&file->sm_ref) > 1)
 428                                 break;
 429                         set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
 430                 }
 431
 432                 if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
 433                         bool cmd = false, tgid = false;
 434
 435                         /* Keep the event disabled, when going to SOFT_MODE. */
 436                         if (soft_disable)
 437                                 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 438
 439                         if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
 440                                 cmd = true;
 441                                 tracing_start_cmdline_record();
 442                                 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 443                         }
 444
 445                         if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
 446                                 tgid = true;
 447                                 tracing_start_tgid_record();
 448                                 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 449                         }
 450
 451                         ret = call->class->reg(call, TRACE_REG_REGISTER, file);
 452                         if (ret) {
 453                                 if (cmd)
 454                                         tracing_stop_cmdline_record();
 455                                 if (tgid)
 456                                         tracing_stop_tgid_record();
 457                                 pr_info("event trace: Could not enable event "
 458                                         "%s\n", trace_event_name(call));
 459                                 break;
 460                         }
 461                         set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 462
 463                         /* WAS_ENABLED gets set but never cleared. */
 464                         set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
 465                 }
 466                 break;
 467         }
 468
 469         /* Enable or disable use of trace_buffered_event */
 470         if ((file_flags & EVENT_FILE_FL_SOFT_DISABLED) !=
 471             (file->flags & EVENT_FILE_FL_SOFT_DISABLED)) {
 472                 if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
 473                         trace_buffered_event_enable();
 474                 else
 475                         trace_buffered_event_disable();
 476         }
 477
 478         return ret;
 479 }
 480
 481 int trace_event_enable_disable(struct trace_event_file *file,
 482                                int enable, int soft_disable)
 483 {
 484         return __ftrace_event_enable_disable(file, enable, soft_disable);
 485 }
 486
 487 static int ftrace_event_enable_disable(struct trace_event_file *file,
 488                                        int enable)
 489 {
 490         return __ftrace_event_enable_disable(file, enable, 0);
 491 }
 492
 493 static void ftrace_clear_events(struct trace_array *tr)
 494 {
 495         struct trace_event_file *file;
 496
 497         mutex_lock(&event_mutex);
 498         list_for_each_entry(file, &tr->events, list) {
 499                 ftrace_event_enable_disable(file, 0);
 500         }
 501         mutex_unlock(&event_mutex);
 502 }
 503
 504 static void
 505 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
 506 {
 507         struct trace_pid_list *pid_list;
 508         struct trace_array *tr = data;
 509
 510         pid_list = rcu_dereference_raw(tr->filtered_pids);
 511         trace_filter_add_remove_task(pid_list, NULL, task);
 512 }
 513
 514 static void
 515 event_filter_pid_sched_process_fork(void *data,
 516                                     struct task_struct *self,
 517                                     struct task_struct *task)
 518 {
 519         struct trace_pid_list *pid_list;
 520         struct trace_array *tr = data;
 521
 522         pid_list = rcu_dereference_sched(tr->filtered_pids);
 523         trace_filter_add_remove_task(pid_list, self, task);
 524 }
 525
 526 void trace_event_follow_fork(struct trace_array *tr, bool enable)
 527 {
 528         if (enable) {
 529                 register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
 530                                                        tr, INT_MIN);
 531                 register_trace_prio_sched_process_exit(event_filter_pid_sched_process_exit,
 532                                                        tr, INT_MAX);
 533         } else {
 534                 unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
 535                                                     tr);
 536                 unregister_trace_sched_process_exit(event_filter_pid_sched_process_exit,
 537                                                     tr);
 538         }
 539 }
 540
 541 static void
 542 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
 543                     struct task_struct *prev, struct task_struct *next)
 544 {
 545         struct trace_array *tr = data;
 546         struct trace_pid_list *pid_list;
 547
 548         pid_list = rcu_dereference_sched(tr->filtered_pids);
 549
 550         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 551                        trace_ignore_this_task(pid_list, prev) &&
 552                        trace_ignore_this_task(pid_list, next));
 553 }
 554
 555 static void
 556 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
 557                     struct task_struct *prev, struct task_struct *next)
 558 {
 559         struct trace_array *tr = data;
 560         struct trace_pid_list *pid_list;
 561
 562         pid_list = rcu_dereference_sched(tr->filtered_pids);
 563
 564         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 565                        trace_ignore_this_task(pid_list, next));
 566 }
 567
 568 static void
 569 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
 570 {
 571         struct trace_array *tr = data;
 572         struct trace_pid_list *pid_list;
 573
 574         /* Nothing to do if we are already tracing */
 575         if (!this_cpu_read(tr->trace_buffer.data->ignore_pid))
 576                 return;
 577
 578         pid_list = rcu_dereference_sched(tr->filtered_pids);
 579
 580         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 581                        trace_ignore_this_task(pid_list, task));
 582 }
 583
 584 static void
 585 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
 586 {
 587         struct trace_array *tr = data;
 588         struct trace_pid_list *pid_list;
 589
 590         /* Nothing to do if we are not tracing */
 591         if (this_cpu_read(tr->trace_buffer.data->ignore_pid))
 592                 return;
 593
 594         pid_list = rcu_dereference_sched(tr->filtered_pids);
 595
 596         /* Set tracing if current is enabled */
 597         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 598                        trace_ignore_this_task(pid_list, current));
 599 }
 600
 601 static void __ftrace_clear_event_pids(struct trace_array *tr)
 602 {
 603         struct trace_pid_list *pid_list;
 604         struct trace_event_file *file;
 605         int cpu;
 606
 607         pid_list = rcu_dereference_protected(tr->filtered_pids,
 608                                              lockdep_is_held(&event_mutex));
 609         if (!pid_list)
 610                 return;
 611
 612         unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
 613         unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
 614
 615         unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
 616         unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
 617
 618         unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
 619         unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
 620
 621         unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
 622         unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
 623
 624         list_for_each_entry(file, &tr->events, list) {
 625                 clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
 626         }
 627
 628         for_each_possible_cpu(cpu)
 629                 per_cpu_ptr(tr->trace_buffer.data, cpu)->ignore_pid = false;
 630
 631         rcu_assign_pointer(tr->filtered_pids, NULL);
 632
 633         /* Wait till all users are no longer using pid filtering */
 634         tracepoint_synchronize_unregister();
 635
 636         trace_free_pid_list(pid_list);
 637 }
 638
 639 static void ftrace_clear_event_pids(struct trace_array *tr)
 640 {
 641         mutex_lock(&event_mutex);
 642         __ftrace_clear_event_pids(tr);
 643         mutex_unlock(&event_mutex);
 644 }
 645
 646 static void __put_system(struct event_subsystem *system)
 647 {
 648         struct event_filter *filter = system->filter;
 649
 650         WARN_ON_ONCE(system_refcount(system) == 0);
 651         if (system_refcount_dec(system))
 652                 return;
 653
 654         list_del(&system->list);
 655
 656         if (filter) {
 657                 kfree(filter->filter_string);
 658                 kfree(filter);
 659         }
 660         kfree_const(system->name);
 661         kfree(system);
 662 }
 663
 664 static void __get_system(struct event_subsystem *system)
 665 {
 666         WARN_ON_ONCE(system_refcount(system) == 0);
 667         system_refcount_inc(system);
 668 }
 669
 670 static void __get_system_dir(struct trace_subsystem_dir *dir)
 671 {
 672         WARN_ON_ONCE(dir->ref_count == 0);
 673         dir->ref_count++;
 674         __get_system(dir->subsystem);
 675 }
 676
 677 static void __put_system_dir(struct trace_subsystem_dir *dir)
 678 {
 679         WARN_ON_ONCE(dir->ref_count == 0);
 680         /* If the subsystem is about to be freed, the dir must be too */
 681         WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
 682
 683         __put_system(dir->subsystem);
 684         if (!--dir->ref_count)
 685                 kfree(dir);
 686 }
 687
 688 static void put_system(struct trace_subsystem_dir *dir)
 689 {
 690         mutex_lock(&event_mutex);
 691         __put_system_dir(dir);
 692         mutex_unlock(&event_mutex);
 693 }
 694
 695 static void remove_subsystem(struct trace_subsystem_dir *dir)
 696 {
 697         if (!dir)
 698                 return;
 699
 700         if (!--dir->nr_events) {
 701                 tracefs_remove_recursive(dir->entry);
 702                 list_del(&dir->list);
 703                 __put_system_dir(dir);
 704         }
 705 }
 706
 707 static void remove_event_file_dir(struct trace_event_file *file)
 708 {
 709         struct dentry *dir = file->dir;
 710         struct dentry *child;
 711
 712         if (dir) {
 713                 spin_lock(&dir->d_lock);        /* probably unneeded */
 714                 list_for_each_entry(child, &dir->d_subdirs, d_child) {
 715                         if (d_really_is_positive(child))        /* probably unneeded */
 716                                 d_inode(child)->i_private = NULL;
 717                 }
 718                 spin_unlock(&dir->d_lock);
 719
 720                 tracefs_remove_recursive(dir);
 721         }
 722
 723         list_del(&file->list);
 724         remove_subsystem(file->system);
 725         free_event_filter(file->filter);
 726         kmem_cache_free(file_cachep, file);
 727 }
 728
 729 /*
 730  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
 731  */
 732 static int
 733 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
 734                               const char *sub, const char *event, int set)
 735 {
 736         struct trace_event_file *file;
 737         struct trace_event_call *call;
 738         const char *name;
 739         int ret = -EINVAL;
 740         int eret = 0;
 741
 742         list_for_each_entry(file, &tr->events, list) {
 743
 744                 call = file->event_call;
 745                 name = trace_event_name(call);
 746
 747                 if (!name || !call->class || !call->class->reg)
 748                         continue;
 749
 750                 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
 751                         continue;
 752
 753                 if (match &&
 754                     strcmp(match, name) != 0 &&
 755                     strcmp(match, call->class->system) != 0)
 756                         continue;
 757
 758                 if (sub && strcmp(sub, call->class->system) != 0)
 759                         continue;
 760
 761                 if (event && strcmp(event, name) != 0)
 762                         continue;
 763
 764                 ret = ftrace_event_enable_disable(file, set);
 765
 766                 /*
 767                  * Save the first error and return that. Some events
 768                  * may still have been enabled, but let the user
 769                  * know that something went wrong.
 770                  */
 771                 if (ret && !eret)
 772                         eret = ret;
 773
 774                 ret = eret;
 775         }
 776
 777         return ret;
 778 }
 779
 780 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
 781                                   const char *sub, const char *event, int set)
 782 {
 783         int ret;
 784
 785         mutex_lock(&event_mutex);
 786         ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
 787         mutex_unlock(&event_mutex);
 788
 789         return ret;
 790 }
 791
 792 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
 793 {
 794         char *event = NULL, *sub = NULL, *match;
 795         int ret;
 796
 797         if (!tr)
 798                 return -ENOENT;
 799         /*
 800          * The buf format can be <subsystem>:<event-name>
 801          *  *:<event-name> means any event by that name.
 802          *  :<event-name> is the same.
 803          *
 804          *  <subsystem>:* means all events in that subsystem
 805          *  <subsystem>: means the same.
 806          *
 807          *  <name> (no ':') means all events in a subsystem with
 808          *  the name <name> or any event that matches <name>
 809          */
 810
 811         match = strsep(&buf, ":");
 812         if (buf) {
 813                 sub = match;
 814                 event = buf;
 815                 match = NULL;
 816
 817                 if (!strlen(sub) || strcmp(sub, "*") == 0)
 818                         sub = NULL;
 819                 if (!strlen(event) || strcmp(event, "*") == 0)
 820                         event = NULL;
 821         }
 822
 823         ret = __ftrace_set_clr_event(tr, match, sub, event, set);
 824
 825         /* Put back the colon to allow this to be called again */
 826         if (buf)
 827                 *(buf - 1) = ':';
 828
 829         return ret;
 830 }
 831
 832 /**
 833  * trace_set_clr_event - enable or disable an event
 834  * @system: system name to match (NULL for any system)
 835  * @event: event name to match (NULL for all events, within system)
 836  * @set: 1 to enable, 0 to disable
 837  *
 838  * This is a way for other parts of the kernel to enable or disable
 839  * event recording.
 840  *
 841  * Returns 0 on success, -EINVAL if the parameters do not match any
 842  * registered events.
 843  */
 844 int trace_set_clr_event(const char *system, const char *event, int set)
 845 {
 846         struct trace_array *tr = top_trace_array();
 847
 848         if (!tr)
 849                 return -ENODEV;
 850
 851         return __ftrace_set_clr_event(tr, NULL, system, event, set);
 852 }
 853 EXPORT_SYMBOL_GPL(trace_set_clr_event);
 854
 855 /**
 856  * trace_array_set_clr_event - enable or disable an event for a trace array.
 857  * @tr: concerned trace array.
 858  * @system: system name to match (NULL for any system)
 859  * @event: event name to match (NULL for all events, within system)
 860  * @enable: true to enable, false to disable
 861  *
 862  * This is a way for other parts of the kernel to enable or disable
 863  * event recording.
 864  *
 865  * Returns 0 on success, -EINVAL if the parameters do not match any
 866  * registered events.
 867  */
 868 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
 869                 const char *event, bool enable)
 870 {
 871         int set;
 872
 873         if (!tr)
 874                 return -ENOENT;
 875
 876         set = (enable == true) ? 1 : 0;
 877         return __ftrace_set_clr_event(tr, NULL, system, event, set);
 878 }
 879 EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
 880
 881 /* 128 should be much more than enough */
 882 #define EVENT_BUF_SIZE          127
 883
 884 static ssize_t
 885 ftrace_event_write(struct file *file, const char __user *ubuf,
 886                    size_t cnt, loff_t *ppos)
 887 {
 888         struct trace_parser parser;
 889         struct seq_file *m = file->private_data;
 890         struct trace_array *tr = m->private;
 891         ssize_t read, ret;
 892
 893         if (!cnt)
 894                 return 0;
 895
 896         ret = tracing_update_buffers();
 897         if (ret < 0)
 898                 return ret;
 899
 900         if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
 901                 return -ENOMEM;
 902
 903         read = trace_get_user(&parser, ubuf, cnt, ppos);
 904
 905         if (read >= 0 && trace_parser_loaded((&parser))) {
 906                 int set = 1;
 907
 908                 if (*parser.buffer == '!')
 909                         set = 0;
 910
 911                 ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
 912                 if (ret)
 913                         goto out_put;
 914         }
 915
 916         ret = read;
 917
 918  out_put:
 919         trace_parser_put(&parser);
 920
 921         return ret;
 922 }
 923
 924 static void *
 925 t_next(struct seq_file *m, void *v, loff_t *pos)
 926 {
 927         struct trace_event_file *file = v;
 928         struct trace_event_call *call;
 929         struct trace_array *tr = m->private;
 930
 931         (*pos)++;
 932
 933         list_for_each_entry_continue(file, &tr->events, list) {
 934                 call = file->event_call;
 935                 /*
 936                  * The ftrace subsystem is for showing formats only.
 937                  * They can not be enabled or disabled via the event files.
 938                  */
 939                 if (call->class && call->class->reg &&
 940                     !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
 941                         return file;
 942         }
 943
 944         return NULL;
 945 }
 946
 947 static void *t_start(struct seq_file *m, loff_t *pos)
 948 {
 949         struct trace_event_file *file;
 950         struct trace_array *tr = m->private;
 951         loff_t l;
 952
 953         mutex_lock(&event_mutex);
 954
 955         file = list_entry(&tr->events, struct trace_event_file, list);
 956         for (l = 0; l <= *pos; ) {
 957                 file = t_next(m, file, &l);
 958                 if (!file)
 959                         break;
 960         }
 961         return file;
 962 }
 963
 964 static void *
 965 s_next(struct seq_file *m, void *v, loff_t *pos)
 966 {
 967         struct trace_event_file *file = v;
 968         struct trace_array *tr = m->private;
 969
 970         (*pos)++;
 971
 972         list_for_each_entry_continue(file, &tr->events, list) {
 973                 if (file->flags & EVENT_FILE_FL_ENABLED)
 974                         return file;
 975         }
 976
 977         return NULL;
 978 }
 979
 980 static void *s_start(struct seq_file *m, loff_t *pos)
 981 {
 982         struct trace_event_file *file;
 983         struct trace_array *tr = m->private;
 984         loff_t l;
 985
 986         mutex_lock(&event_mutex);
 987
 988         file = list_entry(&tr->events, struct trace_event_file, list);
 989         for (l = 0; l <= *pos; ) {
 990                 file = s_next(m, file, &l);
 991                 if (!file)
 992                         break;
 993         }
 994         return file;
 995 }
 996
 997 static int t_show(struct seq_file *m, void *v)
 998 {
 999         struct trace_event_file *file = v;
1000         struct trace_event_call *call = file->event_call;
1001
1002         if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1003                 seq_printf(m, "%s:", call->class->system);
1004         seq_printf(m, "%s\n", trace_event_name(call));
1005
1006         return 0;
1007 }
1008
1009 static void t_stop(struct seq_file *m, void *p)
1010 {
1011         mutex_unlock(&event_mutex);
1012 }
1013
1014 static void *
1015 p_next(struct seq_file *m, void *v, loff_t *pos)
1016 {
1017         struct trace_array *tr = m->private;
1018         struct trace_pid_list *pid_list = rcu_dereference_sched(tr->filtered_pids);
1019
1020         return trace_pid_next(pid_list, v, pos);
1021 }
1022
1023 static void *p_start(struct seq_file *m, loff_t *pos)
1024         __acquires(RCU)
1025 {
1026         struct trace_pid_list *pid_list;
1027         struct trace_array *tr = m->private;
1028
1029         /*
1030          * Grab the mutex, to keep calls to p_next() having the same
1031          * tr->filtered_pids as p_start() has.
1032          * If we just passed the tr->filtered_pids around, then RCU would
1033          * have been enough, but doing that makes things more complex.
1034          */
1035         mutex_lock(&event_mutex);
1036         rcu_read_lock_sched();
1037
1038         pid_list = rcu_dereference_sched(tr->filtered_pids);
1039
1040         if (!pid_list)
1041                 return NULL;
1042
1043         return trace_pid_start(pid_list, pos);
1044 }
1045
1046 static void p_stop(struct seq_file *m, void *p)
1047         __releases(RCU)
1048 {
1049         rcu_read_unlock_sched();
1050         mutex_unlock(&event_mutex);
1051 }
1052
1053 static ssize_t
1054 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1055                   loff_t *ppos)
1056 {
1057         struct trace_event_file *file;
1058         unsigned long flags;
1059         char buf[4] = "0";
1060
1061         mutex_lock(&event_mutex);
1062         file = event_file_data(filp);
1063         if (likely(file))
1064                 flags = file->flags;
1065         mutex_unlock(&event_mutex);
1066
1067         if (!file)
1068                 return -ENODEV;
1069
1070         if (flags & EVENT_FILE_FL_ENABLED &&
1071             !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1072                 strcpy(buf, "1");
1073
1074         if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1075             flags & EVENT_FILE_FL_SOFT_MODE)
1076                 strcat(buf, "*");
1077
1078         strcat(buf, "\n");
1079
1080         return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1081 }
1082
1083 static ssize_t
1084 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1085                    loff_t *ppos)
1086 {
1087         struct trace_event_file *file;
1088         unsigned long val;
1089         int ret;
1090
1091         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1092         if (ret)
1093                 return ret;
1094
1095         ret = tracing_update_buffers();
1096         if (ret < 0)
1097                 return ret;
1098
1099         switch (val) {
1100         case 0:
1101         case 1:
1102                 ret = -ENODEV;
1103                 mutex_lock(&event_mutex);
1104                 file = event_file_data(filp);
1105                 if (likely(file))
1106                         ret = ftrace_event_enable_disable(file, val);
1107                 mutex_unlock(&event_mutex);
1108                 break;
1109
1110         default:
1111                 return -EINVAL;
1112         }
1113
1114         *ppos += cnt;
1115
1116         return ret ? ret : cnt;
1117 }
1118
1119 static ssize_t
1120 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1121                    loff_t *ppos)
1122 {
1123         const char set_to_char[4] = { '?', '0', '1', 'X' };
1124         struct trace_subsystem_dir *dir = filp->private_data;
1125         struct event_subsystem *system = dir->subsystem;
1126         struct trace_event_call *call;
1127         struct trace_event_file *file;
1128         struct trace_array *tr = dir->tr;
1129         char buf[2];
1130         int set = 0;
1131         int ret;
1132
1133         mutex_lock(&event_mutex);
1134         list_for_each_entry(file, &tr->events, list) {
1135                 call = file->event_call;
1136                 if (!trace_event_name(call) || !call->class || !call->class->reg)
1137                         continue;
1138
1139                 if (system && strcmp(call->class->system, system->name) != 0)
1140                         continue;
1141
1142                 /*
1143                  * We need to find out if all the events are set
1144                  * or if all events or cleared, or if we have
1145                  * a mixture.
1146                  */
1147                 set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1148
1149                 /*
1150                  * If we have a mixture, no need to look further.
1151                  */
1152                 if (set == 3)
1153                         break;
1154         }
1155         mutex_unlock(&event_mutex);
1156
1157         buf[0] = set_to_char[set];
1158         buf[1] = '\n';
1159
1160         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1161
1162         return ret;
1163 }
1164
1165 static ssize_t
1166 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1167                     loff_t *ppos)
1168 {
1169         struct trace_subsystem_dir *dir = filp->private_data;
1170         struct event_subsystem *system = dir->subsystem;
1171         const char *name = NULL;
1172         unsigned long val;
1173         ssize_t ret;
1174
1175         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1176         if (ret)
1177                 return ret;
1178
1179         ret = tracing_update_buffers();
1180         if (ret < 0)
1181                 return ret;
1182
1183         if (val != 0 && val != 1)
1184                 return -EINVAL;
1185
1186         /*
1187          * Opening of "enable" adds a ref count to system,
1188          * so the name is safe to use.
1189          */
1190         if (system)
1191                 name = system->name;
1192
1193         ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
1194         if (ret)
1195                 goto out;
1196
1197         ret = cnt;
1198
1199 out:
1200         *ppos += cnt;
1201
1202         return ret;
1203 }
1204
1205 enum {
1206         FORMAT_HEADER           = 1,
1207         FORMAT_FIELD_SEPERATOR  = 2,
1208         FORMAT_PRINTFMT         = 3,
1209 };
1210
1211 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1212 {
1213         struct trace_event_call *call = event_file_data(m->private);
1214         struct list_head *common_head = &ftrace_common_fields;
1215         struct list_head *head = trace_get_fields(call);
1216         struct list_head *node = v;
1217
1218         (*pos)++;
1219
1220         switch ((unsigned long)v) {
1221         case FORMAT_HEADER:
1222                 node = common_head;
1223                 break;
1224
1225         case FORMAT_FIELD_SEPERATOR:
1226                 node = head;
1227                 break;
1228
1229         case FORMAT_PRINTFMT:
1230                 /* all done */
1231                 return NULL;
1232         }
1233
1234         node = node->prev;
1235         if (node == common_head)
1236                 return (void *)FORMAT_FIELD_SEPERATOR;
1237         else if (node == head)
1238                 return (void *)FORMAT_PRINTFMT;
1239         else
1240                 return node;
1241 }
1242
1243 static int f_show(struct seq_file *m, void *v)
1244 {
1245         struct trace_event_call *call = event_file_data(m->private);
1246         struct ftrace_event_field *field;
1247         const char *array_descriptor;
1248
1249         switch ((unsigned long)v) {
1250         case FORMAT_HEADER:
1251                 seq_printf(m, "name: %s\n", trace_event_name(call));
1252                 seq_printf(m, "ID: %d\n", call->event.type);
1253                 seq_puts(m, "format:\n");
1254                 return 0;
1255
1256         case FORMAT_FIELD_SEPERATOR:
1257                 seq_putc(m, '\n');
1258                 return 0;
1259
1260         case FORMAT_PRINTFMT:
1261                 seq_printf(m, "\nprint fmt: %s\n",
1262                            call->print_fmt);
1263                 return 0;
1264         }
1265
1266         field = list_entry(v, struct ftrace_event_field, link);
1267         /*
1268          * Smartly shows the array type(except dynamic array).
1269          * Normal:
1270          *      field:TYPE VAR
1271          * If TYPE := TYPE[LEN], it is shown:
1272          *      field:TYPE VAR[LEN]
1273          */
1274         array_descriptor = strchr(field->type, '[');
1275
1276         if (str_has_prefix(field->type, "__data_loc"))
1277                 array_descriptor = NULL;
1278
1279         if (!array_descriptor)
1280                 seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1281                            field->type, field->name, field->offset,
1282                            field->size, !!field->is_signed);
1283         else
1284                 seq_printf(m, "\tfield:%.*s %s%s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1285                            (int)(array_descriptor - field->type),
1286                            field->type, field->name,
1287                            array_descriptor, field->offset,
1288                            field->size, !!field->is_signed);
1289
1290         return 0;
1291 }
1292
1293 static void *f_start(struct seq_file *m, loff_t *pos)
1294 {
1295         void *p = (void *)FORMAT_HEADER;
1296         loff_t l = 0;
1297
1298         /* ->stop() is called even if ->start() fails */
1299         mutex_lock(&event_mutex);
1300         if (!event_file_data(m->private))
1301                 return ERR_PTR(-ENODEV);
1302
1303         while (l < *pos && p)
1304                 p = f_next(m, p, &l);
1305
1306         return p;
1307 }
1308
1309 static void f_stop(struct seq_file *m, void *p)
1310 {
1311         mutex_unlock(&event_mutex);
1312 }
1313
1314 static const struct seq_operations trace_format_seq_ops = {
1315         .start          = f_start,
1316         .next           = f_next,
1317         .stop           = f_stop,
1318         .show           = f_show,
1319 };
1320
1321 static int trace_format_open(struct inode *inode, struct file *file)
1322 {
1323         struct seq_file *m;
1324         int ret;
1325
1326         /* Do we want to hide event format files on tracefs lockdown? */
1327
1328         ret = seq_open(file, &trace_format_seq_ops);
1329         if (ret < 0)
1330                 return ret;
1331
1332         m = file->private_data;
1333         m->private = file;
1334
1335         return 0;
1336 }
1337
1338 static ssize_t
1339 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1340 {
1341         int id = (long)event_file_data(filp);
1342         char buf[32];
1343         int len;
1344
1345         if (unlikely(!id))
1346                 return -ENODEV;
1347
1348         len = sprintf(buf, "%d\n", id);
1349
1350         return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
1351 }
1352
1353 static ssize_t
1354 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1355                   loff_t *ppos)
1356 {
1357         struct trace_event_file *file;
1358         struct trace_seq *s;
1359         int r = -ENODEV;
1360
1361         if (*ppos)
1362                 return 0;
1363
1364         s = kmalloc(sizeof(*s), GFP_KERNEL);
1365
1366         if (!s)
1367                 return -ENOMEM;
1368
1369         trace_seq_init(s);
1370
1371         mutex_lock(&event_mutex);
1372         file = event_file_data(filp);
1373         if (file)
1374                 print_event_filter(file, s);
1375         mutex_unlock(&event_mutex);
1376
1377         if (file)
1378                 r = simple_read_from_buffer(ubuf, cnt, ppos,
1379                                             s->buffer, trace_seq_used(s));
1380
1381         kfree(s);
1382
1383         return r;
1384 }
1385
1386 static ssize_t
1387 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1388                    loff_t *ppos)
1389 {
1390         struct trace_event_file *file;
1391         char *buf;
1392         int err = -ENODEV;
1393
1394         if (cnt >= PAGE_SIZE)
1395                 return -EINVAL;
1396
1397         buf = memdup_user_nul(ubuf, cnt);
1398         if (IS_ERR(buf))
1399                 return PTR_ERR(buf);
1400
1401         mutex_lock(&event_mutex);
1402         file = event_file_data(filp);
1403         if (file)
1404                 err = apply_event_filter(file, buf);
1405         mutex_unlock(&event_mutex);
1406
1407         kfree(buf);
1408         if (err < 0)
1409                 return err;
1410
1411         *ppos += cnt;
1412
1413         return cnt;
1414 }
1415
1416 static LIST_HEAD(event_subsystems);
1417
1418 static int subsystem_open(struct inode *inode, struct file *filp)
1419 {
1420         struct event_subsystem *system = NULL;
1421         struct trace_subsystem_dir *dir = NULL; /* Initialize for gcc */
1422         struct trace_array *tr;
1423         int ret;
1424
1425         if (tracing_is_disabled())
1426                 return -ENODEV;
1427
1428         /* Make sure the system still exists */
1429         mutex_lock(&event_mutex);
1430         mutex_lock(&trace_types_lock);
1431         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1432                 list_for_each_entry(dir, &tr->systems, list) {
1433                         if (dir == inode->i_private) {
1434                                 /* Don't open systems with no events */
1435                                 if (dir->nr_events) {
1436                                         __get_system_dir(dir);
1437                                         system = dir->subsystem;
1438                                 }
1439                                 goto exit_loop;
1440                         }
1441                 }
1442         }
1443  exit_loop:
1444         mutex_unlock(&trace_types_lock);
1445         mutex_unlock(&event_mutex);
1446
1447         if (!system)
1448                 return -ENODEV;
1449
1450         /* Some versions of gcc think dir can be uninitialized here */
1451         WARN_ON(!dir);
1452
1453         /* Still need to increment the ref count of the system */
1454         if (trace_array_get(tr) < 0) {
1455                 put_system(dir);
1456                 return -ENODEV;
1457         }
1458
1459         ret = tracing_open_generic(inode, filp);
1460         if (ret < 0) {
1461                 trace_array_put(tr);
1462                 put_system(dir);
1463         }
1464
1465         return ret;
1466 }
1467
1468 static int system_tr_open(struct inode *inode, struct file *filp)
1469 {
1470         struct trace_subsystem_dir *dir;
1471         struct trace_array *tr = inode->i_private;
1472         int ret;
1473
1474         /* Make a temporary dir that has no system but points to tr */
1475         dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1476         if (!dir)
1477                 return -ENOMEM;
1478
1479         ret = tracing_open_generic_tr(inode, filp);
1480         if (ret < 0) {
1481                 kfree(dir);
1482                 return ret;
1483         }
1484         dir->tr = tr;
1485         filp->private_data = dir;
1486
1487         return 0;
1488 }
1489
1490 static int subsystem_release(struct inode *inode, struct file *file)
1491 {
1492         struct trace_subsystem_dir *dir = file->private_data;
1493
1494         trace_array_put(dir->tr);
1495
1496         /*
1497          * If dir->subsystem is NULL, then this is a temporary
1498          * descriptor that was made for a trace_array to enable
1499          * all subsystems.
1500          */
1501         if (dir->subsystem)
1502                 put_system(dir);
1503         else
1504                 kfree(dir);
1505
1506         return 0;
1507 }
1508
1509 static ssize_t
1510 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1511                       loff_t *ppos)
1512 {
1513         struct trace_subsystem_dir *dir = filp->private_data;
1514         struct event_subsystem *system = dir->subsystem;
1515         struct trace_seq *s;
1516         int r;
1517
1518         if (*ppos)
1519                 return 0;
1520
1521         s = kmalloc(sizeof(*s), GFP_KERNEL);
1522         if (!s)
1523                 return -ENOMEM;
1524
1525         trace_seq_init(s);
1526
1527         print_subsystem_event_filter(system, s);
1528         r = simple_read_from_buffer(ubuf, cnt, ppos,
1529                                     s->buffer, trace_seq_used(s));
1530
1531         kfree(s);
1532
1533         return r;
1534 }
1535
1536 static ssize_t
1537 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1538                        loff_t *ppos)
1539 {
1540         struct trace_subsystem_dir *dir = filp->private_data;
1541         char *buf;
1542         int err;
1543
1544         if (cnt >= PAGE_SIZE)
1545                 return -EINVAL;
1546
1547         buf = memdup_user_nul(ubuf, cnt);
1548         if (IS_ERR(buf))
1549                 return PTR_ERR(buf);
1550
1551         err = apply_subsystem_event_filter(dir, buf);
1552         kfree(buf);
1553         if (err < 0)
1554                 return err;
1555
1556         *ppos += cnt;
1557
1558         return cnt;
1559 }
1560
1561 static ssize_t
1562 show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1563 {
1564         int (*func)(struct trace_seq *s) = filp->private_data;
1565         struct trace_seq *s;
1566         int r;
1567
1568         if (*ppos)
1569                 return 0;
1570
1571         s = kmalloc(sizeof(*s), GFP_KERNEL);
1572         if (!s)
1573                 return -ENOMEM;
1574
1575         trace_seq_init(s);
1576
1577         func(s);
1578         r = simple_read_from_buffer(ubuf, cnt, ppos,
1579                                     s->buffer, trace_seq_used(s));
1580
1581         kfree(s);
1582
1583         return r;
1584 }
1585
1586 static void ignore_task_cpu(void *data)
1587 {
1588         struct trace_array *tr = data;
1589         struct trace_pid_list *pid_list;
1590
1591         /*
1592          * This function is called by on_each_cpu() while the
1593          * event_mutex is held.
1594          */
1595         pid_list = rcu_dereference_protected(tr->filtered_pids,
1596                                              mutex_is_locked(&event_mutex));
1597
1598         this_cpu_write(tr->trace_buffer.data->ignore_pid,
1599                        trace_ignore_this_task(pid_list, current));
1600 }
1601
1602 static ssize_t
1603 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
1604                        size_t cnt, loff_t *ppos)
1605 {
1606         struct seq_file *m = filp->private_data;
1607         struct trace_array *tr = m->private;
1608         struct trace_pid_list *filtered_pids = NULL;
1609         struct trace_pid_list *pid_list;
1610         struct trace_event_file *file;
1611         ssize_t ret;
1612
1613         if (!cnt)
1614                 return 0;
1615
1616         ret = tracing_update_buffers();
1617         if (ret < 0)
1618                 return ret;
1619
1620         mutex_lock(&event_mutex);
1621
1622         filtered_pids = rcu_dereference_protected(tr->filtered_pids,
1623                                              lockdep_is_held(&event_mutex));
1624
1625         ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
1626         if (ret < 0)
1627                 goto out;
1628
1629         rcu_assign_pointer(tr->filtered_pids, pid_list);
1630
1631         list_for_each_entry(file, &tr->events, list) {
1632                 set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1633         }
1634
1635         if (filtered_pids) {
1636                 tracepoint_synchronize_unregister();
1637                 trace_free_pid_list(filtered_pids);
1638         } else if (pid_list) {
1639                 /*
1640                  * Register a probe that is called before all other probes
1641                  * to set ignore_pid if next or prev do not match.
1642                  * Register a probe this is called after all other probes
1643                  * to only keep ignore_pid set if next pid matches.
1644                  */
1645                 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
1646                                                  tr, INT_MAX);
1647                 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
1648                                                  tr, 0);
1649
1650                 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
1651                                                  tr, INT_MAX);
1652                 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
1653                                                  tr, 0);
1654
1655                 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
1656                                                      tr, INT_MAX);
1657                 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
1658                                                      tr, 0);
1659
1660                 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
1661                                                  tr, INT_MAX);
1662                 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
1663                                                  tr, 0);
1664         }
1665
1666         /*
1667          * Ignoring of pids is done at task switch. But we have to
1668          * check for those tasks that are currently running.
1669          * Always do this in case a pid was appended or removed.
1670          */
1671         on_each_cpu(ignore_task_cpu, tr, 1);
1672
1673  out:
1674         mutex_unlock(&event_mutex);
1675
1676         if (ret > 0)
1677                 *ppos += ret;
1678
1679         return ret;
1680 }
1681
1682 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
1683 static int ftrace_event_set_open(struct inode *inode, struct file *file);
1684 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
1685 static int ftrace_event_release(struct inode *inode, struct file *file);
1686
1687 static const struct seq_operations show_event_seq_ops = {
1688         .start = t_start,
1689         .next = t_next,
1690         .show = t_show,
1691         .stop = t_stop,
1692 };
1693
1694 static const struct seq_operations show_set_event_seq_ops = {
1695         .start = s_start,
1696         .next = s_next,
1697         .show = t_show,
1698         .stop = t_stop,
1699 };
1700
1701 static const struct seq_operations show_set_pid_seq_ops = {
1702         .start = p_start,
1703         .next = p_next,
1704         .show = trace_pid_show,
1705         .stop = p_stop,
1706 };
1707
1708 static const struct file_operations ftrace_avail_fops = {
1709         .open = ftrace_event_avail_open,
1710         .read = seq_read,
1711         .llseek = seq_lseek,
1712         .release = seq_release,
1713 };
1714
1715 static const struct file_operations ftrace_set_event_fops = {
1716         .open = ftrace_event_set_open,
1717         .read = seq_read,
1718         .write = ftrace_event_write,
1719         .llseek = seq_lseek,
1720         .release = ftrace_event_release,
1721 };
1722
1723 static const struct file_operations ftrace_set_event_pid_fops = {
1724         .open = ftrace_event_set_pid_open,
1725         .read = seq_read,
1726         .write = ftrace_event_pid_write,
1727         .llseek = seq_lseek,
1728         .release = ftrace_event_release,
1729 };
1730
1731 static const struct file_operations ftrace_enable_fops = {
1732         .open = tracing_open_generic,
1733         .read = event_enable_read,
1734         .write = event_enable_write,
1735         .llseek = default_llseek,
1736 };
1737
1738 static const struct file_operations ftrace_event_format_fops = {
1739         .open = trace_format_open,
1740         .read = seq_read,
1741         .llseek = seq_lseek,
1742         .release = seq_release,
1743 };
1744
1745 static const struct file_operations ftrace_event_id_fops = {
1746         .read = event_id_read,
1747         .llseek = default_llseek,
1748 };
1749
1750 static const struct file_operations ftrace_event_filter_fops = {
1751         .open = tracing_open_generic,
1752         .read = event_filter_read,
1753         .write = event_filter_write,
1754         .llseek = default_llseek,
1755 };
1756
1757 static const struct file_operations ftrace_subsystem_filter_fops = {
1758         .open = subsystem_open,
1759         .read = subsystem_filter_read,
1760         .write = subsystem_filter_write,
1761         .llseek = default_llseek,
1762         .release = subsystem_release,
1763 };
1764
1765 static const struct file_operations ftrace_system_enable_fops = {
1766         .open = subsystem_open,
1767         .read = system_enable_read,
1768         .write = system_enable_write,
1769         .llseek = default_llseek,
1770         .release = subsystem_release,
1771 };
1772
1773 static const struct file_operations ftrace_tr_enable_fops = {
1774         .open = system_tr_open,
1775         .read = system_enable_read,
1776         .write = system_enable_write,
1777         .llseek = default_llseek,
1778         .release = subsystem_release,
1779 };
1780
1781 static const struct file_operations ftrace_show_header_fops = {
1782         .open = tracing_open_generic,
1783         .read = show_header,
1784         .llseek = default_llseek,
1785 };
1786
1787 static int
1788 ftrace_event_open(struct inode *inode, struct file *file,
1789                   const struct seq_operations *seq_ops)
1790 {
1791         struct seq_file *m;
1792         int ret;
1793
1794         ret = security_locked_down(LOCKDOWN_TRACEFS);
1795         if (ret)
1796                 return ret;
1797
1798         ret = seq_open(file, seq_ops);
1799         if (ret < 0)
1800                 return ret;
1801         m = file->private_data;
1802         /* copy tr over to seq ops */
1803         m->private = inode->i_private;
1804
1805         return ret;
1806 }
1807
1808 static int ftrace_event_release(struct inode *inode, struct file *file)
1809 {
1810         struct trace_array *tr = inode->i_private;
1811
1812         trace_array_put(tr);
1813
1814         return seq_release(inode, file);
1815 }
1816
1817 static int
1818 ftrace_event_avail_open(struct inode *inode, struct file *file)
1819 {
1820         const struct seq_operations *seq_ops = &show_event_seq_ops;
1821
1822         /* Checks for tracefs lockdown */
1823         return ftrace_event_open(inode, file, seq_ops);
1824 }
1825
1826 static int
1827 ftrace_event_set_open(struct inode *inode, struct file *file)
1828 {
1829         const struct seq_operations *seq_ops = &show_set_event_seq_ops;
1830         struct trace_array *tr = inode->i_private;
1831         int ret;
1832
1833         ret = tracing_check_open_get_tr(tr);
1834         if (ret)
1835                 return ret;
1836
1837         if ((file->f_mode & FMODE_WRITE) &&
1838             (file->f_flags & O_TRUNC))
1839                 ftrace_clear_events(tr);
1840
1841         ret = ftrace_event_open(inode, file, seq_ops);
1842         if (ret < 0)
1843                 trace_array_put(tr);
1844         return ret;
1845 }
1846
1847 static int
1848 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
1849 {
1850         const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
1851         struct trace_array *tr = inode->i_private;
1852         int ret;
1853
1854         ret = tracing_check_open_get_tr(tr);
1855         if (ret)
1856                 return ret;
1857
1858         if ((file->f_mode & FMODE_WRITE) &&
1859             (file->f_flags & O_TRUNC))
1860                 ftrace_clear_event_pids(tr);
1861
1862         ret = ftrace_event_open(inode, file, seq_ops);
1863         if (ret < 0)
1864                 trace_array_put(tr);
1865         return ret;
1866 }
1867
1868 static struct event_subsystem *
1869 create_new_subsystem(const char *name)
1870 {
1871         struct event_subsystem *system;
1872
1873         /* need to create new entry */
1874         system = kmalloc(sizeof(*system), GFP_KERNEL);
1875         if (!system)
1876                 return NULL;
1877
1878         system->ref_count = 1;
1879
1880         /* Only allocate if dynamic (kprobes and modules) */
1881         system->name = kstrdup_const(name, GFP_KERNEL);
1882         if (!system->name)
1883                 goto out_free;
1884
1885         system->filter = NULL;
1886
1887         system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
1888         if (!system->filter)
1889                 goto out_free;
1890
1891         list_add(&system->list, &event_subsystems);
1892
1893         return system;
1894
1895  out_free:
1896         kfree_const(system->name);
1897         kfree(system);
1898         return NULL;
1899 }
1900
1901 static struct dentry *
1902 event_subsystem_dir(struct trace_array *tr, const char *name,
1903                     struct trace_event_file *file, struct dentry *parent)
1904 {
1905         struct trace_subsystem_dir *dir;
1906         struct event_subsystem *system;
1907         struct dentry *entry;
1908
1909         /* First see if we did not already create this dir */
1910         list_for_each_entry(dir, &tr->systems, list) {
1911                 system = dir->subsystem;
1912                 if (strcmp(system->name, name) == 0) {
1913                         dir->nr_events++;
1914                         file->system = dir;
1915                         return dir->entry;
1916                 }
1917         }
1918
1919         /* Now see if the system itself exists. */
1920         list_for_each_entry(system, &event_subsystems, list) {
1921                 if (strcmp(system->name, name) == 0)
1922                         break;
1923         }
1924         /* Reset system variable when not found */
1925         if (&system->list == &event_subsystems)
1926                 system = NULL;
1927
1928         dir = kmalloc(sizeof(*dir), GFP_KERNEL);
1929         if (!dir)
1930                 goto out_fail;
1931
1932         if (!system) {
1933                 system = create_new_subsystem(name);
1934                 if (!system)
1935                         goto out_free;
1936         } else
1937                 __get_system(system);
1938
1939         dir->entry = tracefs_create_dir(name, parent);
1940         if (!dir->entry) {
1941                 pr_warn("Failed to create system directory %s\n", name);
1942                 __put_system(system);
1943                 goto out_free;
1944         }
1945
1946         dir->tr = tr;
1947         dir->ref_count = 1;
1948         dir->nr_events = 1;
1949         dir->subsystem = system;
1950         file->system = dir;
1951
1952         entry = tracefs_create_file("filter", 0644, dir->entry, dir,
1953                                     &ftrace_subsystem_filter_fops);
1954         if (!entry) {
1955                 kfree(system->filter);
1956                 system->filter = NULL;
1957                 pr_warn("Could not create tracefs '%s/filter' entry\n", name);
1958         }
1959
1960         trace_create_file("enable", 0644, dir->entry, dir,
1961                           &ftrace_system_enable_fops);
1962
1963         list_add(&dir->list, &tr->systems);
1964
1965         return dir->entry;
1966
1967  out_free:
1968         kfree(dir);
1969  out_fail:
1970         /* Only print this message if failed on memory allocation */
1971         if (!dir || !system)
1972                 pr_warn("No memory to create event subsystem %s\n", name);
1973         return NULL;
1974 }
1975
1976 static int
1977 event_create_dir(struct dentry *parent, struct trace_event_file *file)
1978 {
1979         struct trace_event_call *call = file->event_call;
1980         struct trace_array *tr = file->tr;
1981         struct list_head *head;
1982         struct dentry *d_events;
1983         const char *name;
1984         int ret;
1985
1986         /*
1987          * If the trace point header did not define TRACE_SYSTEM
1988          * then the system would be called "TRACE_SYSTEM".
1989          */
1990         if (strcmp(call->class->system, TRACE_SYSTEM) != 0) {
1991                 d_events = event_subsystem_dir(tr, call->class->system, file, parent);
1992                 if (!d_events)
1993                         return -ENOMEM;
1994         } else
1995                 d_events = parent;
1996
1997         name = trace_event_name(call);
1998         file->dir = tracefs_create_dir(name, d_events);
1999         if (!file->dir) {
2000                 pr_warn("Could not create tracefs '%s' directory\n", name);
2001                 return -1;
2002         }
2003
2004         if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
2005                 trace_create_file("enable", 0644, file->dir, file,
2006                                   &ftrace_enable_fops);
2007
2008 #ifdef CONFIG_PERF_EVENTS
2009         if (call->event.type && call->class->reg)
2010                 trace_create_file("id", 0444, file->dir,
2011                                   (void *)(long)call->event.type,
2012                                   &ftrace_event_id_fops);
2013 #endif
2014
2015         /*
2016          * Other events may have the same class. Only update
2017          * the fields if they are not already defined.
2018          */
2019         head = trace_get_fields(call);
2020         if (list_empty(head)) {
2021                 struct trace_event_fields *field = call->class->fields_array;
2022                 unsigned int offset = sizeof(struct trace_entry);
2023
2024                 for (; field->type; field++) {
2025                         if (field->type == TRACE_FUNCTION_TYPE) {
2026                                 ret = field->define_fields(call);
2027                                 break;
2028                         }
2029
2030                         offset = ALIGN(offset, field->align);
2031                         ret = trace_define_field(call, field->type, field->name,
2032                                                  offset, field->size,
2033                                                  field->is_signed, field->filter_type);
2034                         if (ret)
2035                                 break;
2036
2037                         offset += field->size;
2038                 }
2039                 if (ret < 0) {
2040                         pr_warn("Could not initialize trace point events/%s\n",
2041                                 name);
2042                         return -1;
2043                 }
2044         }
2045
2046         /*
2047          * Only event directories that can be enabled should have
2048          * triggers or filters.
2049          */
2050         if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2051                 trace_create_file("filter", 0644, file->dir, file,
2052                                   &ftrace_event_filter_fops);
2053
2054                 trace_create_file("trigger", 0644, file->dir, file,
2055                                   &event_trigger_fops);
2056         }
2057
2058 #ifdef CONFIG_HIST_TRIGGERS
2059         trace_create_file("hist", 0444, file->dir, file,
2060                           &event_hist_fops);
2061 #endif
2062         trace_create_file("format", 0444, file->dir, call,
2063                           &ftrace_event_format_fops);
2064
2065 #ifdef CONFIG_TRACE_EVENT_INJECT
2066         if (call->event.type && call->class->reg)
2067                 trace_create_file("inject", 0200, file->dir, file,
2068                                   &event_inject_fops);
2069 #endif
2070
2071         return 0;
2072 }
2073
2074 static void remove_event_from_tracers(struct trace_event_call *call)
2075 {
2076         struct trace_event_file *file;
2077         struct trace_array *tr;
2078
2079         do_for_each_event_file_safe(tr, file) {
2080                 if (file->event_call != call)
2081                         continue;
2082
2083                 remove_event_file_dir(file);
2084                 /*
2085                  * The do_for_each_event_file_safe() is
2086                  * a double loop. After finding the call for this
2087                  * trace_array, we use break to jump to the next
2088                  * trace_array.
2089                  */
2090                 break;
2091         } while_for_each_event_file();
2092 }
2093
2094 static void event_remove(struct trace_event_call *call)
2095 {
2096         struct trace_array *tr;
2097         struct trace_event_file *file;
2098
2099         do_for_each_event_file(tr, file) {
2100                 if (file->event_call != call)
2101                         continue;
2102
2103                 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
2104                         tr->clear_trace = true;
2105
2106                 ftrace_event_enable_disable(file, 0);
2107                 /*
2108                  * The do_for_each_event_file() is
2109                  * a double loop. After finding the call for this
2110                  * trace_array, we use break to jump to the next
2111                  * trace_array.
2112                  */
2113                 break;
2114         } while_for_each_event_file();
2115
2116         if (call->event.funcs)
2117                 __unregister_trace_event(&call->event);
2118         remove_event_from_tracers(call);
2119         list_del(&call->list);
2120 }
2121
2122 static int event_init(struct trace_event_call *call)
2123 {
2124         int ret = 0;
2125         const char *name;
2126
2127         name = trace_event_name(call);
2128         if (WARN_ON(!name))
2129                 return -EINVAL;
2130
2131         if (call->class->raw_init) {
2132                 ret = call->class->raw_init(call);
2133                 if (ret < 0 && ret != -ENOSYS)
2134                         pr_warn("Could not initialize trace events/%s\n", name);
2135         }
2136
2137         return ret;
2138 }
2139
2140 static int
2141 __register_event(struct trace_event_call *call, struct module *mod)
2142 {
2143         int ret;
2144
2145         ret = event_init(call);
2146         if (ret < 0)
2147                 return ret;
2148
2149         list_add(&call->list, &ftrace_events);
2150         call->mod = mod;
2151
2152         return 0;
2153 }
2154
2155 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
2156 {
2157         int rlen;
2158         int elen;
2159
2160         /* Find the length of the eval value as a string */
2161         elen = snprintf(ptr, 0, "%ld", map->eval_value);
2162         /* Make sure there's enough room to replace the string with the value */
2163         if (len < elen)
2164                 return NULL;
2165
2166         snprintf(ptr, elen + 1, "%ld", map->eval_value);
2167
2168         /* Get the rest of the string of ptr */
2169         rlen = strlen(ptr + len);
2170         memmove(ptr + elen, ptr + len, rlen);
2171         /* Make sure we end the new string */
2172         ptr[elen + rlen] = 0;
2173
2174         return ptr + elen;
2175 }
2176
2177 static void update_event_printk(struct trace_event_call *call,
2178                                 struct trace_eval_map *map)
2179 {
2180         char *ptr;
2181         int quote = 0;
2182         int len = strlen(map->eval_string);
2183
2184         for (ptr = call->print_fmt; *ptr; ptr++) {
2185                 if (*ptr == '\\') {
2186                         ptr++;
2187                         /* paranoid */
2188                         if (!*ptr)
2189                                 break;
2190                         continue;
2191                 }
2192                 if (*ptr == '"') {
2193                         quote ^= 1;
2194                         continue;
2195                 }
2196                 if (quote)
2197                         continue;
2198                 if (isdigit(*ptr)) {
2199                         /* skip numbers */
2200                         do {
2201                                 ptr++;
2202                                 /* Check for alpha chars like ULL */
2203                         } while (isalnum(*ptr));
2204                         if (!*ptr)
2205                                 break;
2206                         /*
2207                          * A number must have some kind of delimiter after
2208                          * it, and we can ignore that too.
2209                          */
2210                         continue;
2211                 }
2212                 if (isalpha(*ptr) || *ptr == '_') {
2213                         if (strncmp(map->eval_string, ptr, len) == 0 &&
2214                             !isalnum(ptr[len]) && ptr[len] != '_') {
2215                                 ptr = eval_replace(ptr, map, len);
2216                                 /* enum/sizeof string smaller than value */
2217                                 if (WARN_ON_ONCE(!ptr))
2218                                         return;
2219                                 /*
2220                                  * No need to decrement here, as eval_replace()
2221                                  * returns the pointer to the character passed
2222                                  * the eval, and two evals can not be placed
2223                                  * back to back without something in between.
2224                                  * We can skip that something in between.
2225                                  */
2226                                 continue;
2227                         }
2228                 skip_more:
2229                         do {
2230                                 ptr++;
2231                         } while (isalnum(*ptr) || *ptr == '_');
2232                         if (!*ptr)
2233                                 break;
2234                         /*
2235                          * If what comes after this variable is a '.' or
2236                          * '->' then we can continue to ignore that string.
2237                          */
2238                         if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
2239                                 ptr += *ptr == '.' ? 1 : 2;
2240                                 if (!*ptr)
2241                                         break;
2242                                 goto skip_more;
2243                         }
2244                         /*
2245                          * Once again, we can skip the delimiter that came
2246                          * after the string.
2247                          */
2248                         continue;
2249                 }
2250         }
2251 }
2252
2253 void trace_event_eval_update(struct trace_eval_map **map, int len)
2254 {
2255         struct trace_event_call *call, *p;
2256         const char *last_system = NULL;
2257         bool first = false;
2258         int last_i;
2259         int i;
2260
2261         down_write(&trace_event_sem);
2262         list_for_each_entry_safe(call, p, &ftrace_events, list) {
2263                 /* events are usually grouped together with systems */
2264                 if (!last_system || call->class->system != last_system) {
2265                         first = true;
2266                         last_i = 0;
2267                         last_system = call->class->system;
2268                 }
2269
2270                 /*
2271                  * Since calls are grouped by systems, the likelyhood that the
2272                  * next call in the iteration belongs to the same system as the
2273                  * previous call is high. As an optimization, we skip seaching
2274                  * for a map[] that matches the call's system if the last call
2275                  * was from the same system. That's what last_i is for. If the
2276                  * call has the same system as the previous call, then last_i
2277                  * will be the index of the first map[] that has a matching
2278                  * system.
2279                  */
2280                 for (i = last_i; i < len; i++) {
2281                         if (call->class->system == map[i]->system) {
2282                                 /* Save the first system if need be */
2283                                 if (first) {
2284                                         last_i = i;
2285                                         first = false;
2286                                 }
2287                                 update_event_printk(call, map[i]);
2288                         }
2289                 }
2290         }
2291         up_write(&trace_event_sem);
2292 }
2293
2294 static struct trace_event_file *
2295 trace_create_new_event(struct trace_event_call *call,
2296                        struct trace_array *tr)
2297 {
2298         struct trace_event_file *file;
2299
2300         file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2301         if (!file)
2302                 return NULL;
2303
2304         file->event_call = call;
2305         file->tr = tr;
2306         atomic_set(&file->sm_ref, 0);
2307         atomic_set(&file->tm_ref, 0);
2308         INIT_LIST_HEAD(&file->triggers);
2309         list_add(&file->list, &tr->events);
2310
2311         return file;
2312 }
2313
2314 /* Add an event to a trace directory */
2315 static int
2316 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
2317 {
2318         struct trace_event_file *file;
2319
2320         file = trace_create_new_event(call, tr);
2321         if (!file)
2322                 return -ENOMEM;
2323
2324         return event_create_dir(tr->event_dir, file);
2325 }
2326
2327 /*
2328  * Just create a decriptor for early init. A descriptor is required
2329  * for enabling events at boot. We want to enable events before
2330  * the filesystem is initialized.
2331  */
2332 static __init int
2333 __trace_early_add_new_event(struct trace_event_call *call,
2334                             struct trace_array *tr)
2335 {
2336         struct trace_event_file *file;
2337
2338         file = trace_create_new_event(call, tr);
2339         if (!file)
2340                 return -ENOMEM;
2341
2342         return 0;
2343 }
2344
2345 struct ftrace_module_file_ops;
2346 static void __add_event_to_tracers(struct trace_event_call *call);
2347
2348 /* Add an additional event_call dynamically */
2349 int trace_add_event_call(struct trace_event_call *call)
2350 {
2351         int ret;
2352         lockdep_assert_held(&event_mutex);
2353
2354         mutex_lock(&trace_types_lock);
2355
2356         ret = __register_event(call, NULL);
2357         if (ret >= 0)
2358                 __add_event_to_tracers(call);
2359
2360         mutex_unlock(&trace_types_lock);
2361         return ret;
2362 }
2363
2364 /*
2365  * Must be called under locking of trace_types_lock, event_mutex and
2366  * trace_event_sem.
2367  */
2368 static void __trace_remove_event_call(struct trace_event_call *call)
2369 {
2370         event_remove(call);
2371         trace_destroy_fields(call);
2372         free_event_filter(call->filter);
2373         call->filter = NULL;
2374 }
2375
2376 static int probe_remove_event_call(struct trace_event_call *call)
2377 {
2378         struct trace_array *tr;
2379         struct trace_event_file *file;
2380
2381 #ifdef CONFIG_PERF_EVENTS
2382         if (call->perf_refcount)
2383                 return -EBUSY;
2384 #endif
2385         do_for_each_event_file(tr, file) {
2386                 if (file->event_call != call)
2387                         continue;
2388                 /*
2389                  * We can't rely on ftrace_event_enable_disable(enable => 0)
2390                  * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
2391                  * TRACE_REG_UNREGISTER.
2392                  */
2393                 if (file->flags & EVENT_FILE_FL_ENABLED)
2394                         return -EBUSY;
2395                 /*
2396                  * The do_for_each_event_file_safe() is
2397                  * a double loop. After finding the call for this
2398                  * trace_array, we use break to jump to the next
2399                  * trace_array.
2400                  */
2401                 break;
2402         } while_for_each_event_file();
2403
2404         __trace_remove_event_call(call);
2405
2406         return 0;
2407 }
2408
2409 /* Remove an event_call */
2410 int trace_remove_event_call(struct trace_event_call *call)
2411 {
2412         int ret;
2413
2414         lockdep_assert_held(&event_mutex);
2415
2416         mutex_lock(&trace_types_lock);
2417         down_write(&trace_event_sem);
2418         ret = probe_remove_event_call(call);
2419         up_write(&trace_event_sem);
2420         mutex_unlock(&trace_types_lock);
2421
2422         return ret;
2423 }
2424
2425 #define for_each_event(event, start, end)                       \
2426         for (event = start;                                     \
2427              (unsigned long)event < (unsigned long)end;         \
2428              event++)
2429
2430 #ifdef CONFIG_MODULES
2431
2432 static void trace_module_add_events(struct module *mod)
2433 {
2434         struct trace_event_call **call, **start, **end;
2435
2436         if (!mod->num_trace_events)
2437                 return;
2438
2439         /* Don't add infrastructure for mods without tracepoints */
2440         if (trace_module_has_bad_taint(mod)) {
2441                 pr_err("%s: module has bad taint, not creating trace events\n",
2442                        mod->name);
2443                 return;
2444         }
2445
2446         start = mod->trace_events;
2447         end = mod->trace_events + mod->num_trace_events;
2448
2449         for_each_event(call, start, end) {
2450                 __register_event(*call, mod);
2451                 __add_event_to_tracers(*call);
2452         }
2453 }
2454
2455 static void trace_module_remove_events(struct module *mod)
2456 {
2457         struct trace_event_call *call, *p;
2458
2459         down_write(&trace_event_sem);
2460         list_for_each_entry_safe(call, p, &ftrace_events, list) {
2461                 if (call->mod == mod)
2462                         __trace_remove_event_call(call);
2463         }
2464         up_write(&trace_event_sem);
2465
2466         /*
2467          * It is safest to reset the ring buffer if the module being unloaded
2468          * registered any events that were used. The only worry is if
2469          * a new module gets loaded, and takes on the same id as the events
2470          * of this module. When printing out the buffer, traced events left
2471          * over from this module may be passed to the new module events and
2472          * unexpected results may occur.
2473          */
2474         tracing_reset_all_online_cpus();
2475 }
2476
2477 static int trace_module_notify(struct notifier_block *self,
2478                                unsigned long val, void *data)
2479 {
2480         struct module *mod = data;
2481
2482         mutex_lock(&event_mutex);
2483         mutex_lock(&trace_types_lock);
2484         switch (val) {
2485         case MODULE_STATE_COMING:
2486                 trace_module_add_events(mod);
2487                 break;
2488         case MODULE_STATE_GOING:
2489                 trace_module_remove_events(mod);
2490                 break;
2491         }
2492         mutex_unlock(&trace_types_lock);
2493         mutex_unlock(&event_mutex);
2494
2495         return 0;
2496 }
2497
2498 static struct notifier_block trace_module_nb = {
2499         .notifier_call = trace_module_notify,
2500         .priority = 1, /* higher than trace.c module notify */
2501 };
2502 #endif /* CONFIG_MODULES */
2503
2504 /* Create a new event directory structure for a trace directory. */
2505 static void
2506 __trace_add_event_dirs(struct trace_array *tr)
2507 {
2508         struct trace_event_call *call;
2509         int ret;
2510
2511         list_for_each_entry(call, &ftrace_events, list) {
2512                 ret = __trace_add_new_event(call, tr);
2513                 if (ret < 0)
2514                         pr_warn("Could not create directory for event %s\n",
2515                                 trace_event_name(call));
2516         }
2517 }
2518
2519 /* Returns any file that matches the system and event */
2520 struct trace_event_file *
2521 __find_event_file(struct trace_array *tr, const char *system, const char *event)
2522 {
2523         struct trace_event_file *file;
2524         struct trace_event_call *call;
2525         const char *name;
2526
2527         list_for_each_entry(file, &tr->events, list) {
2528
2529                 call = file->event_call;
2530                 name = trace_event_name(call);
2531
2532                 if (!name || !call->class)
2533                         continue;
2534
2535                 if (strcmp(event, name) == 0 &&
2536                     strcmp(system, call->class->system) == 0)
2537                         return file;
2538         }
2539         return NULL;
2540 }
2541
2542 /* Returns valid trace event files that match system and event */
2543 struct trace_event_file *
2544 find_event_file(struct trace_array *tr, const char *system, const char *event)
2545 {
2546         struct trace_event_file *file;
2547
2548         file = __find_event_file(tr, system, event);
2549         if (!file || !file->event_call->class->reg ||
2550             file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
2551                 return NULL;
2552
2553         return file;
2554 }
2555
2556 #ifdef CONFIG_DYNAMIC_FTRACE
2557
2558 /* Avoid typos */
2559 #define ENABLE_EVENT_STR        "enable_event"
2560 #define DISABLE_EVENT_STR       "disable_event"
2561
2562 struct event_probe_data {
2563         struct trace_event_file *file;
2564         unsigned long                   count;
2565         int                             ref;
2566         bool                            enable;
2567 };
2568
2569 static void update_event_probe(struct event_probe_data *data)
2570 {
2571         if (data->enable)
2572                 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
2573         else
2574                 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
2575 }
2576
2577 static void
2578 event_enable_probe(unsigned long ip, unsigned long parent_ip,
2579                    struct trace_array *tr, struct ftrace_probe_ops *ops,
2580                    void *data)
2581 {
2582         struct ftrace_func_mapper *mapper = data;
2583         struct event_probe_data *edata;
2584         void **pdata;
2585
2586         pdata = ftrace_func_mapper_find_ip(mapper, ip);
2587         if (!pdata || !*pdata)
2588                 return;
2589
2590         edata = *pdata;
2591         update_event_probe(edata);
2592 }
2593
2594 static void
2595 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
2596                          struct trace_array *tr, struct ftrace_probe_ops *ops,
2597                          void *data)
2598 {
2599         struct ftrace_func_mapper *mapper = data;
2600         struct event_probe_data *edata;
2601         void **pdata;
2602
2603         pdata = ftrace_func_mapper_find_ip(mapper, ip);
2604         if (!pdata || !*pdata)
2605                 return;
2606
2607         edata = *pdata;
2608
2609         if (!edata->count)
2610                 return;
2611
2612         /* Skip if the event is in a state we want to switch to */
2613         if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
2614                 return;
2615
2616         if (edata->count != -1)
2617                 (edata->count)--;
2618
2619         update_event_probe(edata);
2620 }
2621
2622 static int
2623 event_enable_print(struct seq_file *m, unsigned long ip,
2624                    struct ftrace_probe_ops *ops, void *data)
2625 {
2626         struct ftrace_func_mapper *mapper = data;
2627         struct event_probe_data *edata;
2628         void **pdata;
2629
2630         pdata = ftrace_func_mapper_find_ip(mapper, ip);
2631
2632         if (WARN_ON_ONCE(!pdata || !*pdata))
2633                 return 0;
2634
2635         edata = *pdata;
2636
2637         seq_printf(m, "%ps:", (void *)ip);
2638
2639         seq_printf(m, "%s:%s:%s",
2640                    edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
2641                    edata->file->event_call->class->system,
2642                    trace_event_name(edata->file->event_call));
2643
2644         if (edata->count == -1)
2645                 seq_puts(m, ":unlimited\n");
2646         else
2647                 seq_printf(m, ":count=%ld\n", edata->count);
2648
2649         return 0;
2650 }
2651
2652 static int
2653 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
2654                   unsigned long ip, void *init_data, void **data)
2655 {
2656         struct ftrace_func_mapper *mapper = *data;
2657         struct event_probe_data *edata = init_data;
2658         int ret;
2659
2660         if (!mapper) {
2661                 mapper = allocate_ftrace_func_mapper();
2662                 if (!mapper)
2663                         return -ENODEV;
2664                 *data = mapper;
2665         }
2666
2667         ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
2668         if (ret < 0)
2669                 return ret;
2670
2671         edata->ref++;
2672
2673         return 0;
2674 }
2675
2676 static int free_probe_data(void *data)
2677 {
2678         struct event_probe_data *edata = data;
2679
2680         edata->ref--;
2681         if (!edata->ref) {
2682                 /* Remove the SOFT_MODE flag */
2683                 __ftrace_event_enable_disable(edata->file, 0, 1);
2684                 module_put(edata->file->event_call->mod);
2685                 kfree(edata);
2686         }
2687         return 0;
2688 }
2689
2690 static void
2691 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
2692                   unsigned long ip, void *data)
2693 {
2694         struct ftrace_func_mapper *mapper = data;
2695         struct event_probe_data *edata;
2696
2697         if (!ip) {
2698                 if (!mapper)
2699                         return;
2700                 free_ftrace_func_mapper(mapper, free_probe_data);
2701                 return;
2702         }
2703
2704         edata = ftrace_func_mapper_remove_ip(mapper, ip);
2705
2706         if (WARN_ON_ONCE(!edata))
2707                 return;
2708
2709         if (WARN_ON_ONCE(edata->ref <= 0))
2710                 return;
2711
2712         free_probe_data(edata);
2713 }
2714
2715 static struct ftrace_probe_ops event_enable_probe_ops = {
2716         .func                   = event_enable_probe,
2717         .print                  = event_enable_print,
2718         .init                   = event_enable_init,
2719         .free                   = event_enable_free,
2720 };
2721
2722 static struct ftrace_probe_ops event_enable_count_probe_ops = {
2723         .func                   = event_enable_count_probe,
2724         .print                  = event_enable_print,
2725         .init                   = event_enable_init,
2726         .free                   = event_enable_free,
2727 };
2728
2729 static struct ftrace_probe_ops event_disable_probe_ops = {
2730         .func                   = event_enable_probe,
2731         .print                  = event_enable_print,
2732         .init                   = event_enable_init,
2733         .free                   = event_enable_free,
2734 };
2735
2736 static struct ftrace_probe_ops event_disable_count_probe_ops = {
2737         .func                   = event_enable_count_probe,
2738         .print                  = event_enable_print,
2739         .init                   = event_enable_init,
2740         .free                   = event_enable_free,
2741 };
2742
2743 static int
2744 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
2745                   char *glob, char *cmd, char *param, int enabled)
2746 {
2747         struct trace_event_file *file;
2748         struct ftrace_probe_ops *ops;
2749         struct event_probe_data *data;
2750         const char *system;
2751         const char *event;
2752         char *number;
2753         bool enable;
2754         int ret;
2755
2756         if (!tr)
2757                 return -ENODEV;
2758
2759         /* hash funcs only work with set_ftrace_filter */
2760         if (!enabled || !param)
2761                 return -EINVAL;
2762
2763         system = strsep(&param, ":");
2764         if (!param)
2765                 return -EINVAL;
2766
2767         event = strsep(&param, ":");
2768
2769         mutex_lock(&event_mutex);
2770
2771         ret = -EINVAL;
2772         file = find_event_file(tr, system, event);
2773         if (!file)
2774                 goto out;
2775
2776         enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
2777
2778         if (enable)
2779                 ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
2780         else
2781                 ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
2782
2783         if (glob[0] == '!') {
2784                 ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);
2785                 goto out;
2786         }
2787
2788         ret = -ENOMEM;
2789
2790         data = kzalloc(sizeof(*data), GFP_KERNEL);
2791         if (!data)
2792                 goto out;
2793
2794         data->enable = enable;
2795         data->count = -1;
2796         data->file = file;
2797
2798         if (!param)
2799                 goto out_reg;
2800
2801         number = strsep(&param, ":");
2802
2803         ret = -EINVAL;
2804         if (!strlen(number))
2805                 goto out_free;
2806
2807         /*
2808          * We use the callback data field (which is a pointer)
2809          * as our counter.
2810          */
2811         ret = kstrtoul(number, 0, &data->count);
2812         if (ret)
2813                 goto out_free;
2814
2815  out_reg:
2816         /* Don't let event modules unload while probe registered */
2817         ret = try_module_get(file->event_call->mod);
2818         if (!ret) {
2819                 ret = -EBUSY;
2820                 goto out_free;
2821         }
2822
2823         ret = __ftrace_event_enable_disable(file, 1, 1);
2824         if (ret < 0)
2825                 goto out_put;
2826
2827         ret = register_ftrace_function_probe(glob, tr, ops, data);
2828         /*
2829          * The above returns on success the # of functions enabled,
2830          * but if it didn't find any functions it returns zero.
2831          * Consider no functions a failure too.
2832          */
2833         if (!ret) {
2834                 ret = -ENOENT;
2835                 goto out_disable;
2836         } else if (ret < 0)
2837                 goto out_disable;
2838         /* Just return zero, not the number of enabled functions */
2839         ret = 0;
2840  out:
2841         mutex_unlock(&event_mutex);
2842         return ret;
2843
2844  out_disable:
2845         __ftrace_event_enable_disable(file, 0, 1);
2846  out_put:
2847         module_put(file->event_call->mod);
2848  out_free:
2849         kfree(data);
2850         goto out;
2851 }
2852
2853 static struct ftrace_func_command event_enable_cmd = {
2854         .name                   = ENABLE_EVENT_STR,
2855         .func                   = event_enable_func,
2856 };
2857
2858 static struct ftrace_func_command event_disable_cmd = {
2859         .name                   = DISABLE_EVENT_STR,
2860         .func                   = event_enable_func,
2861 };
2862
2863 static __init int register_event_cmds(void)
2864 {
2865         int ret;
2866
2867         ret = register_ftrace_command(&event_enable_cmd);
2868         if (WARN_ON(ret < 0))
2869                 return ret;
2870         ret = register_ftrace_command(&event_disable_cmd);
2871         if (WARN_ON(ret < 0))
2872                 unregister_ftrace_command(&event_enable_cmd);
2873         return ret;
2874 }
2875 #else
2876 static inline int register_event_cmds(void) { return 0; }
2877 #endif /* CONFIG_DYNAMIC_FTRACE */
2878
2879 /*
2880  * The top level array has already had its trace_event_file
2881  * descriptors created in order to allow for early events to
2882  * be recorded. This function is called after the tracefs has been
2883  * initialized, and we now have to create the files associated
2884  * to the events.
2885  */
2886 static __init void
2887 __trace_early_add_event_dirs(struct trace_array *tr)
2888 {
2889         struct trace_event_file *file;
2890         int ret;
2891
2892
2893         list_for_each_entry(file, &tr->events, list) {
2894                 ret = event_create_dir(tr->event_dir, file);
2895                 if (ret < 0)
2896                         pr_warn("Could not create directory for event %s\n",
2897                                 trace_event_name(file->event_call));
2898         }
2899 }
2900
2901 /*
2902  * For early boot up, the top trace array requires to have
2903  * a list of events that can be enabled. This must be done before
2904  * the filesystem is set up in order to allow events to be traced
2905  * early.
2906  */
2907 static __init void
2908 __trace_early_add_events(struct trace_array *tr)
2909 {
2910         struct trace_event_call *call;
2911         int ret;
2912
2913         list_for_each_entry(call, &ftrace_events, list) {
2914                 /* Early boot up should not have any modules loaded */
2915                 if (WARN_ON_ONCE(call->mod))
2916                         continue;
2917
2918                 ret = __trace_early_add_new_event(call, tr);
2919                 if (ret < 0)
2920                         pr_warn("Could not create early event %s\n",
2921                                 trace_event_name(call));
2922         }
2923 }
2924
2925 /* Remove the event directory structure for a trace directory. */
2926 static void
2927 __trace_remove_event_dirs(struct trace_array *tr)
2928 {
2929         struct trace_event_file *file, *next;
2930
2931         list_for_each_entry_safe(file, next, &tr->events, list)
2932                 remove_event_file_dir(file);
2933 }
2934
2935 static void __add_event_to_tracers(struct trace_event_call *call)
2936 {
2937         struct trace_array *tr;
2938
2939         list_for_each_entry(tr, &ftrace_trace_arrays, list)
2940                 __trace_add_new_event(call, tr);
2941 }
2942
2943 extern struct trace_event_call *__start_ftrace_events[];
2944 extern struct trace_event_call *__stop_ftrace_events[];
2945
2946 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
2947
2948 static __init int setup_trace_event(char *str)
2949 {
2950         strlcpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
2951         ring_buffer_expanded = true;
2952         tracing_selftest_disabled = true;
2953
2954         return 1;
2955 }
2956 __setup("trace_event=", setup_trace_event);
2957
2958 /* Expects to have event_mutex held when called */
2959 static int
2960 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
2961 {
2962         struct dentry *d_events;
2963         struct dentry *entry;
2964
2965         entry = tracefs_create_file("set_event", 0644, parent,
2966                                     tr, &ftrace_set_event_fops);
2967         if (!entry) {
2968                 pr_warn("Could not create tracefs 'set_event' entry\n");
2969                 return -ENOMEM;
2970         }
2971
2972         d_events = tracefs_create_dir("events", parent);
2973         if (!d_events) {
2974                 pr_warn("Could not create tracefs 'events' directory\n");
2975                 return -ENOMEM;
2976         }
2977
2978         entry = trace_create_file("enable", 0644, d_events,
2979                                   tr, &ftrace_tr_enable_fops);
2980         if (!entry) {
2981                 pr_warn("Could not create tracefs 'enable' entry\n");
2982                 return -ENOMEM;
2983         }
2984
2985         /* There are not as crucial, just warn if they are not created */
2986
2987         entry = tracefs_create_file("set_event_pid", 0644, parent,
2988                                     tr, &ftrace_set_event_pid_fops);
2989         if (!entry)
2990                 pr_warn("Could not create tracefs 'set_event_pid' entry\n");
2991
2992         /* ring buffer internal formats */
2993         entry = trace_create_file("header_page", 0444, d_events,
2994                                   ring_buffer_print_page_header,
2995                                   &ftrace_show_header_fops);
2996         if (!entry)
2997                 pr_warn("Could not create tracefs 'header_page' entry\n");
2998
2999         entry = trace_create_file("header_event", 0444, d_events,
3000                                   ring_buffer_print_entry_header,
3001                                   &ftrace_show_header_fops);
3002         if (!entry)
3003                 pr_warn("Could not create tracefs 'header_event' entry\n");
3004
3005         tr->event_dir = d_events;
3006
3007         return 0;
3008 }
3009
3010 /**
3011  * event_trace_add_tracer - add a instance of a trace_array to events
3012  * @parent: The parent dentry to place the files/directories for events in
3013  * @tr: The trace array associated with these events
3014  *
3015  * When a new instance is created, it needs to set up its events
3016  * directory, as well as other files associated with events. It also
3017  * creates the event hierachry in the @parent/events directory.
3018  *
3019  * Returns 0 on success.
3020  *
3021  * Must be called with event_mutex held.
3022  */
3023 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
3024 {
3025         int ret;
3026
3027         lockdep_assert_held(&event_mutex);
3028
3029         ret = create_event_toplevel_files(parent, tr);
3030         if (ret)
3031                 goto out;
3032
3033         down_write(&trace_event_sem);
3034         __trace_add_event_dirs(tr);
3035         up_write(&trace_event_sem);
3036
3037  out:
3038         return ret;
3039 }
3040
3041 /*
3042  * The top trace array already had its file descriptors created.
3043  * Now the files themselves need to be created.
3044  */
3045 static __init int
3046 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
3047 {
3048         int ret;
3049
3050         mutex_lock(&event_mutex);
3051
3052         ret = create_event_toplevel_files(parent, tr);
3053         if (ret)
3054                 goto out_unlock;
3055
3056         down_write(&trace_event_sem);
3057         __trace_early_add_event_dirs(tr);
3058         up_write(&trace_event_sem);
3059
3060  out_unlock:
3061         mutex_unlock(&event_mutex);
3062
3063         return ret;
3064 }
3065
3066 /* Must be called with event_mutex held */
3067 int event_trace_del_tracer(struct trace_array *tr)
3068 {
3069         lockdep_assert_held(&event_mutex);
3070
3071         /* Disable any event triggers and associated soft-disabled events */
3072         clear_event_triggers(tr);
3073
3074         /* Clear the pid list */
3075         __ftrace_clear_event_pids(tr);
3076
3077         /* Disable any running events */
3078         __ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
3079
3080         /* Make sure no more events are being executed */
3081         tracepoint_synchronize_unregister();
3082
3083         down_write(&trace_event_sem);
3084         __trace_remove_event_dirs(tr);
3085         tracefs_remove_recursive(tr->event_dir);
3086         up_write(&trace_event_sem);
3087
3088         tr->event_dir = NULL;
3089
3090         return 0;
3091 }
3092
3093 static __init int event_trace_memsetup(void)
3094 {
3095         field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
3096         file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
3097         return 0;
3098 }
3099
3100 static __init void
3101 early_enable_events(struct trace_array *tr, bool disable_first)
3102 {
3103         char *buf = bootup_event_buf;
3104         char *token;
3105         int ret;
3106
3107         while (true) {
3108                 token = strsep(&buf, ",");
3109
3110                 if (!token)
3111                         break;
3112
3113                 if (*token) {
3114                         /* Restarting syscalls requires that we stop them first */
3115                         if (disable_first)
3116                                 ftrace_set_clr_event(tr, token, 0);
3117
3118                         ret = ftrace_set_clr_event(tr, token, 1);
3119                         if (ret)
3120                                 pr_warn("Failed to enable trace event: %s\n", token);
3121                 }
3122
3123                 /* Put back the comma to allow this to be called again */
3124                 if (buf)
3125                         *(buf - 1) = ',';
3126         }
3127 }
3128
3129 static __init int event_trace_enable(void)
3130 {
3131         struct trace_array *tr = top_trace_array();
3132         struct trace_event_call **iter, *call;
3133         int ret;
3134
3135         if (!tr)
3136                 return -ENODEV;
3137
3138         for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
3139
3140                 call = *iter;
3141                 ret = event_init(call);
3142                 if (!ret)
3143                         list_add(&call->list, &ftrace_events);
3144         }
3145
3146         /*
3147          * We need the top trace array to have a working set of trace
3148          * points at early init, before the debug files and directories
3149          * are created. Create the file entries now, and attach them
3150          * to the actual file dentries later.
3151          */
3152         __trace_early_add_events(tr);
3153
3154         early_enable_events(tr, false);
3155
3156         trace_printk_start_comm();
3157
3158         register_event_cmds();
3159
3160         register_trigger_cmds();
3161
3162         return 0;
3163 }
3164
3165 /*
3166  * event_trace_enable() is called from trace_event_init() first to
3167  * initialize events and perhaps start any events that are on the
3168  * command line. Unfortunately, there are some events that will not
3169  * start this early, like the system call tracepoints that need
3170  * to set the TIF_SYSCALL_TRACEPOINT flag of pid 1. But event_trace_enable()
3171  * is called before pid 1 starts, and this flag is never set, making
3172  * the syscall tracepoint never get reached, but the event is enabled
3173  * regardless (and not doing anything).
3174  */
3175 static __init int event_trace_enable_again(void)
3176 {
3177         struct trace_array *tr;
3178
3179         tr = top_trace_array();
3180         if (!tr)
3181                 return -ENODEV;
3182
3183         early_enable_events(tr, true);
3184
3185         return 0;
3186 }
3187
3188 early_initcall(event_trace_enable_again);
3189
3190 __init int event_trace_init(void)
3191 {
3192         struct trace_array *tr;
3193         struct dentry *d_tracer;
3194         struct dentry *entry;
3195         int ret;
3196
3197         tr = top_trace_array();
3198         if (!tr)
3199                 return -ENODEV;
3200
3201         d_tracer = tracing_init_dentry();
3202         if (IS_ERR(d_tracer))
3203                 return 0;
3204
3205         entry = tracefs_create_file("available_events", 0444, d_tracer,
3206                                     tr, &ftrace_avail_fops);
3207         if (!entry)
3208                 pr_warn("Could not create tracefs 'available_events' entry\n");
3209
3210         if (trace_define_generic_fields())
3211                 pr_warn("tracing: Failed to allocated generic fields");
3212
3213         if (trace_define_common_fields())
3214                 pr_warn("tracing: Failed to allocate common fields");
3215
3216         ret = early_event_add_tracer(d_tracer, tr);
3217         if (ret)
3218                 return ret;
3219
3220 #ifdef CONFIG_MODULES
3221         ret = register_module_notifier(&trace_module_nb);
3222         if (ret)
3223                 pr_warn("Failed to register trace events module notifier\n");
3224 #endif
3225         return 0;
3226 }
3227
3228 void __init trace_event_init(void)
3229 {
3230         event_trace_memsetup();
3231         init_ftrace_syscalls();
3232         event_trace_enable();
3233 }
3234
3235 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
3236
3237 static DEFINE_SPINLOCK(test_spinlock);
3238 static DEFINE_SPINLOCK(test_spinlock_irq);
3239 static DEFINE_MUTEX(test_mutex);
3240
3241 static __init void test_work(struct work_struct *dummy)
3242 {
3243         spin_lock(&test_spinlock);
3244         spin_lock_irq(&test_spinlock_irq);
3245         udelay(1);
3246         spin_unlock_irq(&test_spinlock_irq);
3247         spin_unlock(&test_spinlock);
3248
3249         mutex_lock(&test_mutex);
3250         msleep(1);
3251         mutex_unlock(&test_mutex);
3252 }
3253
3254 static __init int event_test_thread(void *unused)
3255 {
3256         void *test_malloc;
3257
3258         test_malloc = kmalloc(1234, GFP_KERNEL);
3259         if (!test_malloc)
3260                 pr_info("failed to kmalloc\n");
3261
3262         schedule_on_each_cpu(test_work);
3263
3264         kfree(test_malloc);
3265
3266         set_current_state(TASK_INTERRUPTIBLE);
3267         while (!kthread_should_stop()) {
3268                 schedule();
3269                 set_current_state(TASK_INTERRUPTIBLE);
3270         }
3271         __set_current_state(TASK_RUNNING);
3272
3273         return 0;
3274 }
3275
3276 /*
3277  * Do various things that may trigger events.
3278  */
3279 static __init void event_test_stuff(void)
3280 {
3281         struct task_struct *test_thread;
3282
3283         test_thread = kthread_run(event_test_thread, NULL, "test-events");
3284         msleep(1);
3285         kthread_stop(test_thread);
3286 }
3287
3288 /*
3289  * For every trace event defined, we will test each trace point separately,
3290  * and then by groups, and finally all trace points.
3291  */
3292 static __init void event_trace_self_tests(void)
3293 {
3294         struct trace_subsystem_dir *dir;
3295         struct trace_event_file *file;
3296         struct trace_event_call *call;
3297         struct event_subsystem *system;
3298         struct trace_array *tr;
3299         int ret;
3300
3301         tr = top_trace_array();
3302         if (!tr)
3303                 return;
3304
3305         pr_info("Running tests on trace events:\n");
3306
3307         list_for_each_entry(file, &tr->events, list) {
3308
3309                 call = file->event_call;
3310
3311                 /* Only test those that have a probe */
3312                 if (!call->class || !call->class->probe)
3313                         continue;
3314
3315 /*
3316  * Testing syscall events here is pretty useless, but
3317  * we still do it if configured. But this is time consuming.
3318  * What we really need is a user thread to perform the
3319  * syscalls as we test.
3320  */
3321 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
3322                 if (call->class->system &&
3323                     strcmp(call->class->system, "syscalls") == 0)
3324                         continue;
3325 #endif
3326
3327                 pr_info("Testing event %s: ", trace_event_name(call));
3328
3329                 /*
3330                  * If an event is already enabled, someone is using
3331                  * it and the self test should not be on.
3332                  */
3333                 if (file->flags & EVENT_FILE_FL_ENABLED) {
3334                         pr_warn("Enabled event during self test!\n");
3335                         WARN_ON_ONCE(1);
3336                         continue;
3337                 }
3338
3339                 ftrace_event_enable_disable(file, 1);
3340                 event_test_stuff();
3341                 ftrace_event_enable_disable(file, 0);
3342
3343                 pr_cont("OK\n");
3344         }
3345
3346         /* Now test at the sub system level */
3347
3348         pr_info("Running tests on trace event systems:\n");
3349
3350         list_for_each_entry(dir, &tr->systems, list) {
3351
3352                 system = dir->subsystem;
3353
3354                 /* the ftrace system is special, skip it */
3355                 if (strcmp(system->name, "ftrace") == 0)
3356                         continue;
3357
3358                 pr_info("Testing event system %s: ", system->name);
3359
3360                 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
3361                 if (WARN_ON_ONCE(ret)) {
3362                         pr_warn("error enabling system %s\n",
3363                                 system->name);
3364                         continue;
3365                 }
3366
3367                 event_test_stuff();
3368
3369                 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
3370                 if (WARN_ON_ONCE(ret)) {
3371                         pr_warn("error disabling system %s\n",
3372                                 system->name);
3373                         continue;
3374                 }
3375
3376                 pr_cont("OK\n");
3377         }
3378
3379         /* Test with all events enabled */
3380
3381         pr_info("Running tests on all trace events:\n");
3382         pr_info("Testing all events: ");
3383
3384         ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
3385         if (WARN_ON_ONCE(ret)) {
3386                 pr_warn("error enabling all events\n");
3387                 return;
3388         }
3389
3390         event_test_stuff();
3391
3392         /* reset sysname */
3393         ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
3394         if (WARN_ON_ONCE(ret)) {
3395                 pr_warn("error disabling all events\n");
3396                 return;
3397         }
3398
3399         pr_cont("OK\n");
3400 }
3401
3402 #ifdef CONFIG_FUNCTION_TRACER
3403
3404 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
3405
3406 static struct trace_event_file event_trace_file __initdata;
3407
3408 static void __init
3409 function_test_events_call(unsigned long ip, unsigned long parent_ip,
3410                           struct ftrace_ops *op, struct pt_regs *pt_regs)
3411 {
3412         struct ring_buffer_event *event;
3413         struct ring_buffer *buffer;
3414         struct ftrace_entry *entry;
3415         unsigned long flags;
3416         long disabled;
3417         int cpu;
3418         int pc;
3419
3420         pc = preempt_count();
3421         preempt_disable_notrace();
3422         cpu = raw_smp_processor_id();
3423         disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
3424
3425         if (disabled != 1)
3426                 goto out;
3427
3428         local_save_flags(flags);
3429
3430         event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
3431                                                 TRACE_FN, sizeof(*entry),
3432                                                 flags, pc);
3433         if (!event)
3434                 goto out;
3435         entry   = ring_buffer_event_data(event);
3436         entry->ip                       = ip;
3437         entry->parent_ip                = parent_ip;
3438
3439         event_trigger_unlock_commit(&event_trace_file, buffer, event,
3440                                     entry, flags, pc);
3441  out:
3442         atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
3443         preempt_enable_notrace();
3444 }
3445
3446 static struct ftrace_ops trace_ops __initdata  =
3447 {
3448         .func = function_test_events_call,
3449         .flags = FTRACE_OPS_FL_RECURSION_SAFE,
3450 };
3451
3452 static __init void event_trace_self_test_with_function(void)
3453 {
3454         int ret;
3455
3456         event_trace_file.tr = top_trace_array();
3457         if (WARN_ON(!event_trace_file.tr))
3458                 return;
3459
3460         ret = register_ftrace_function(&trace_ops);
3461         if (WARN_ON(ret < 0)) {
3462                 pr_info("Failed to enable function tracer for event tests\n");
3463                 return;
3464         }
3465         pr_info("Running tests again, along with the function tracer\n");
3466         event_trace_self_tests();
3467         unregister_ftrace_function(&trace_ops);
3468 }
3469 #else
3470 static __init void event_trace_self_test_with_function(void)
3471 {
3472 }
3473 #endif
3474
3475 static __init int event_trace_self_tests_init(void)
3476 {
3477         if (!tracing_selftest_disabled) {
3478                 event_trace_self_tests();
3479                 event_trace_self_test_with_function();
3480         }
3481
3482         return 0;
3483 }
3484
3485 late_initcall(event_trace_self_tests_init);
3486
3487 #endif