kernel/trace/trace_probe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common code for probe-based Dynamic events.
   4  *
   5  * This code was copied from kernel/trace/trace_kprobe.c written by
   6  * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
   7  *
   8  * Updates to make this generic:
   9  * Copyright (C) IBM Corporation, 2010-2011
  10  * Author:     Srikar Dronamraju
  11  */
  12 #define pr_fmt(fmt)     "trace_probe: " fmt
  13
  14 #include "trace_probe.h"
  15
  16 #undef C
  17 #define C(a, b)         b
  18
  19 static const char *trace_probe_err_text[] = { ERRORS };
  20
  21 static const char *reserved_field_names[] = {
  22         "common_type",
  23         "common_flags",
  24         "common_preempt_count",
  25         "common_pid",
  26         "common_tgid",
  27         FIELD_STRING_IP,
  28         FIELD_STRING_RETIP,
  29         FIELD_STRING_FUNC,
  30 };
  31
  32 /* Printing  in basic type function template */
  33 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)                  \
  34 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
  35 {                                                                       \
  36         trace_seq_printf(s, fmt, *(type *)data);                        \
  37         return !trace_seq_has_overflowed(s);                            \
  38 }                                                                       \
  39 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
  40
  41 DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
  42 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
  43 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
  44 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
  45 DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
  46 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
  47 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
  48 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
  49 DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
  50 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
  51 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
  52 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
  53 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
  54
  55 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
  56 {
  57         trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
  58         return !trace_seq_has_overflowed(s);
  59 }
  60 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
  61
  62 /* Print type function for string type */
  63 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
  64 {
  65         int len = *(u32 *)data >> 16;
  66
  67         if (!len)
  68                 trace_seq_puts(s, "(fault)");
  69         else
  70                 trace_seq_printf(s, "\"%s\"",
  71                                  (const char *)get_loc_data(data, ent));
  72         return !trace_seq_has_overflowed(s);
  73 }
  74
  75 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
  76
  77 /* Fetch type information table */
  78 static const struct fetch_type probe_fetch_types[] = {
  79         /* Special types */
  80         __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
  81                             "__data_loc char[]"),
  82         __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
  83                             "__data_loc char[]"),
  84         __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
  85                             "__data_loc char[]"),
  86         /* Basic types */
  87         ASSIGN_FETCH_TYPE(u8,  u8,  0),
  88         ASSIGN_FETCH_TYPE(u16, u16, 0),
  89         ASSIGN_FETCH_TYPE(u32, u32, 0),
  90         ASSIGN_FETCH_TYPE(u64, u64, 0),
  91         ASSIGN_FETCH_TYPE(s8,  u8,  1),
  92         ASSIGN_FETCH_TYPE(s16, u16, 1),
  93         ASSIGN_FETCH_TYPE(s32, u32, 1),
  94         ASSIGN_FETCH_TYPE(s64, u64, 1),
  95         ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
  96         ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
  97         ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
  98         ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
  99         ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
 100         ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
 101
 102         ASSIGN_FETCH_TYPE_END
 103 };
 104
 105 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
 106 {
 107         int i;
 108
 109         /* Reject the symbol/symstr for uprobes */
 110         if (type && (flags & TPARG_FL_USER) &&
 111             (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
 112                 return NULL;
 113
 114         if (!type)
 115                 type = DEFAULT_FETCH_TYPE_STR;
 116
 117         /* Special case: bitfield */
 118         if (*type == 'b') {
 119                 unsigned long bs;
 120
 121                 type = strchr(type, '/');
 122                 if (!type)
 123                         goto fail;
 124
 125                 type++;
 126                 if (kstrtoul(type, 0, &bs))
 127                         goto fail;
 128
 129                 switch (bs) {
 130                 case 8:
 131                         return find_fetch_type("u8", flags);
 132                 case 16:
 133                         return find_fetch_type("u16", flags);
 134                 case 32:
 135                         return find_fetch_type("u32", flags);
 136                 case 64:
 137                         return find_fetch_type("u64", flags);
 138                 default:
 139                         goto fail;
 140                 }
 141         }
 142
 143         for (i = 0; probe_fetch_types[i].name; i++) {
 144                 if (strcmp(type, probe_fetch_types[i].name) == 0)
 145                         return &probe_fetch_types[i];
 146         }
 147
 148 fail:
 149         return NULL;
 150 }
 151
 152 static struct trace_probe_log trace_probe_log;
 153
 154 void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
 155 {
 156         trace_probe_log.subsystem = subsystem;
 157         trace_probe_log.argc = argc;
 158         trace_probe_log.argv = argv;
 159         trace_probe_log.index = 0;
 160 }
 161
 162 void trace_probe_log_clear(void)
 163 {
 164         memset(&trace_probe_log, 0, sizeof(trace_probe_log));
 165 }
 166
 167 void trace_probe_log_set_index(int index)
 168 {
 169         trace_probe_log.index = index;
 170 }
 171
 172 void __trace_probe_log_err(int offset, int err_type)
 173 {
 174         char *command, *p;
 175         int i, len = 0, pos = 0;
 176
 177         if (!trace_probe_log.argv)
 178                 return;
 179
 180         /* Recalculate the length and allocate buffer */
 181         for (i = 0; i < trace_probe_log.argc; i++) {
 182                 if (i == trace_probe_log.index)
 183                         pos = len;
 184                 len += strlen(trace_probe_log.argv[i]) + 1;
 185         }
 186         command = kzalloc(len, GFP_KERNEL);
 187         if (!command)
 188                 return;
 189
 190         if (trace_probe_log.index >= trace_probe_log.argc) {
 191                 /**
 192                  * Set the error position is next to the last arg + space.
 193                  * Note that len includes the terminal null and the cursor
 194                  * appears at pos + 1.
 195                  */
 196                 pos = len;
 197                 offset = 0;
 198         }
 199
 200         /* And make a command string from argv array */
 201         p = command;
 202         for (i = 0; i < trace_probe_log.argc; i++) {
 203                 len = strlen(trace_probe_log.argv[i]);
 204                 strcpy(p, trace_probe_log.argv[i]);
 205                 p[len] = ' ';
 206                 p += len + 1;
 207         }
 208         *(p - 1) = '\0';
 209
 210         tracing_log_err(NULL, trace_probe_log.subsystem, command,
 211                         trace_probe_err_text, err_type, pos + offset);
 212
 213         kfree(command);
 214 }
 215
 216 /* Split symbol and offset. */
 217 int traceprobe_split_symbol_offset(char *symbol, long *offset)
 218 {
 219         char *tmp;
 220         int ret;
 221
 222         if (!offset)
 223                 return -EINVAL;
 224
 225         tmp = strpbrk(symbol, "+-");
 226         if (tmp) {
 227                 ret = kstrtol(tmp, 0, offset);
 228                 if (ret)
 229                         return ret;
 230                 *tmp = '\0';
 231         } else
 232                 *offset = 0;
 233
 234         return 0;
 235 }
 236
 237 /* @buf must has MAX_EVENT_NAME_LEN size */
 238 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
 239                                 char *buf, int offset)
 240 {
 241         const char *slash, *event = *pevent;
 242         int len;
 243
 244         slash = strchr(event, '/');
 245         if (!slash)
 246                 slash = strchr(event, '.');
 247
 248         if (slash) {
 249                 if (slash == event) {
 250                         trace_probe_log_err(offset, NO_GROUP_NAME);
 251                         return -EINVAL;
 252                 }
 253                 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
 254                         trace_probe_log_err(offset, GROUP_TOO_LONG);
 255                         return -EINVAL;
 256                 }
 257                 strlcpy(buf, event, slash - event + 1);
 258                 if (!is_good_system_name(buf)) {
 259                         trace_probe_log_err(offset, BAD_GROUP_NAME);
 260                         return -EINVAL;
 261                 }
 262                 *pgroup = buf;
 263                 *pevent = slash + 1;
 264                 offset += slash - event + 1;
 265                 event = *pevent;
 266         }
 267         len = strlen(event);
 268         if (len == 0) {
 269                 if (slash) {
 270                         *pevent = NULL;
 271                         return 0;
 272                 }
 273                 trace_probe_log_err(offset, NO_EVENT_NAME);
 274                 return -EINVAL;
 275         } else if (len > MAX_EVENT_NAME_LEN) {
 276                 trace_probe_log_err(offset, EVENT_TOO_LONG);
 277                 return -EINVAL;
 278         }
 279         if (!is_good_name(event)) {
 280                 trace_probe_log_err(offset, BAD_EVENT_NAME);
 281                 return -EINVAL;
 282         }
 283         return 0;
 284 }
 285
 286 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
 287
 288 static int parse_probe_vars(char *arg, const struct fetch_type *t,
 289                         struct fetch_insn *code, unsigned int flags, int offs)
 290 {
 291         unsigned long param;
 292         int ret = 0;
 293         int len;
 294
 295         if (flags & TPARG_FL_TPOINT) {
 296                 if (code->data)
 297                         return -EFAULT;
 298                 code->data = kstrdup(arg, GFP_KERNEL);
 299                 if (!code->data)
 300                         return -ENOMEM;
 301                 code->op = FETCH_OP_TP_ARG;
 302         } else if (strcmp(arg, "retval") == 0) {
 303                 if (flags & TPARG_FL_RETURN) {
 304                         code->op = FETCH_OP_RETVAL;
 305                 } else {
 306                         trace_probe_log_err(offs, RETVAL_ON_PROBE);
 307                         ret = -EINVAL;
 308                 }
 309         } else if ((len = str_has_prefix(arg, "stack"))) {
 310                 if (arg[len] == '\0') {
 311                         code->op = FETCH_OP_STACKP;
 312                 } else if (isdigit(arg[len])) {
 313                         ret = kstrtoul(arg + len, 10, &param);
 314                         if (ret) {
 315                                 goto inval_var;
 316                         } else if ((flags & TPARG_FL_KERNEL) &&
 317                                     param > PARAM_MAX_STACK) {
 318                                 trace_probe_log_err(offs, BAD_STACK_NUM);
 319                                 ret = -EINVAL;
 320                         } else {
 321                                 code->op = FETCH_OP_STACK;
 322                                 code->param = (unsigned int)param;
 323                         }
 324                 } else
 325                         goto inval_var;
 326         } else if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 327                 code->op = FETCH_OP_COMM;
 328 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 329         } else if (((flags & TPARG_FL_MASK) ==
 330                     (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) &&
 331                    (len = str_has_prefix(arg, "arg"))) {
 332                 ret = kstrtoul(arg + len, 10, &param);
 333                 if (ret) {
 334                         goto inval_var;
 335                 } else if (!param || param > PARAM_MAX_STACK) {
 336                         trace_probe_log_err(offs, BAD_ARG_NUM);
 337                         return -EINVAL;
 338                 }
 339                 code->op = FETCH_OP_ARG;
 340                 code->param = (unsigned int)param - 1;
 341 #endif
 342         } else
 343                 goto inval_var;
 344
 345         return ret;
 346
 347 inval_var:
 348         trace_probe_log_err(offs, BAD_VAR);
 349         return -EINVAL;
 350 }
 351
 352 static int str_to_immediate(char *str, unsigned long *imm)
 353 {
 354         if (isdigit(str[0]))
 355                 return kstrtoul(str, 0, imm);
 356         else if (str[0] == '-')
 357                 return kstrtol(str, 0, (long *)imm);
 358         else if (str[0] == '+')
 359                 return kstrtol(str + 1, 0, (long *)imm);
 360         return -EINVAL;
 361 }
 362
 363 static int __parse_imm_string(char *str, char **pbuf, int offs)
 364 {
 365         size_t len = strlen(str);
 366
 367         if (str[len - 1] != '"') {
 368                 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
 369                 return -EINVAL;
 370         }
 371         *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
 372         if (!*pbuf)
 373                 return -ENOMEM;
 374         return 0;
 375 }
 376
 377 /* Recursive argument parser */
 378 static int
 379 parse_probe_arg(char *arg, const struct fetch_type *type,
 380                 struct fetch_insn **pcode, struct fetch_insn *end,
 381                 unsigned int flags, int offs)
 382 {
 383         struct fetch_insn *code = *pcode;
 384         unsigned long param;
 385         int deref = FETCH_OP_DEREF;
 386         long offset = 0;
 387         char *tmp;
 388         int ret = 0;
 389
 390         switch (arg[0]) {
 391         case '$':
 392                 ret = parse_probe_vars(arg + 1, type, code, flags, offs);
 393                 break;
 394
 395         case '%':       /* named register */
 396                 if (flags & TPARG_FL_TPOINT) {
 397                         /* eprobes do not handle registers */
 398                         trace_probe_log_err(offs, BAD_VAR);
 399                         break;
 400                 }
 401                 ret = regs_query_register_offset(arg + 1);
 402                 if (ret >= 0) {
 403                         code->op = FETCH_OP_REG;
 404                         code->param = (unsigned int)ret;
 405                         ret = 0;
 406                 } else
 407                         trace_probe_log_err(offs, BAD_REG_NAME);
 408                 break;
 409
 410         case '@':       /* memory, file-offset or symbol */
 411                 if (isdigit(arg[1])) {
 412                         ret = kstrtoul(arg + 1, 0, &param);
 413                         if (ret) {
 414                                 trace_probe_log_err(offs, BAD_MEM_ADDR);
 415                                 break;
 416                         }
 417                         /* load address */
 418                         code->op = FETCH_OP_IMM;
 419                         code->immediate = param;
 420                 } else if (arg[1] == '+') {
 421                         /* kprobes don't support file offsets */
 422                         if (flags & TPARG_FL_KERNEL) {
 423                                 trace_probe_log_err(offs, FILE_ON_KPROBE);
 424                                 return -EINVAL;
 425                         }
 426                         ret = kstrtol(arg + 2, 0, &offset);
 427                         if (ret) {
 428                                 trace_probe_log_err(offs, BAD_FILE_OFFS);
 429                                 break;
 430                         }
 431
 432                         code->op = FETCH_OP_FOFFS;
 433                         code->immediate = (unsigned long)offset;  // imm64?
 434                 } else {
 435                         /* uprobes don't support symbols */
 436                         if (!(flags & TPARG_FL_KERNEL)) {
 437                                 trace_probe_log_err(offs, SYM_ON_UPROBE);
 438                                 return -EINVAL;
 439                         }
 440                         /* Preserve symbol for updating */
 441                         code->op = FETCH_NOP_SYMBOL;
 442                         code->data = kstrdup(arg + 1, GFP_KERNEL);
 443                         if (!code->data)
 444                                 return -ENOMEM;
 445                         if (++code == end) {
 446                                 trace_probe_log_err(offs, TOO_MANY_OPS);
 447                                 return -EINVAL;
 448                         }
 449                         code->op = FETCH_OP_IMM;
 450                         code->immediate = 0;
 451                 }
 452                 /* These are fetching from memory */
 453                 if (++code == end) {
 454                         trace_probe_log_err(offs, TOO_MANY_OPS);
 455                         return -EINVAL;
 456                 }
 457                 *pcode = code;
 458                 code->op = FETCH_OP_DEREF;
 459                 code->offset = offset;
 460                 break;
 461
 462         case '+':       /* deref memory */
 463         case '-':
 464                 if (arg[1] == 'u') {
 465                         deref = FETCH_OP_UDEREF;
 466                         arg[1] = arg[0];
 467                         arg++;
 468                 }
 469                 if (arg[0] == '+')
 470                         arg++;  /* Skip '+', because kstrtol() rejects it. */
 471                 tmp = strchr(arg, '(');
 472                 if (!tmp) {
 473                         trace_probe_log_err(offs, DEREF_NEED_BRACE);
 474                         return -EINVAL;
 475                 }
 476                 *tmp = '\0';
 477                 ret = kstrtol(arg, 0, &offset);
 478                 if (ret) {
 479                         trace_probe_log_err(offs, BAD_DEREF_OFFS);
 480                         break;
 481                 }
 482                 offs += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
 483                 arg = tmp + 1;
 484                 tmp = strrchr(arg, ')');
 485                 if (!tmp) {
 486                         trace_probe_log_err(offs + strlen(arg),
 487                                             DEREF_OPEN_BRACE);
 488                         return -EINVAL;
 489                 } else {
 490                         const struct fetch_type *t2 = find_fetch_type(NULL, flags);
 491
 492                         *tmp = '\0';
 493                         ret = parse_probe_arg(arg, t2, &code, end, flags, offs);
 494                         if (ret)
 495                                 break;
 496                         if (code->op == FETCH_OP_COMM ||
 497                             code->op == FETCH_OP_DATA) {
 498                                 trace_probe_log_err(offs, COMM_CANT_DEREF);
 499                                 return -EINVAL;
 500                         }
 501                         if (++code == end) {
 502                                 trace_probe_log_err(offs, TOO_MANY_OPS);
 503                                 return -EINVAL;
 504                         }
 505                         *pcode = code;
 506
 507                         code->op = deref;
 508                         code->offset = offset;
 509                 }
 510                 break;
 511         case '\\':      /* Immediate value */
 512                 if (arg[1] == '"') {    /* Immediate string */
 513                         ret = __parse_imm_string(arg + 2, &tmp, offs + 2);
 514                         if (ret)
 515                                 break;
 516                         code->op = FETCH_OP_DATA;
 517                         code->data = tmp;
 518                 } else {
 519                         ret = str_to_immediate(arg + 1, &code->immediate);
 520                         if (ret)
 521                                 trace_probe_log_err(offs + 1, BAD_IMM);
 522                         else
 523                                 code->op = FETCH_OP_IMM;
 524                 }
 525                 break;
 526         }
 527         if (!ret && code->op == FETCH_OP_NOP) {
 528                 /* Parsed, but do not find fetch method */
 529                 trace_probe_log_err(offs, BAD_FETCH_ARG);
 530                 ret = -EINVAL;
 531         }
 532         return ret;
 533 }
 534
 535 #define BYTES_TO_BITS(nb)       ((BITS_PER_LONG * (nb)) / sizeof(long))
 536
 537 /* Bitfield type needs to be parsed into a fetch function */
 538 static int __parse_bitfield_probe_arg(const char *bf,
 539                                       const struct fetch_type *t,
 540                                       struct fetch_insn **pcode)
 541 {
 542         struct fetch_insn *code = *pcode;
 543         unsigned long bw, bo;
 544         char *tail;
 545
 546         if (*bf != 'b')
 547                 return 0;
 548
 549         bw = simple_strtoul(bf + 1, &tail, 0);  /* Use simple one */
 550
 551         if (bw == 0 || *tail != '@')
 552                 return -EINVAL;
 553
 554         bf = tail + 1;
 555         bo = simple_strtoul(bf, &tail, 0);
 556
 557         if (tail == bf || *tail != '/')
 558                 return -EINVAL;
 559         code++;
 560         if (code->op != FETCH_OP_NOP)
 561                 return -EINVAL;
 562         *pcode = code;
 563
 564         code->op = FETCH_OP_MOD_BF;
 565         code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
 566         code->rshift = BYTES_TO_BITS(t->size) - bw;
 567         code->basesize = t->size;
 568
 569         return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
 570 }
 571
 572 /* String length checking wrapper */
 573 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
 574                 struct probe_arg *parg, unsigned int flags, int offset)
 575 {
 576         struct fetch_insn *code, *scode, *tmp = NULL;
 577         char *t, *t2, *t3;
 578         char *arg;
 579         int ret, len;
 580
 581         arg = kstrdup(argv, GFP_KERNEL);
 582         if (!arg)
 583                 return -ENOMEM;
 584
 585         ret = -EINVAL;
 586         len = strlen(arg);
 587         if (len > MAX_ARGSTR_LEN) {
 588                 trace_probe_log_err(offset, ARG_TOO_LONG);
 589                 goto out;
 590         } else if (len == 0) {
 591                 trace_probe_log_err(offset, NO_ARG_BODY);
 592                 goto out;
 593         }
 594
 595         ret = -ENOMEM;
 596         parg->comm = kstrdup(arg, GFP_KERNEL);
 597         if (!parg->comm)
 598                 goto out;
 599
 600         ret = -EINVAL;
 601         t = strchr(arg, ':');
 602         if (t) {
 603                 *t = '\0';
 604                 t2 = strchr(++t, '[');
 605                 if (t2) {
 606                         *t2++ = '\0';
 607                         t3 = strchr(t2, ']');
 608                         if (!t3) {
 609                                 offset += t2 + strlen(t2) - arg;
 610                                 trace_probe_log_err(offset,
 611                                                     ARRAY_NO_CLOSE);
 612                                 goto out;
 613                         } else if (t3[1] != '\0') {
 614                                 trace_probe_log_err(offset + t3 + 1 - arg,
 615                                                     BAD_ARRAY_SUFFIX);
 616                                 goto out;
 617                         }
 618                         *t3 = '\0';
 619                         if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
 620                                 trace_probe_log_err(offset + t2 - arg,
 621                                                     BAD_ARRAY_NUM);
 622                                 goto out;
 623                         }
 624                         if (parg->count > MAX_ARRAY_LEN) {
 625                                 trace_probe_log_err(offset + t2 - arg,
 626                                                     ARRAY_TOO_BIG);
 627                                 goto out;
 628                         }
 629                 }
 630         }
 631
 632         /*
 633          * Since $comm and immediate string can not be dereferenced,
 634          * we can find those by strcmp. But ignore for eprobes.
 635          */
 636         if (!(flags & TPARG_FL_TPOINT) &&
 637             (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
 638              strncmp(arg, "\\\"", 2) == 0)) {
 639                 /* The type of $comm must be "string", and not an array. */
 640                 if (parg->count || (t && strcmp(t, "string")))
 641                         goto out;
 642                 parg->type = find_fetch_type("string", flags);
 643         } else
 644                 parg->type = find_fetch_type(t, flags);
 645         if (!parg->type) {
 646                 trace_probe_log_err(offset + (t ? (t - arg) : 0), BAD_TYPE);
 647                 goto out;
 648         }
 649         parg->offset = *size;
 650         *size += parg->type->size * (parg->count ?: 1);
 651
 652         ret = -ENOMEM;
 653         if (parg->count) {
 654                 len = strlen(parg->type->fmttype) + 6;
 655                 parg->fmt = kmalloc(len, GFP_KERNEL);
 656                 if (!parg->fmt)
 657                         goto out;
 658                 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
 659                          parg->count);
 660         }
 661
 662         code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
 663         if (!code)
 664                 goto out;
 665         code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
 666
 667         ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
 668                               flags, offset);
 669         if (ret)
 670                 goto fail;
 671
 672         ret = -EINVAL;
 673         /* Store operation */
 674         if (parg->type->is_string) {
 675                 if (!strcmp(parg->type->name, "symstr")) {
 676                         if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
 677                             code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
 678                             code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
 679                                 trace_probe_log_err(offset + (t ? (t - arg) : 0),
 680                                                     BAD_SYMSTRING);
 681                                 goto fail;
 682                         }
 683                 } else {
 684                         if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
 685                             code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
 686                             code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
 687                                 trace_probe_log_err(offset + (t ? (t - arg) : 0),
 688                                                     BAD_STRING);
 689                                 goto fail;
 690                         }
 691                 }
 692                 if (!strcmp(parg->type->name, "symstr") ||
 693                     (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
 694                      code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
 695                      parg->count) {
 696                         /*
 697                          * IMM, DATA and COMM is pointing actual address, those
 698                          * must be kept, and if parg->count != 0, this is an
 699                          * array of string pointers instead of string address
 700                          * itself.
 701                          * For the symstr, it doesn't need to dereference, thus
 702                          * it just get the value.
 703                          */
 704                         code++;
 705                         if (code->op != FETCH_OP_NOP) {
 706                                 trace_probe_log_err(offset, TOO_MANY_OPS);
 707                                 goto fail;
 708                         }
 709                 }
 710                 /* If op == DEREF, replace it with STRING */
 711                 if (!strcmp(parg->type->name, "ustring") ||
 712                     code->op == FETCH_OP_UDEREF)
 713                         code->op = FETCH_OP_ST_USTRING;
 714                 else if (!strcmp(parg->type->name, "symstr"))
 715                         code->op = FETCH_OP_ST_SYMSTR;
 716                 else
 717                         code->op = FETCH_OP_ST_STRING;
 718                 code->size = parg->type->size;
 719                 parg->dynamic = true;
 720         } else if (code->op == FETCH_OP_DEREF) {
 721                 code->op = FETCH_OP_ST_MEM;
 722                 code->size = parg->type->size;
 723         } else if (code->op == FETCH_OP_UDEREF) {
 724                 code->op = FETCH_OP_ST_UMEM;
 725                 code->size = parg->type->size;
 726         } else {
 727                 code++;
 728                 if (code->op != FETCH_OP_NOP) {
 729                         trace_probe_log_err(offset, TOO_MANY_OPS);
 730                         goto fail;
 731                 }
 732                 code->op = FETCH_OP_ST_RAW;
 733                 code->size = parg->type->size;
 734         }
 735         scode = code;
 736         /* Modify operation */
 737         if (t != NULL) {
 738                 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
 739                 if (ret) {
 740                         trace_probe_log_err(offset + t - arg, BAD_BITFIELD);
 741                         goto fail;
 742                 }
 743         }
 744         ret = -EINVAL;
 745         /* Loop(Array) operation */
 746         if (parg->count) {
 747                 if (scode->op != FETCH_OP_ST_MEM &&
 748                     scode->op != FETCH_OP_ST_STRING &&
 749                     scode->op != FETCH_OP_ST_USTRING) {
 750                         trace_probe_log_err(offset + (t ? (t - arg) : 0),
 751                                             BAD_STRING);
 752                         goto fail;
 753                 }
 754                 code++;
 755                 if (code->op != FETCH_OP_NOP) {
 756                         trace_probe_log_err(offset, TOO_MANY_OPS);
 757                         goto fail;
 758                 }
 759                 code->op = FETCH_OP_LP_ARRAY;
 760                 code->param = parg->count;
 761         }
 762         code++;
 763         code->op = FETCH_OP_END;
 764
 765         ret = 0;
 766         /* Shrink down the code buffer */
 767         parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
 768         if (!parg->code)
 769                 ret = -ENOMEM;
 770         else
 771                 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
 772
 773 fail:
 774         if (ret) {
 775                 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
 776                         if (code->op == FETCH_NOP_SYMBOL ||
 777                             code->op == FETCH_OP_DATA)
 778                                 kfree(code->data);
 779         }
 780         kfree(tmp);
 781 out:
 782         kfree(arg);
 783
 784         return ret;
 785 }
 786
 787 /* Return 1 if name is reserved or already used by another argument */
 788 static int traceprobe_conflict_field_name(const char *name,
 789                                           struct probe_arg *args, int narg)
 790 {
 791         int i;
 792
 793         for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
 794                 if (strcmp(reserved_field_names[i], name) == 0)
 795                         return 1;
 796
 797         for (i = 0; i < narg; i++)
 798                 if (strcmp(args[i].name, name) == 0)
 799                         return 1;
 800
 801         return 0;
 802 }
 803
 804 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
 805                                 unsigned int flags)
 806 {
 807         struct probe_arg *parg = &tp->args[i];
 808         const char *body;
 809
 810         /* Increment count for freeing args in error case */
 811         tp->nr_args++;
 812
 813         body = strchr(arg, '=');
 814         if (body) {
 815                 if (body - arg > MAX_ARG_NAME_LEN) {
 816                         trace_probe_log_err(0, ARG_NAME_TOO_LONG);
 817                         return -EINVAL;
 818                 } else if (body == arg) {
 819                         trace_probe_log_err(0, NO_ARG_NAME);
 820                         return -EINVAL;
 821                 }
 822                 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
 823                 body++;
 824         } else {
 825                 /* If argument name is omitted, set "argN" */
 826                 parg->name = kasprintf(GFP_KERNEL, "arg%d", i + 1);
 827                 body = arg;
 828         }
 829         if (!parg->name)
 830                 return -ENOMEM;
 831
 832         if (!is_good_name(parg->name)) {
 833                 trace_probe_log_err(0, BAD_ARG_NAME);
 834                 return -EINVAL;
 835         }
 836         if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
 837                 trace_probe_log_err(0, USED_ARG_NAME);
 838                 return -EINVAL;
 839         }
 840         /* Parse fetch argument */
 841         return traceprobe_parse_probe_arg_body(body, &tp->size, parg, flags,
 842                                                body - arg);
 843 }
 844
 845 void traceprobe_free_probe_arg(struct probe_arg *arg)
 846 {
 847         struct fetch_insn *code = arg->code;
 848
 849         while (code && code->op != FETCH_OP_END) {
 850                 if (code->op == FETCH_NOP_SYMBOL ||
 851                     code->op == FETCH_OP_DATA)
 852                         kfree(code->data);
 853                 code++;
 854         }
 855         kfree(arg->code);
 856         kfree(arg->name);
 857         kfree(arg->comm);
 858         kfree(arg->fmt);
 859 }
 860
 861 int traceprobe_update_arg(struct probe_arg *arg)
 862 {
 863         struct fetch_insn *code = arg->code;
 864         long offset;
 865         char *tmp;
 866         char c;
 867         int ret = 0;
 868
 869         while (code && code->op != FETCH_OP_END) {
 870                 if (code->op == FETCH_NOP_SYMBOL) {
 871                         if (code[1].op != FETCH_OP_IMM)
 872                                 return -EINVAL;
 873
 874                         tmp = strpbrk(code->data, "+-");
 875                         if (tmp)
 876                                 c = *tmp;
 877                         ret = traceprobe_split_symbol_offset(code->data,
 878                                                              &offset);
 879                         if (ret)
 880                                 return ret;
 881
 882                         code[1].immediate =
 883                                 (unsigned long)kallsyms_lookup_name(code->data);
 884                         if (tmp)
 885                                 *tmp = c;
 886                         if (!code[1].immediate)
 887                                 return -ENOENT;
 888                         code[1].immediate += offset;
 889                 }
 890                 code++;
 891         }
 892         return 0;
 893 }
 894
 895 /* When len=0, we just calculate the needed length */
 896 #define LEN_OR_ZERO (len ? len - pos : 0)
 897 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
 898                            enum probe_print_type ptype)
 899 {
 900         struct probe_arg *parg;
 901         int i, j;
 902         int pos = 0;
 903         const char *fmt, *arg;
 904
 905         switch (ptype) {
 906         case PROBE_PRINT_NORMAL:
 907                 fmt = "(%lx)";
 908                 arg = ", REC->" FIELD_STRING_IP;
 909                 break;
 910         case PROBE_PRINT_RETURN:
 911                 fmt = "(%lx <- %lx)";
 912                 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
 913                 break;
 914         case PROBE_PRINT_EVENT:
 915                 fmt = "";
 916                 arg = "";
 917                 break;
 918         default:
 919                 WARN_ON_ONCE(1);
 920                 return 0;
 921         }
 922
 923         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
 924
 925         for (i = 0; i < tp->nr_args; i++) {
 926                 parg = tp->args + i;
 927                 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
 928                 if (parg->count) {
 929                         pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
 930                                         parg->type->fmt);
 931                         for (j = 1; j < parg->count; j++)
 932                                 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
 933                                                 parg->type->fmt);
 934                         pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
 935                 } else
 936                         pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
 937                                         parg->type->fmt);
 938         }
 939
 940         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
 941
 942         for (i = 0; i < tp->nr_args; i++) {
 943                 parg = tp->args + i;
 944                 if (parg->count) {
 945                         if (parg->type->is_string)
 946                                 fmt = ", __get_str(%s[%d])";
 947                         else
 948                                 fmt = ", REC->%s[%d]";
 949                         for (j = 0; j < parg->count; j++)
 950                                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 951                                                 fmt, parg->name, j);
 952                 } else {
 953                         if (parg->type->is_string)
 954                                 fmt = ", __get_str(%s)";
 955                         else
 956                                 fmt = ", REC->%s";
 957                         pos += snprintf(buf + pos, LEN_OR_ZERO,
 958                                         fmt, parg->name);
 959                 }
 960         }
 961
 962         /* return the length of print_fmt */
 963         return pos;
 964 }
 965 #undef LEN_OR_ZERO
 966
 967 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
 968 {
 969         struct trace_event_call *call = trace_probe_event_call(tp);
 970         int len;
 971         char *print_fmt;
 972
 973         /* First: called with 0 length to calculate the needed length */
 974         len = __set_print_fmt(tp, NULL, 0, ptype);
 975         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 976         if (!print_fmt)
 977                 return -ENOMEM;
 978
 979         /* Second: actually write the @print_fmt */
 980         __set_print_fmt(tp, print_fmt, len + 1, ptype);
 981         call->print_fmt = print_fmt;
 982
 983         return 0;
 984 }
 985
 986 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 987                                  size_t offset, struct trace_probe *tp)
 988 {
 989         int ret, i;
 990
 991         /* Set argument names as fields */
 992         for (i = 0; i < tp->nr_args; i++) {
 993                 struct probe_arg *parg = &tp->args[i];
 994                 const char *fmt = parg->type->fmttype;
 995                 int size = parg->type->size;
 996
 997                 if (parg->fmt)
 998                         fmt = parg->fmt;
 999                 if (parg->count)
1000                         size *= parg->count;
1001                 ret = trace_define_field(event_call, fmt, parg->name,
1002                                          offset + parg->offset, size,
1003                                          parg->type->is_signed,
1004                                          FILTER_OTHER);
1005                 if (ret)
1006                         return ret;
1007         }
1008         return 0;
1009 }
1010
1011 static void trace_probe_event_free(struct trace_probe_event *tpe)
1012 {
1013         kfree(tpe->class.system);
1014         kfree(tpe->call.name);
1015         kfree(tpe->call.print_fmt);
1016         kfree(tpe);
1017 }
1018
1019 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1020 {
1021         if (trace_probe_has_sibling(tp))
1022                 return -EBUSY;
1023
1024         list_del_init(&tp->list);
1025         trace_probe_event_free(tp->event);
1026
1027         tp->event = to->event;
1028         list_add_tail(&tp->list, trace_probe_probe_list(to));
1029
1030         return 0;
1031 }
1032
1033 void trace_probe_unlink(struct trace_probe *tp)
1034 {
1035         list_del_init(&tp->list);
1036         if (list_empty(trace_probe_probe_list(tp)))
1037                 trace_probe_event_free(tp->event);
1038         tp->event = NULL;
1039 }
1040
1041 void trace_probe_cleanup(struct trace_probe *tp)
1042 {
1043         int i;
1044
1045         for (i = 0; i < tp->nr_args; i++)
1046                 traceprobe_free_probe_arg(&tp->args[i]);
1047
1048         if (tp->event)
1049                 trace_probe_unlink(tp);
1050 }
1051
1052 int trace_probe_init(struct trace_probe *tp, const char *event,
1053                      const char *group, bool alloc_filter)
1054 {
1055         struct trace_event_call *call;
1056         size_t size = sizeof(struct trace_probe_event);
1057         int ret = 0;
1058
1059         if (!event || !group)
1060                 return -EINVAL;
1061
1062         if (alloc_filter)
1063                 size += sizeof(struct trace_uprobe_filter);
1064
1065         tp->event = kzalloc(size, GFP_KERNEL);
1066         if (!tp->event)
1067                 return -ENOMEM;
1068
1069         INIT_LIST_HEAD(&tp->event->files);
1070         INIT_LIST_HEAD(&tp->event->class.fields);
1071         INIT_LIST_HEAD(&tp->event->probes);
1072         INIT_LIST_HEAD(&tp->list);
1073         list_add(&tp->list, &tp->event->probes);
1074
1075         call = trace_probe_event_call(tp);
1076         call->class = &tp->event->class;
1077         call->name = kstrdup(event, GFP_KERNEL);
1078         if (!call->name) {
1079                 ret = -ENOMEM;
1080                 goto error;
1081         }
1082
1083         tp->event->class.system = kstrdup(group, GFP_KERNEL);
1084         if (!tp->event->class.system) {
1085                 ret = -ENOMEM;
1086                 goto error;
1087         }
1088
1089         return 0;
1090
1091 error:
1092         trace_probe_cleanup(tp);
1093         return ret;
1094 }
1095
1096 static struct trace_event_call *
1097 find_trace_event_call(const char *system, const char *event_name)
1098 {
1099         struct trace_event_call *tp_event;
1100         const char *name;
1101
1102         list_for_each_entry(tp_event, &ftrace_events, list) {
1103                 if (!tp_event->class->system ||
1104                     strcmp(system, tp_event->class->system))
1105                         continue;
1106                 name = trace_event_name(tp_event);
1107                 if (!name || strcmp(event_name, name))
1108                         continue;
1109                 return tp_event;
1110         }
1111
1112         return NULL;
1113 }
1114
1115 int trace_probe_register_event_call(struct trace_probe *tp)
1116 {
1117         struct trace_event_call *call = trace_probe_event_call(tp);
1118         int ret;
1119
1120         lockdep_assert_held(&event_mutex);
1121
1122         if (find_trace_event_call(trace_probe_group_name(tp),
1123                                   trace_probe_name(tp)))
1124                 return -EEXIST;
1125
1126         ret = register_trace_event(&call->event);
1127         if (!ret)
1128                 return -ENODEV;
1129
1130         ret = trace_add_event_call(call);
1131         if (ret)
1132                 unregister_trace_event(&call->event);
1133
1134         return ret;
1135 }
1136
1137 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
1138 {
1139         struct event_file_link *link;
1140
1141         link = kmalloc(sizeof(*link), GFP_KERNEL);
1142         if (!link)
1143                 return -ENOMEM;
1144
1145         link->file = file;
1146         INIT_LIST_HEAD(&link->list);
1147         list_add_tail_rcu(&link->list, &tp->event->files);
1148         trace_probe_set_flag(tp, TP_FLAG_TRACE);
1149         return 0;
1150 }
1151
1152 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
1153                                                   struct trace_event_file *file)
1154 {
1155         struct event_file_link *link;
1156
1157         trace_probe_for_each_link(link, tp) {
1158                 if (link->file == file)
1159                         return link;
1160         }
1161
1162         return NULL;
1163 }
1164
1165 int trace_probe_remove_file(struct trace_probe *tp,
1166                             struct trace_event_file *file)
1167 {
1168         struct event_file_link *link;
1169
1170         link = trace_probe_get_file_link(tp, file);
1171         if (!link)
1172                 return -ENOENT;
1173
1174         list_del_rcu(&link->list);
1175         kvfree_rcu_mightsleep(link);
1176
1177         if (list_empty(&tp->event->files))
1178                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1179
1180         return 0;
1181 }
1182
1183 /*
1184  * Return the smallest index of different type argument (start from 1).
1185  * If all argument types and name are same, return 0.
1186  */
1187 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
1188 {
1189         int i;
1190
1191         /* In case of more arguments */
1192         if (a->nr_args < b->nr_args)
1193                 return a->nr_args + 1;
1194         if (a->nr_args > b->nr_args)
1195                 return b->nr_args + 1;
1196
1197         for (i = 0; i < a->nr_args; i++) {
1198                 if ((b->nr_args <= i) ||
1199                     ((a->args[i].type != b->args[i].type) ||
1200                      (a->args[i].count != b->args[i].count) ||
1201                      strcmp(a->args[i].name, b->args[i].name)))
1202                         return i + 1;
1203         }
1204
1205         return 0;
1206 }
1207
1208 bool trace_probe_match_command_args(struct trace_probe *tp,
1209                                     int argc, const char **argv)
1210 {
1211         char buf[MAX_ARGSTR_LEN + 1];
1212         int i;
1213
1214         if (tp->nr_args < argc)
1215                 return false;
1216
1217         for (i = 0; i < argc; i++) {
1218                 snprintf(buf, sizeof(buf), "%s=%s",
1219                          tp->args[i].name, tp->args[i].comm);
1220                 if (strcmp(buf, argv[i]))
1221                         return false;
1222         }
1223         return true;
1224 }
1225
1226 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
1227 {
1228         int argc = 0, ret = 0;
1229         char **argv;
1230
1231         argv = argv_split(GFP_KERNEL, raw_command, &argc);
1232         if (!argv)
1233                 return -ENOMEM;
1234
1235         if (argc)
1236                 ret = createfn(argc, (const char **)argv);
1237
1238         argv_free(argv);
1239
1240         return ret;
1241 }
1242
1243 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
1244                  u8 *data, void *field)
1245 {
1246         void *p;
1247         int i, j;
1248
1249         for (i = 0; i < nr_args; i++) {
1250                 struct probe_arg *a = args + i;
1251
1252                 trace_seq_printf(s, " %s=", a->name);
1253                 if (likely(!a->count)) {
1254                         if (!a->type->print(s, data + a->offset, field))
1255                                 return -ENOMEM;
1256                         continue;
1257                 }
1258                 trace_seq_putc(s, '{');
1259                 p = data + a->offset;
1260                 for (j = 0; j < a->count; j++) {
1261                         if (!a->type->print(s, p, field))
1262                                 return -ENOMEM;
1263                         trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
1264                         p += a->type->size;
1265                 }
1266         }
1267         return 0;
1268 }