kernel/bpf/bpf_iter.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2020 Facebook */
   3
   4 #include <linux/fs.h>
   5 #include <linux/anon_inodes.h>
   6 #include <linux/filter.h>
   7 #include <linux/bpf.h>
   8
   9 struct bpf_iter_target_info {
  10         struct list_head list;
  11         const struct bpf_iter_reg *reg_info;
  12         u32 btf_id;     /* cached value */
  13 };
  14
  15 struct bpf_iter_link {
  16         struct bpf_link link;
  17         struct bpf_iter_aux_info aux;
  18         struct bpf_iter_target_info *tinfo;
  19 };
  20
  21 struct bpf_iter_priv_data {
  22         struct bpf_iter_target_info *tinfo;
  23         const struct bpf_iter_seq_info *seq_info;
  24         struct bpf_prog *prog;
  25         u64 session_id;
  26         u64 seq_num;
  27         bool done_stop;
  28         u8 target_private[] __aligned(8);
  29 };
  30
  31 static struct list_head targets = LIST_HEAD_INIT(targets);
  32 static DEFINE_MUTEX(targets_mutex);
  33
  34 /* protect bpf_iter_link changes */
  35 static DEFINE_MUTEX(link_mutex);
  36
  37 /* incremented on every opened seq_file */
  38 static atomic64_t session_id;
  39
  40 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
  41                             const struct bpf_iter_seq_info *seq_info);
  42
  43 static void bpf_iter_inc_seq_num(struct seq_file *seq)
  44 {
  45         struct bpf_iter_priv_data *iter_priv;
  46
  47         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  48                                  target_private);
  49         iter_priv->seq_num++;
  50 }
  51
  52 static void bpf_iter_dec_seq_num(struct seq_file *seq)
  53 {
  54         struct bpf_iter_priv_data *iter_priv;
  55
  56         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  57                                  target_private);
  58         iter_priv->seq_num--;
  59 }
  60
  61 static void bpf_iter_done_stop(struct seq_file *seq)
  62 {
  63         struct bpf_iter_priv_data *iter_priv;
  64
  65         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  66                                  target_private);
  67         iter_priv->done_stop = true;
  68 }
  69
  70 static bool bpf_iter_support_resched(struct seq_file *seq)
  71 {
  72         struct bpf_iter_priv_data *iter_priv;
  73
  74         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  75                                  target_private);
  76         return iter_priv->tinfo->reg_info->feature & BPF_ITER_RESCHED;
  77 }
  78
  79 /* maximum visited objects before bailing out */
  80 #define MAX_ITER_OBJECTS        1000000
  81
  82 /* bpf_seq_read, a customized and simpler version for bpf iterator.
  83  * no_llseek is assumed for this file.
  84  * The following are differences from seq_read():
  85  *  . fixed buffer size (PAGE_SIZE)
  86  *  . assuming no_llseek
  87  *  . stop() may call bpf program, handling potential overflow there
  88  */
  89 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
  90                             loff_t *ppos)
  91 {
  92         struct seq_file *seq = file->private_data;
  93         size_t n, offs, copied = 0;
  94         int err = 0, num_objs = 0;
  95         bool can_resched;
  96         void *p;
  97
  98         mutex_lock(&seq->lock);
  99
 100         if (!seq->buf) {
 101                 seq->size = PAGE_SIZE << 3;
 102                 seq->buf = kvmalloc(seq->size, GFP_KERNEL);
 103                 if (!seq->buf) {
 104                         err = -ENOMEM;
 105                         goto done;
 106                 }
 107         }
 108
 109         if (seq->count) {
 110                 n = min(seq->count, size);
 111                 err = copy_to_user(buf, seq->buf + seq->from, n);
 112                 if (err) {
 113                         err = -EFAULT;
 114                         goto done;
 115                 }
 116                 seq->count -= n;
 117                 seq->from += n;
 118                 copied = n;
 119                 goto done;
 120         }
 121
 122         seq->from = 0;
 123         p = seq->op->start(seq, &seq->index);
 124         if (!p)
 125                 goto stop;
 126         if (IS_ERR(p)) {
 127                 err = PTR_ERR(p);
 128                 seq->op->stop(seq, p);
 129                 seq->count = 0;
 130                 goto done;
 131         }
 132
 133         err = seq->op->show(seq, p);
 134         if (err > 0) {
 135                 /* object is skipped, decrease seq_num, so next
 136                  * valid object can reuse the same seq_num.
 137                  */
 138                 bpf_iter_dec_seq_num(seq);
 139                 seq->count = 0;
 140         } else if (err < 0 || seq_has_overflowed(seq)) {
 141                 if (!err)
 142                         err = -E2BIG;
 143                 seq->op->stop(seq, p);
 144                 seq->count = 0;
 145                 goto done;
 146         }
 147
 148         can_resched = bpf_iter_support_resched(seq);
 149         while (1) {
 150                 loff_t pos = seq->index;
 151
 152                 num_objs++;
 153                 offs = seq->count;
 154                 p = seq->op->next(seq, p, &seq->index);
 155                 if (pos == seq->index) {
 156                         pr_info_ratelimited("buggy seq_file .next function %ps "
 157                                 "did not updated position index\n",
 158                                 seq->op->next);
 159                         seq->index++;
 160                 }
 161
 162                 if (IS_ERR_OR_NULL(p))
 163                         break;
 164
 165                 /* got a valid next object, increase seq_num */
 166                 bpf_iter_inc_seq_num(seq);
 167
 168                 if (seq->count >= size)
 169                         break;
 170
 171                 if (num_objs >= MAX_ITER_OBJECTS) {
 172                         if (offs == 0) {
 173                                 err = -EAGAIN;
 174                                 seq->op->stop(seq, p);
 175                                 goto done;
 176                         }
 177                         break;
 178                 }
 179
 180                 err = seq->op->show(seq, p);
 181                 if (err > 0) {
 182                         bpf_iter_dec_seq_num(seq);
 183                         seq->count = offs;
 184                 } else if (err < 0 || seq_has_overflowed(seq)) {
 185                         seq->count = offs;
 186                         if (offs == 0) {
 187                                 if (!err)
 188                                         err = -E2BIG;
 189                                 seq->op->stop(seq, p);
 190                                 goto done;
 191                         }
 192                         break;
 193                 }
 194
 195                 if (can_resched)
 196                         cond_resched();
 197         }
 198 stop:
 199         offs = seq->count;
 200         /* bpf program called if !p */
 201         seq->op->stop(seq, p);
 202         if (!p) {
 203                 if (!seq_has_overflowed(seq)) {
 204                         bpf_iter_done_stop(seq);
 205                 } else {
 206                         seq->count = offs;
 207                         if (offs == 0) {
 208                                 err = -E2BIG;
 209                                 goto done;
 210                         }
 211                 }
 212         }
 213
 214         n = min(seq->count, size);
 215         err = copy_to_user(buf, seq->buf, n);
 216         if (err) {
 217                 err = -EFAULT;
 218                 goto done;
 219         }
 220         copied = n;
 221         seq->count -= n;
 222         seq->from = n;
 223 done:
 224         if (!copied)
 225                 copied = err;
 226         else
 227                 *ppos += copied;
 228         mutex_unlock(&seq->lock);
 229         return copied;
 230 }
 231
 232 static const struct bpf_iter_seq_info *
 233 __get_seq_info(struct bpf_iter_link *link)
 234 {
 235         const struct bpf_iter_seq_info *seq_info;
 236
 237         if (link->aux.map) {
 238                 seq_info = link->aux.map->ops->iter_seq_info;
 239                 if (seq_info)
 240                         return seq_info;
 241         }
 242
 243         return link->tinfo->reg_info->seq_info;
 244 }
 245
 246 static int iter_open(struct inode *inode, struct file *file)
 247 {
 248         struct bpf_iter_link *link = inode->i_private;
 249
 250         return prepare_seq_file(file, link, __get_seq_info(link));
 251 }
 252
 253 static int iter_release(struct inode *inode, struct file *file)
 254 {
 255         struct bpf_iter_priv_data *iter_priv;
 256         struct seq_file *seq;
 257
 258         seq = file->private_data;
 259         if (!seq)
 260                 return 0;
 261
 262         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
 263                                  target_private);
 264
 265         if (iter_priv->seq_info->fini_seq_private)
 266                 iter_priv->seq_info->fini_seq_private(seq->private);
 267
 268         bpf_prog_put(iter_priv->prog);
 269         seq->private = iter_priv;
 270
 271         return seq_release_private(inode, file);
 272 }
 273
 274 const struct file_operations bpf_iter_fops = {
 275         .open           = iter_open,
 276         .llseek         = no_llseek,
 277         .read           = bpf_seq_read,
 278         .release        = iter_release,
 279 };
 280
 281 /* The argument reg_info will be cached in bpf_iter_target_info.
 282  * The common practice is to declare target reg_info as
 283  * a const static variable and passed as an argument to
 284  * bpf_iter_reg_target().
 285  */
 286 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
 287 {
 288         struct bpf_iter_target_info *tinfo;
 289
 290         tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
 291         if (!tinfo)
 292                 return -ENOMEM;
 293
 294         tinfo->reg_info = reg_info;
 295         INIT_LIST_HEAD(&tinfo->list);
 296
 297         mutex_lock(&targets_mutex);
 298         list_add(&tinfo->list, &targets);
 299         mutex_unlock(&targets_mutex);
 300
 301         return 0;
 302 }
 303
 304 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
 305 {
 306         struct bpf_iter_target_info *tinfo;
 307         bool found = false;
 308
 309         mutex_lock(&targets_mutex);
 310         list_for_each_entry(tinfo, &targets, list) {
 311                 if (reg_info == tinfo->reg_info) {
 312                         list_del(&tinfo->list);
 313                         kfree(tinfo);
 314                         found = true;
 315                         break;
 316                 }
 317         }
 318         mutex_unlock(&targets_mutex);
 319
 320         WARN_ON(found == false);
 321 }
 322
 323 static void cache_btf_id(struct bpf_iter_target_info *tinfo,
 324                          struct bpf_prog *prog)
 325 {
 326         tinfo->btf_id = prog->aux->attach_btf_id;
 327 }
 328
 329 bool bpf_iter_prog_supported(struct bpf_prog *prog)
 330 {
 331         const char *attach_fname = prog->aux->attach_func_name;
 332         u32 prog_btf_id = prog->aux->attach_btf_id;
 333         const char *prefix = BPF_ITER_FUNC_PREFIX;
 334         struct bpf_iter_target_info *tinfo;
 335         int prefix_len = strlen(prefix);
 336         bool supported = false;
 337
 338         if (strncmp(attach_fname, prefix, prefix_len))
 339                 return false;
 340
 341         mutex_lock(&targets_mutex);
 342         list_for_each_entry(tinfo, &targets, list) {
 343                 if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
 344                         supported = true;
 345                         break;
 346                 }
 347                 if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
 348                         cache_btf_id(tinfo, prog);
 349                         supported = true;
 350                         break;
 351                 }
 352         }
 353         mutex_unlock(&targets_mutex);
 354
 355         if (supported) {
 356                 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
 357                 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
 358         }
 359
 360         return supported;
 361 }
 362
 363 static void bpf_iter_link_release(struct bpf_link *link)
 364 {
 365         struct bpf_iter_link *iter_link =
 366                 container_of(link, struct bpf_iter_link, link);
 367
 368         if (iter_link->tinfo->reg_info->detach_target)
 369                 iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
 370 }
 371
 372 static void bpf_iter_link_dealloc(struct bpf_link *link)
 373 {
 374         struct bpf_iter_link *iter_link =
 375                 container_of(link, struct bpf_iter_link, link);
 376
 377         kfree(iter_link);
 378 }
 379
 380 static int bpf_iter_link_replace(struct bpf_link *link,
 381                                  struct bpf_prog *new_prog,
 382                                  struct bpf_prog *old_prog)
 383 {
 384         int ret = 0;
 385
 386         mutex_lock(&link_mutex);
 387         if (old_prog && link->prog != old_prog) {
 388                 ret = -EPERM;
 389                 goto out_unlock;
 390         }
 391
 392         if (link->prog->type != new_prog->type ||
 393             link->prog->expected_attach_type != new_prog->expected_attach_type ||
 394             link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
 395                 ret = -EINVAL;
 396                 goto out_unlock;
 397         }
 398
 399         old_prog = xchg(&link->prog, new_prog);
 400         bpf_prog_put(old_prog);
 401
 402 out_unlock:
 403         mutex_unlock(&link_mutex);
 404         return ret;
 405 }
 406
 407 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
 408                                       struct seq_file *seq)
 409 {
 410         struct bpf_iter_link *iter_link =
 411                 container_of(link, struct bpf_iter_link, link);
 412         bpf_iter_show_fdinfo_t show_fdinfo;
 413
 414         seq_printf(seq,
 415                    "target_name:\t%s\n",
 416                    iter_link->tinfo->reg_info->target);
 417
 418         show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
 419         if (show_fdinfo)
 420                 show_fdinfo(&iter_link->aux, seq);
 421 }
 422
 423 static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
 424                                         struct bpf_link_info *info)
 425 {
 426         struct bpf_iter_link *iter_link =
 427                 container_of(link, struct bpf_iter_link, link);
 428         char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
 429         bpf_iter_fill_link_info_t fill_link_info;
 430         u32 ulen = info->iter.target_name_len;
 431         const char *target_name;
 432         u32 target_len;
 433
 434         if (!ulen ^ !ubuf)
 435                 return -EINVAL;
 436
 437         target_name = iter_link->tinfo->reg_info->target;
 438         target_len =  strlen(target_name);
 439         info->iter.target_name_len = target_len + 1;
 440
 441         if (ubuf) {
 442                 if (ulen >= target_len + 1) {
 443                         if (copy_to_user(ubuf, target_name, target_len + 1))
 444                                 return -EFAULT;
 445                 } else {
 446                         char zero = '\0';
 447
 448                         if (copy_to_user(ubuf, target_name, ulen - 1))
 449                                 return -EFAULT;
 450                         if (put_user(zero, ubuf + ulen - 1))
 451                                 return -EFAULT;
 452                         return -ENOSPC;
 453                 }
 454         }
 455
 456         fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
 457         if (fill_link_info)
 458                 return fill_link_info(&iter_link->aux, info);
 459
 460         return 0;
 461 }
 462
 463 static const struct bpf_link_ops bpf_iter_link_lops = {
 464         .release = bpf_iter_link_release,
 465         .dealloc = bpf_iter_link_dealloc,
 466         .update_prog = bpf_iter_link_replace,
 467         .show_fdinfo = bpf_iter_link_show_fdinfo,
 468         .fill_link_info = bpf_iter_link_fill_link_info,
 469 };
 470
 471 bool bpf_link_is_iter(struct bpf_link *link)
 472 {
 473         return link->ops == &bpf_iter_link_lops;
 474 }
 475
 476 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
 477                          struct bpf_prog *prog)
 478 {
 479         struct bpf_link_primer link_primer;
 480         struct bpf_iter_target_info *tinfo;
 481         union bpf_iter_link_info linfo;
 482         struct bpf_iter_link *link;
 483         u32 prog_btf_id, linfo_len;
 484         bool existed = false;
 485         bpfptr_t ulinfo;
 486         int err;
 487
 488         if (attr->link_create.target_fd || attr->link_create.flags)
 489                 return -EINVAL;
 490
 491         memset(&linfo, 0, sizeof(union bpf_iter_link_info));
 492
 493         ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
 494         linfo_len = attr->link_create.iter_info_len;
 495         if (bpfptr_is_null(ulinfo) ^ !linfo_len)
 496                 return -EINVAL;
 497
 498         if (!bpfptr_is_null(ulinfo)) {
 499                 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
 500                                                linfo_len);
 501                 if (err)
 502                         return err;
 503                 linfo_len = min_t(u32, linfo_len, sizeof(linfo));
 504                 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
 505                         return -EFAULT;
 506         }
 507
 508         prog_btf_id = prog->aux->attach_btf_id;
 509         mutex_lock(&targets_mutex);
 510         list_for_each_entry(tinfo, &targets, list) {
 511                 if (tinfo->btf_id == prog_btf_id) {
 512                         existed = true;
 513                         break;
 514                 }
 515         }
 516         mutex_unlock(&targets_mutex);
 517         if (!existed)
 518                 return -ENOENT;
 519
 520         link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
 521         if (!link)
 522                 return -ENOMEM;
 523
 524         bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
 525         link->tinfo = tinfo;
 526
 527         err  = bpf_link_prime(&link->link, &link_primer);
 528         if (err) {
 529                 kfree(link);
 530                 return err;
 531         }
 532
 533         if (tinfo->reg_info->attach_target) {
 534                 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
 535                 if (err) {
 536                         bpf_link_cleanup(&link_primer);
 537                         return err;
 538                 }
 539         }
 540
 541         return bpf_link_settle(&link_primer);
 542 }
 543
 544 static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
 545                           struct bpf_iter_target_info *tinfo,
 546                           const struct bpf_iter_seq_info *seq_info,
 547                           struct bpf_prog *prog)
 548 {
 549         priv_data->tinfo = tinfo;
 550         priv_data->seq_info = seq_info;
 551         priv_data->prog = prog;
 552         priv_data->session_id = atomic64_inc_return(&session_id);
 553         priv_data->seq_num = 0;
 554         priv_data->done_stop = false;
 555 }
 556
 557 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
 558                             const struct bpf_iter_seq_info *seq_info)
 559 {
 560         struct bpf_iter_priv_data *priv_data;
 561         struct bpf_iter_target_info *tinfo;
 562         struct bpf_prog *prog;
 563         u32 total_priv_dsize;
 564         struct seq_file *seq;
 565         int err = 0;
 566
 567         mutex_lock(&link_mutex);
 568         prog = link->link.prog;
 569         bpf_prog_inc(prog);
 570         mutex_unlock(&link_mutex);
 571
 572         tinfo = link->tinfo;
 573         total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
 574                            seq_info->seq_priv_size;
 575         priv_data = __seq_open_private(file, seq_info->seq_ops,
 576                                        total_priv_dsize);
 577         if (!priv_data) {
 578                 err = -ENOMEM;
 579                 goto release_prog;
 580         }
 581
 582         if (seq_info->init_seq_private) {
 583                 err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
 584                 if (err)
 585                         goto release_seq_file;
 586         }
 587
 588         init_seq_meta(priv_data, tinfo, seq_info, prog);
 589         seq = file->private_data;
 590         seq->private = priv_data->target_private;
 591
 592         return 0;
 593
 594 release_seq_file:
 595         seq_release_private(file->f_inode, file);
 596         file->private_data = NULL;
 597 release_prog:
 598         bpf_prog_put(prog);
 599         return err;
 600 }
 601
 602 int bpf_iter_new_fd(struct bpf_link *link)
 603 {
 604         struct bpf_iter_link *iter_link;
 605         struct file *file;
 606         unsigned int flags;
 607         int err, fd;
 608
 609         if (link->ops != &bpf_iter_link_lops)
 610                 return -EINVAL;
 611
 612         flags = O_RDONLY | O_CLOEXEC;
 613         fd = get_unused_fd_flags(flags);
 614         if (fd < 0)
 615                 return fd;
 616
 617         file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
 618         if (IS_ERR(file)) {
 619                 err = PTR_ERR(file);
 620                 goto free_fd;
 621         }
 622
 623         iter_link = container_of(link, struct bpf_iter_link, link);
 624         err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
 625         if (err)
 626                 goto free_file;
 627
 628         fd_install(fd, file);
 629         return fd;
 630
 631 free_file:
 632         fput(file);
 633 free_fd:
 634         put_unused_fd(fd);
 635         return err;
 636 }
 637
 638 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
 639 {
 640         struct bpf_iter_priv_data *iter_priv;
 641         struct seq_file *seq;
 642         void *seq_priv;
 643
 644         seq = meta->seq;
 645         if (seq->file->f_op != &bpf_iter_fops)
 646                 return NULL;
 647
 648         seq_priv = seq->private;
 649         iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
 650                                  target_private);
 651
 652         if (in_stop && iter_priv->done_stop)
 653                 return NULL;
 654
 655         meta->session_id = iter_priv->session_id;
 656         meta->seq_num = iter_priv->seq_num;
 657
 658         return iter_priv->prog;
 659 }
 660
 661 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 662 {
 663         int ret;
 664
 665         rcu_read_lock();
 666         migrate_disable();
 667         ret = BPF_PROG_RUN(prog, ctx);
 668         migrate_enable();
 669         rcu_read_unlock();
 670
 671         /* bpf program can only return 0 or 1:
 672          *  0 : okay
 673          *  1 : retry the same object
 674          * The bpf_iter_run_prog() return value
 675          * will be seq_ops->show() return value.
 676          */
 677         return ret == 0 ? 0 : -EAGAIN;
 678 }
 679
 680 BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
 681            void *, callback_ctx, u64, flags)
 682 {
 683         return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
 684 }
 685
 686 const struct bpf_func_proto bpf_for_each_map_elem_proto = {
 687         .func           = bpf_for_each_map_elem,
 688         .gpl_only       = false,
 689         .ret_type       = RET_INTEGER,
 690         .arg1_type      = ARG_CONST_MAP_PTR,
 691         .arg2_type      = ARG_PTR_TO_FUNC,
 692         .arg3_type      = ARG_PTR_TO_STACK_OR_NULL,
 693         .arg4_type      = ARG_ANYTHING,
 694 };