kernel/bpf/bpf_iter.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2020 Facebook */
   3
   4 #include <linux/fs.h>
   5 #include <linux/anon_inodes.h>
   6 #include <linux/filter.h>
   7 #include <linux/bpf.h>
   8
   9 struct bpf_iter_target_info {
  10         struct list_head list;
  11         const struct bpf_iter_reg *reg_info;
  12         u32 btf_id;     /* cached value */
  13 };
  14
  15 struct bpf_iter_link {
  16         struct bpf_link link;
  17         struct bpf_iter_aux_info aux;
  18         struct bpf_iter_target_info *tinfo;
  19 };
  20
  21 struct bpf_iter_priv_data {
  22         struct bpf_iter_target_info *tinfo;
  23         const struct bpf_iter_seq_info *seq_info;
  24         struct bpf_prog *prog;
  25         u64 session_id;
  26         u64 seq_num;
  27         bool done_stop;
  28         u8 target_private[] __aligned(8);
  29 };
  30
  31 static struct list_head targets = LIST_HEAD_INIT(targets);
  32 static DEFINE_MUTEX(targets_mutex);
  33
  34 /* protect bpf_iter_link changes */
  35 static DEFINE_MUTEX(link_mutex);
  36
  37 /* incremented on every opened seq_file */
  38 static atomic64_t session_id;
  39
  40 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
  41                             const struct bpf_iter_seq_info *seq_info);
  42
  43 static void bpf_iter_inc_seq_num(struct seq_file *seq)
  44 {
  45         struct bpf_iter_priv_data *iter_priv;
  46
  47         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  48                                  target_private);
  49         iter_priv->seq_num++;
  50 }
  51
  52 static void bpf_iter_dec_seq_num(struct seq_file *seq)
  53 {
  54         struct bpf_iter_priv_data *iter_priv;
  55
  56         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  57                                  target_private);
  58         iter_priv->seq_num--;
  59 }
  60
  61 static void bpf_iter_done_stop(struct seq_file *seq)
  62 {
  63         struct bpf_iter_priv_data *iter_priv;
  64
  65         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  66                                  target_private);
  67         iter_priv->done_stop = true;
  68 }
  69
  70 /* maximum visited objects before bailing out */
  71 #define MAX_ITER_OBJECTS        1000000
  72
  73 /* bpf_seq_read, a customized and simpler version for bpf iterator.
  74  * no_llseek is assumed for this file.
  75  * The following are differences from seq_read():
  76  *  . fixed buffer size (PAGE_SIZE)
  77  *  . assuming no_llseek
  78  *  . stop() may call bpf program, handling potential overflow there
  79  */
  80 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
  81                             loff_t *ppos)
  82 {
  83         struct seq_file *seq = file->private_data;
  84         size_t n, offs, copied = 0;
  85         int err = 0, num_objs = 0;
  86         void *p;
  87
  88         mutex_lock(&seq->lock);
  89
  90         if (!seq->buf) {
  91                 seq->size = PAGE_SIZE << 3;
  92                 seq->buf = kvmalloc(seq->size, GFP_KERNEL);
  93                 if (!seq->buf) {
  94                         err = -ENOMEM;
  95                         goto done;
  96                 }
  97         }
  98
  99         if (seq->count) {
 100                 n = min(seq->count, size);
 101                 err = copy_to_user(buf, seq->buf + seq->from, n);
 102                 if (err) {
 103                         err = -EFAULT;
 104                         goto done;
 105                 }
 106                 seq->count -= n;
 107                 seq->from += n;
 108                 copied = n;
 109                 goto done;
 110         }
 111
 112         seq->from = 0;
 113         p = seq->op->start(seq, &seq->index);
 114         if (!p)
 115                 goto stop;
 116         if (IS_ERR(p)) {
 117                 err = PTR_ERR(p);
 118                 seq->op->stop(seq, p);
 119                 seq->count = 0;
 120                 goto done;
 121         }
 122
 123         err = seq->op->show(seq, p);
 124         if (err > 0) {
 125                 /* object is skipped, decrease seq_num, so next
 126                  * valid object can reuse the same seq_num.
 127                  */
 128                 bpf_iter_dec_seq_num(seq);
 129                 seq->count = 0;
 130         } else if (err < 0 || seq_has_overflowed(seq)) {
 131                 if (!err)
 132                         err = -E2BIG;
 133                 seq->op->stop(seq, p);
 134                 seq->count = 0;
 135                 goto done;
 136         }
 137
 138         while (1) {
 139                 loff_t pos = seq->index;
 140
 141                 num_objs++;
 142                 offs = seq->count;
 143                 p = seq->op->next(seq, p, &seq->index);
 144                 if (pos == seq->index) {
 145                         pr_info_ratelimited("buggy seq_file .next function %ps "
 146                                 "did not updated position index\n",
 147                                 seq->op->next);
 148                         seq->index++;
 149                 }
 150
 151                 if (IS_ERR_OR_NULL(p))
 152                         break;
 153
 154                 /* got a valid next object, increase seq_num */
 155                 bpf_iter_inc_seq_num(seq);
 156
 157                 if (seq->count >= size)
 158                         break;
 159
 160                 if (num_objs >= MAX_ITER_OBJECTS) {
 161                         if (offs == 0) {
 162                                 err = -EAGAIN;
 163                                 seq->op->stop(seq, p);
 164                                 goto done;
 165                         }
 166                         break;
 167                 }
 168
 169                 err = seq->op->show(seq, p);
 170                 if (err > 0) {
 171                         bpf_iter_dec_seq_num(seq);
 172                         seq->count = offs;
 173                 } else if (err < 0 || seq_has_overflowed(seq)) {
 174                         seq->count = offs;
 175                         if (offs == 0) {
 176                                 if (!err)
 177                                         err = -E2BIG;
 178                                 seq->op->stop(seq, p);
 179                                 goto done;
 180                         }
 181                         break;
 182                 }
 183         }
 184 stop:
 185         offs = seq->count;
 186         /* bpf program called if !p */
 187         seq->op->stop(seq, p);
 188         if (!p) {
 189                 if (!seq_has_overflowed(seq)) {
 190                         bpf_iter_done_stop(seq);
 191                 } else {
 192                         seq->count = offs;
 193                         if (offs == 0) {
 194                                 err = -E2BIG;
 195                                 goto done;
 196                         }
 197                 }
 198         }
 199
 200         n = min(seq->count, size);
 201         err = copy_to_user(buf, seq->buf, n);
 202         if (err) {
 203                 err = -EFAULT;
 204                 goto done;
 205         }
 206         copied = n;
 207         seq->count -= n;
 208         seq->from = n;
 209 done:
 210         if (!copied)
 211                 copied = err;
 212         else
 213                 *ppos += copied;
 214         mutex_unlock(&seq->lock);
 215         return copied;
 216 }
 217
 218 static const struct bpf_iter_seq_info *
 219 __get_seq_info(struct bpf_iter_link *link)
 220 {
 221         const struct bpf_iter_seq_info *seq_info;
 222
 223         if (link->aux.map) {
 224                 seq_info = link->aux.map->ops->iter_seq_info;
 225                 if (seq_info)
 226                         return seq_info;
 227         }
 228
 229         return link->tinfo->reg_info->seq_info;
 230 }
 231
 232 static int iter_open(struct inode *inode, struct file *file)
 233 {
 234         struct bpf_iter_link *link = inode->i_private;
 235
 236         return prepare_seq_file(file, link, __get_seq_info(link));
 237 }
 238
 239 static int iter_release(struct inode *inode, struct file *file)
 240 {
 241         struct bpf_iter_priv_data *iter_priv;
 242         struct seq_file *seq;
 243
 244         seq = file->private_data;
 245         if (!seq)
 246                 return 0;
 247
 248         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
 249                                  target_private);
 250
 251         if (iter_priv->seq_info->fini_seq_private)
 252                 iter_priv->seq_info->fini_seq_private(seq->private);
 253
 254         bpf_prog_put(iter_priv->prog);
 255         seq->private = iter_priv;
 256
 257         return seq_release_private(inode, file);
 258 }
 259
 260 const struct file_operations bpf_iter_fops = {
 261         .open           = iter_open,
 262         .llseek         = no_llseek,
 263         .read           = bpf_seq_read,
 264         .release        = iter_release,
 265 };
 266
 267 /* The argument reg_info will be cached in bpf_iter_target_info.
 268  * The common practice is to declare target reg_info as
 269  * a const static variable and passed as an argument to
 270  * bpf_iter_reg_target().
 271  */
 272 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
 273 {
 274         struct bpf_iter_target_info *tinfo;
 275
 276         tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);
 277         if (!tinfo)
 278                 return -ENOMEM;
 279
 280         tinfo->reg_info = reg_info;
 281         INIT_LIST_HEAD(&tinfo->list);
 282
 283         mutex_lock(&targets_mutex);
 284         list_add(&tinfo->list, &targets);
 285         mutex_unlock(&targets_mutex);
 286
 287         return 0;
 288 }
 289
 290 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
 291 {
 292         struct bpf_iter_target_info *tinfo;
 293         bool found = false;
 294
 295         mutex_lock(&targets_mutex);
 296         list_for_each_entry(tinfo, &targets, list) {
 297                 if (reg_info == tinfo->reg_info) {
 298                         list_del(&tinfo->list);
 299                         kfree(tinfo);
 300                         found = true;
 301                         break;
 302                 }
 303         }
 304         mutex_unlock(&targets_mutex);
 305
 306         WARN_ON(found == false);
 307 }
 308
 309 static void cache_btf_id(struct bpf_iter_target_info *tinfo,
 310                          struct bpf_prog *prog)
 311 {
 312         tinfo->btf_id = prog->aux->attach_btf_id;
 313 }
 314
 315 bool bpf_iter_prog_supported(struct bpf_prog *prog)
 316 {
 317         const char *attach_fname = prog->aux->attach_func_name;
 318         u32 prog_btf_id = prog->aux->attach_btf_id;
 319         const char *prefix = BPF_ITER_FUNC_PREFIX;
 320         struct bpf_iter_target_info *tinfo;
 321         int prefix_len = strlen(prefix);
 322         bool supported = false;
 323
 324         if (strncmp(attach_fname, prefix, prefix_len))
 325                 return false;
 326
 327         mutex_lock(&targets_mutex);
 328         list_for_each_entry(tinfo, &targets, list) {
 329                 if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
 330                         supported = true;
 331                         break;
 332                 }
 333                 if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
 334                         cache_btf_id(tinfo, prog);
 335                         supported = true;
 336                         break;
 337                 }
 338         }
 339         mutex_unlock(&targets_mutex);
 340
 341         if (supported) {
 342                 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
 343                 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
 344         }
 345
 346         return supported;
 347 }
 348
 349 static void bpf_iter_link_release(struct bpf_link *link)
 350 {
 351         struct bpf_iter_link *iter_link =
 352                 container_of(link, struct bpf_iter_link, link);
 353
 354         if (iter_link->tinfo->reg_info->detach_target)
 355                 iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
 356 }
 357
 358 static void bpf_iter_link_dealloc(struct bpf_link *link)
 359 {
 360         struct bpf_iter_link *iter_link =
 361                 container_of(link, struct bpf_iter_link, link);
 362
 363         kfree(iter_link);
 364 }
 365
 366 static int bpf_iter_link_replace(struct bpf_link *link,
 367                                  struct bpf_prog *new_prog,
 368                                  struct bpf_prog *old_prog)
 369 {
 370         int ret = 0;
 371
 372         mutex_lock(&link_mutex);
 373         if (old_prog && link->prog != old_prog) {
 374                 ret = -EPERM;
 375                 goto out_unlock;
 376         }
 377
 378         if (link->prog->type != new_prog->type ||
 379             link->prog->expected_attach_type != new_prog->expected_attach_type ||
 380             link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
 381                 ret = -EINVAL;
 382                 goto out_unlock;
 383         }
 384
 385         old_prog = xchg(&link->prog, new_prog);
 386         bpf_prog_put(old_prog);
 387
 388 out_unlock:
 389         mutex_unlock(&link_mutex);
 390         return ret;
 391 }
 392
 393 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
 394                                       struct seq_file *seq)
 395 {
 396         struct bpf_iter_link *iter_link =
 397                 container_of(link, struct bpf_iter_link, link);
 398         bpf_iter_show_fdinfo_t show_fdinfo;
 399
 400         seq_printf(seq,
 401                    "target_name:\t%s\n",
 402                    iter_link->tinfo->reg_info->target);
 403
 404         show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
 405         if (show_fdinfo)
 406                 show_fdinfo(&iter_link->aux, seq);
 407 }
 408
 409 static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
 410                                         struct bpf_link_info *info)
 411 {
 412         struct bpf_iter_link *iter_link =
 413                 container_of(link, struct bpf_iter_link, link);
 414         char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
 415         bpf_iter_fill_link_info_t fill_link_info;
 416         u32 ulen = info->iter.target_name_len;
 417         const char *target_name;
 418         u32 target_len;
 419
 420         if (!ulen ^ !ubuf)
 421                 return -EINVAL;
 422
 423         target_name = iter_link->tinfo->reg_info->target;
 424         target_len =  strlen(target_name);
 425         info->iter.target_name_len = target_len + 1;
 426
 427         if (ubuf) {
 428                 if (ulen >= target_len + 1) {
 429                         if (copy_to_user(ubuf, target_name, target_len + 1))
 430                                 return -EFAULT;
 431                 } else {
 432                         char zero = '\0';
 433
 434                         if (copy_to_user(ubuf, target_name, ulen - 1))
 435                                 return -EFAULT;
 436                         if (put_user(zero, ubuf + ulen - 1))
 437                                 return -EFAULT;
 438                         return -ENOSPC;
 439                 }
 440         }
 441
 442         fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
 443         if (fill_link_info)
 444                 return fill_link_info(&iter_link->aux, info);
 445
 446         return 0;
 447 }
 448
 449 static const struct bpf_link_ops bpf_iter_link_lops = {
 450         .release = bpf_iter_link_release,
 451         .dealloc = bpf_iter_link_dealloc,
 452         .update_prog = bpf_iter_link_replace,
 453         .show_fdinfo = bpf_iter_link_show_fdinfo,
 454         .fill_link_info = bpf_iter_link_fill_link_info,
 455 };
 456
 457 bool bpf_link_is_iter(struct bpf_link *link)
 458 {
 459         return link->ops == &bpf_iter_link_lops;
 460 }
 461
 462 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
 463 {
 464         union bpf_iter_link_info __user *ulinfo;
 465         struct bpf_link_primer link_primer;
 466         struct bpf_iter_target_info *tinfo;
 467         union bpf_iter_link_info linfo;
 468         struct bpf_iter_link *link;
 469         u32 prog_btf_id, linfo_len;
 470         bool existed = false;
 471         int err;
 472
 473         if (attr->link_create.target_fd || attr->link_create.flags)
 474                 return -EINVAL;
 475
 476         memset(&linfo, 0, sizeof(union bpf_iter_link_info));
 477
 478         ulinfo = u64_to_user_ptr(attr->link_create.iter_info);
 479         linfo_len = attr->link_create.iter_info_len;
 480         if (!ulinfo ^ !linfo_len)
 481                 return -EINVAL;
 482
 483         if (ulinfo) {
 484                 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
 485                                                linfo_len);
 486                 if (err)
 487                         return err;
 488                 linfo_len = min_t(u32, linfo_len, sizeof(linfo));
 489                 if (copy_from_user(&linfo, ulinfo, linfo_len))
 490                         return -EFAULT;
 491         }
 492
 493         prog_btf_id = prog->aux->attach_btf_id;
 494         mutex_lock(&targets_mutex);
 495         list_for_each_entry(tinfo, &targets, list) {
 496                 if (tinfo->btf_id == prog_btf_id) {
 497                         existed = true;
 498                         break;
 499                 }
 500         }
 501         mutex_unlock(&targets_mutex);
 502         if (!existed)
 503                 return -ENOENT;
 504
 505         link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
 506         if (!link)
 507                 return -ENOMEM;
 508
 509         bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
 510         link->tinfo = tinfo;
 511
 512         err  = bpf_link_prime(&link->link, &link_primer);
 513         if (err) {
 514                 kfree(link);
 515                 return err;
 516         }
 517
 518         if (tinfo->reg_info->attach_target) {
 519                 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
 520                 if (err) {
 521                         bpf_link_cleanup(&link_primer);
 522                         return err;
 523                 }
 524         }
 525
 526         return bpf_link_settle(&link_primer);
 527 }
 528
 529 static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
 530                           struct bpf_iter_target_info *tinfo,
 531                           const struct bpf_iter_seq_info *seq_info,
 532                           struct bpf_prog *prog)
 533 {
 534         priv_data->tinfo = tinfo;
 535         priv_data->seq_info = seq_info;
 536         priv_data->prog = prog;
 537         priv_data->session_id = atomic64_inc_return(&session_id);
 538         priv_data->seq_num = 0;
 539         priv_data->done_stop = false;
 540 }
 541
 542 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
 543                             const struct bpf_iter_seq_info *seq_info)
 544 {
 545         struct bpf_iter_priv_data *priv_data;
 546         struct bpf_iter_target_info *tinfo;
 547         struct bpf_prog *prog;
 548         u32 total_priv_dsize;
 549         struct seq_file *seq;
 550         int err = 0;
 551
 552         mutex_lock(&link_mutex);
 553         prog = link->link.prog;
 554         bpf_prog_inc(prog);
 555         mutex_unlock(&link_mutex);
 556
 557         tinfo = link->tinfo;
 558         total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
 559                            seq_info->seq_priv_size;
 560         priv_data = __seq_open_private(file, seq_info->seq_ops,
 561                                        total_priv_dsize);
 562         if (!priv_data) {
 563                 err = -ENOMEM;
 564                 goto release_prog;
 565         }
 566
 567         if (seq_info->init_seq_private) {
 568                 err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
 569                 if (err)
 570                         goto release_seq_file;
 571         }
 572
 573         init_seq_meta(priv_data, tinfo, seq_info, prog);
 574         seq = file->private_data;
 575         seq->private = priv_data->target_private;
 576
 577         return 0;
 578
 579 release_seq_file:
 580         seq_release_private(file->f_inode, file);
 581         file->private_data = NULL;
 582 release_prog:
 583         bpf_prog_put(prog);
 584         return err;
 585 }
 586
 587 int bpf_iter_new_fd(struct bpf_link *link)
 588 {
 589         struct bpf_iter_link *iter_link;
 590         struct file *file;
 591         unsigned int flags;
 592         int err, fd;
 593
 594         if (link->ops != &bpf_iter_link_lops)
 595                 return -EINVAL;
 596
 597         flags = O_RDONLY | O_CLOEXEC;
 598         fd = get_unused_fd_flags(flags);
 599         if (fd < 0)
 600                 return fd;
 601
 602         file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
 603         if (IS_ERR(file)) {
 604                 err = PTR_ERR(file);
 605                 goto free_fd;
 606         }
 607
 608         iter_link = container_of(link, struct bpf_iter_link, link);
 609         err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
 610         if (err)
 611                 goto free_file;
 612
 613         fd_install(fd, file);
 614         return fd;
 615
 616 free_file:
 617         fput(file);
 618 free_fd:
 619         put_unused_fd(fd);
 620         return err;
 621 }
 622
 623 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
 624 {
 625         struct bpf_iter_priv_data *iter_priv;
 626         struct seq_file *seq;
 627         void *seq_priv;
 628
 629         seq = meta->seq;
 630         if (seq->file->f_op != &bpf_iter_fops)
 631                 return NULL;
 632
 633         seq_priv = seq->private;
 634         iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
 635                                  target_private);
 636
 637         if (in_stop && iter_priv->done_stop)
 638                 return NULL;
 639
 640         meta->session_id = iter_priv->session_id;
 641         meta->seq_num = iter_priv->seq_num;
 642
 643         return iter_priv->prog;
 644 }
 645
 646 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 647 {
 648         int ret;
 649
 650         rcu_read_lock();
 651         migrate_disable();
 652         ret = BPF_PROG_RUN(prog, ctx);
 653         migrate_enable();
 654         rcu_read_unlock();
 655
 656         /* bpf program can only return 0 or 1:
 657          *  0 : okay
 658          *  1 : retry the same object
 659          * The bpf_iter_run_prog() return value
 660          * will be seq_ops->show() return value.
 661          */
 662         return ret == 0 ? 0 : -EAGAIN;
 663 }