1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2020 Facebook */
5 #include <linux/anon_inodes.h>
6 #include <linux/filter.h>
9 struct bpf_iter_target_info {
10 struct list_head list;
11 const struct bpf_iter_reg *reg_info;
12 u32 btf_id; /* cached value */
15 struct bpf_iter_link {
17 struct bpf_iter_aux_info aux;
18 struct bpf_iter_target_info *tinfo;
21 struct bpf_iter_priv_data {
22 struct bpf_iter_target_info *tinfo;
23 const struct bpf_iter_seq_info *seq_info;
24 struct bpf_prog *prog;
28 u8 target_private[] __aligned(8);
31 static struct list_head targets = LIST_HEAD_INIT(targets);
32 static DEFINE_MUTEX(targets_mutex);
34 /* protect bpf_iter_link changes */
35 static DEFINE_MUTEX(link_mutex);
37 /* incremented on every opened seq_file */
38 static atomic64_t session_id;
40 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
41 const struct bpf_iter_seq_info *seq_info);
43 static void bpf_iter_inc_seq_num(struct seq_file *seq)
45 struct bpf_iter_priv_data *iter_priv;
47 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
52 static void bpf_iter_dec_seq_num(struct seq_file *seq)
54 struct bpf_iter_priv_data *iter_priv;
56 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
61 static void bpf_iter_done_stop(struct seq_file *seq)
63 struct bpf_iter_priv_data *iter_priv;
65 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
67 iter_priv->done_stop = true;
70 /* bpf_seq_read, a customized and simpler version for bpf iterator.
71 * no_llseek is assumed for this file.
72 * The following are differences from seq_read():
73 * . fixed buffer size (PAGE_SIZE)
74 * . assuming no_llseek
75 * . stop() may call bpf program, handling potential overflow there
77 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
80 struct seq_file *seq = file->private_data;
81 size_t n, offs, copied = 0;
85 mutex_lock(&seq->lock);
88 seq->size = PAGE_SIZE;
89 seq->buf = kmalloc(seq->size, GFP_KERNEL);
97 n = min(seq->count, size);
98 err = copy_to_user(buf, seq->buf + seq->from, n);
110 p = seq->op->start(seq, &seq->index);
115 seq->op->stop(seq, p);
120 err = seq->op->show(seq, p);
122 /* object is skipped, decrease seq_num, so next
123 * valid object can reuse the same seq_num.
125 bpf_iter_dec_seq_num(seq);
127 } else if (err < 0 || seq_has_overflowed(seq)) {
130 seq->op->stop(seq, p);
136 loff_t pos = seq->index;
139 p = seq->op->next(seq, p, &seq->index);
140 if (pos == seq->index) {
141 pr_info_ratelimited("buggy seq_file .next function %ps "
142 "did not updated position index\n",
147 if (IS_ERR_OR_NULL(p))
150 /* got a valid next object, increase seq_num */
151 bpf_iter_inc_seq_num(seq);
153 if (seq->count >= size)
156 err = seq->op->show(seq, p);
158 bpf_iter_dec_seq_num(seq);
160 } else if (err < 0 || seq_has_overflowed(seq)) {
165 seq->op->stop(seq, p);
173 /* bpf program called if !p */
174 seq->op->stop(seq, p);
176 if (!seq_has_overflowed(seq)) {
177 bpf_iter_done_stop(seq);
187 n = min(seq->count, size);
188 err = copy_to_user(buf, seq->buf, n);
201 mutex_unlock(&seq->lock);
205 static const struct bpf_iter_seq_info *
206 __get_seq_info(struct bpf_iter_link *link)
208 const struct bpf_iter_seq_info *seq_info;
211 seq_info = link->aux.map->ops->iter_seq_info;
216 return link->tinfo->reg_info->seq_info;
219 static int iter_open(struct inode *inode, struct file *file)
221 struct bpf_iter_link *link = inode->i_private;
223 return prepare_seq_file(file, link, __get_seq_info(link));
226 static int iter_release(struct inode *inode, struct file *file)
228 struct bpf_iter_priv_data *iter_priv;
229 struct seq_file *seq;
231 seq = file->private_data;
235 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
238 if (iter_priv->seq_info->fini_seq_private)
239 iter_priv->seq_info->fini_seq_private(seq->private);
241 bpf_prog_put(iter_priv->prog);
242 seq->private = iter_priv;
244 return seq_release_private(inode, file);
247 const struct file_operations bpf_iter_fops = {
250 .read = bpf_seq_read,
251 .release = iter_release,
254 /* The argument reg_info will be cached in bpf_iter_target_info.
255 * The common practice is to declare target reg_info as
256 * a const static variable and passed as an argument to
257 * bpf_iter_reg_target().
259 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
261 struct bpf_iter_target_info *tinfo;
263 tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);
267 tinfo->reg_info = reg_info;
268 INIT_LIST_HEAD(&tinfo->list);
270 mutex_lock(&targets_mutex);
271 list_add(&tinfo->list, &targets);
272 mutex_unlock(&targets_mutex);
277 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
279 struct bpf_iter_target_info *tinfo;
282 mutex_lock(&targets_mutex);
283 list_for_each_entry(tinfo, &targets, list) {
284 if (reg_info == tinfo->reg_info) {
285 list_del(&tinfo->list);
291 mutex_unlock(&targets_mutex);
293 WARN_ON(found == false);
296 static void cache_btf_id(struct bpf_iter_target_info *tinfo,
297 struct bpf_prog *prog)
299 tinfo->btf_id = prog->aux->attach_btf_id;
302 bool bpf_iter_prog_supported(struct bpf_prog *prog)
304 const char *attach_fname = prog->aux->attach_func_name;
305 u32 prog_btf_id = prog->aux->attach_btf_id;
306 const char *prefix = BPF_ITER_FUNC_PREFIX;
307 struct bpf_iter_target_info *tinfo;
308 int prefix_len = strlen(prefix);
309 bool supported = false;
311 if (strncmp(attach_fname, prefix, prefix_len))
314 mutex_lock(&targets_mutex);
315 list_for_each_entry(tinfo, &targets, list) {
316 if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
320 if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
321 cache_btf_id(tinfo, prog);
326 mutex_unlock(&targets_mutex);
329 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
330 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
336 static void bpf_iter_link_release(struct bpf_link *link)
338 struct bpf_iter_link *iter_link =
339 container_of(link, struct bpf_iter_link, link);
341 if (iter_link->tinfo->reg_info->detach_target)
342 iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
345 static void bpf_iter_link_dealloc(struct bpf_link *link)
347 struct bpf_iter_link *iter_link =
348 container_of(link, struct bpf_iter_link, link);
353 static int bpf_iter_link_replace(struct bpf_link *link,
354 struct bpf_prog *new_prog,
355 struct bpf_prog *old_prog)
359 mutex_lock(&link_mutex);
360 if (old_prog && link->prog != old_prog) {
365 if (link->prog->type != new_prog->type ||
366 link->prog->expected_attach_type != new_prog->expected_attach_type ||
367 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
372 old_prog = xchg(&link->prog, new_prog);
373 bpf_prog_put(old_prog);
376 mutex_unlock(&link_mutex);
380 static const struct bpf_link_ops bpf_iter_link_lops = {
381 .release = bpf_iter_link_release,
382 .dealloc = bpf_iter_link_dealloc,
383 .update_prog = bpf_iter_link_replace,
386 bool bpf_link_is_iter(struct bpf_link *link)
388 return link->ops == &bpf_iter_link_lops;
391 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
393 union bpf_iter_link_info __user *ulinfo;
394 struct bpf_link_primer link_primer;
395 struct bpf_iter_target_info *tinfo;
396 union bpf_iter_link_info linfo;
397 struct bpf_iter_link *link;
398 u32 prog_btf_id, linfo_len;
399 bool existed = false;
402 if (attr->link_create.target_fd || attr->link_create.flags)
405 memset(&linfo, 0, sizeof(union bpf_iter_link_info));
407 ulinfo = u64_to_user_ptr(attr->link_create.iter_info);
408 linfo_len = attr->link_create.iter_info_len;
409 if (!ulinfo ^ !linfo_len)
413 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
417 linfo_len = min_t(u32, linfo_len, sizeof(linfo));
418 if (copy_from_user(&linfo, ulinfo, linfo_len))
422 prog_btf_id = prog->aux->attach_btf_id;
423 mutex_lock(&targets_mutex);
424 list_for_each_entry(tinfo, &targets, list) {
425 if (tinfo->btf_id == prog_btf_id) {
430 mutex_unlock(&targets_mutex);
434 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
438 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
441 err = bpf_link_prime(&link->link, &link_primer);
447 if (tinfo->reg_info->attach_target) {
448 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
450 bpf_link_cleanup(&link_primer);
455 return bpf_link_settle(&link_primer);
458 static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
459 struct bpf_iter_target_info *tinfo,
460 const struct bpf_iter_seq_info *seq_info,
461 struct bpf_prog *prog)
463 priv_data->tinfo = tinfo;
464 priv_data->seq_info = seq_info;
465 priv_data->prog = prog;
466 priv_data->session_id = atomic64_inc_return(&session_id);
467 priv_data->seq_num = 0;
468 priv_data->done_stop = false;
471 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
472 const struct bpf_iter_seq_info *seq_info)
474 struct bpf_iter_priv_data *priv_data;
475 struct bpf_iter_target_info *tinfo;
476 struct bpf_prog *prog;
477 u32 total_priv_dsize;
478 struct seq_file *seq;
481 mutex_lock(&link_mutex);
482 prog = link->link.prog;
484 mutex_unlock(&link_mutex);
487 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
488 seq_info->seq_priv_size;
489 priv_data = __seq_open_private(file, seq_info->seq_ops,
496 if (seq_info->init_seq_private) {
497 err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
499 goto release_seq_file;
502 init_seq_meta(priv_data, tinfo, seq_info, prog);
503 seq = file->private_data;
504 seq->private = priv_data->target_private;
509 seq_release_private(file->f_inode, file);
510 file->private_data = NULL;
516 int bpf_iter_new_fd(struct bpf_link *link)
518 struct bpf_iter_link *iter_link;
523 if (link->ops != &bpf_iter_link_lops)
526 flags = O_RDONLY | O_CLOEXEC;
527 fd = get_unused_fd_flags(flags);
531 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
537 iter_link = container_of(link, struct bpf_iter_link, link);
538 err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
542 fd_install(fd, file);
552 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
554 struct bpf_iter_priv_data *iter_priv;
555 struct seq_file *seq;
559 if (seq->file->f_op != &bpf_iter_fops)
562 seq_priv = seq->private;
563 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
566 if (in_stop && iter_priv->done_stop)
569 meta->session_id = iter_priv->session_id;
570 meta->seq_num = iter_priv->seq_num;
572 return iter_priv->prog;
575 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
581 ret = BPF_PROG_RUN(prog, ctx);
585 /* bpf program can only return 0 or 1:
587 * 1 : retry the same object
588 * The bpf_iter_run_prog() return value
589 * will be seq_ops->show() return value.
591 return ret == 0 ? 0 : -EAGAIN;