1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2020 Facebook */
5 #include <linux/anon_inodes.h>
6 #include <linux/filter.h>
9 struct bpf_iter_target_info {
10 struct list_head list;
11 const struct bpf_iter_reg *reg_info;
12 u32 btf_id; /* cached value */
15 struct bpf_iter_link {
17 struct bpf_iter_target_info *tinfo;
20 struct bpf_iter_priv_data {
21 struct bpf_iter_target_info *tinfo;
22 struct bpf_prog *prog;
26 u8 target_private[] __aligned(8);
29 static struct list_head targets = LIST_HEAD_INIT(targets);
30 static DEFINE_MUTEX(targets_mutex);
32 /* protect bpf_iter_link changes */
33 static DEFINE_MUTEX(link_mutex);
35 /* incremented on every opened seq_file */
36 static atomic64_t session_id;
38 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link);
40 static void bpf_iter_inc_seq_num(struct seq_file *seq)
42 struct bpf_iter_priv_data *iter_priv;
44 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
49 static void bpf_iter_dec_seq_num(struct seq_file *seq)
51 struct bpf_iter_priv_data *iter_priv;
53 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
58 static void bpf_iter_done_stop(struct seq_file *seq)
60 struct bpf_iter_priv_data *iter_priv;
62 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
64 iter_priv->done_stop = true;
67 /* bpf_seq_read, a customized and simpler version for bpf iterator.
68 * no_llseek is assumed for this file.
69 * The following are differences from seq_read():
70 * . fixed buffer size (PAGE_SIZE)
71 * . assuming no_llseek
72 * . stop() may call bpf program, handling potential overflow there
74 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
77 struct seq_file *seq = file->private_data;
78 size_t n, offs, copied = 0;
82 mutex_lock(&seq->lock);
85 seq->size = PAGE_SIZE;
86 seq->buf = kmalloc(seq->size, GFP_KERNEL);
94 n = min(seq->count, size);
95 err = copy_to_user(buf, seq->buf + seq->from, n);
107 p = seq->op->start(seq, &seq->index);
112 seq->op->stop(seq, p);
117 err = seq->op->show(seq, p);
119 /* object is skipped, decrease seq_num, so next
120 * valid object can reuse the same seq_num.
122 bpf_iter_dec_seq_num(seq);
124 } else if (err < 0 || seq_has_overflowed(seq)) {
127 seq->op->stop(seq, p);
133 loff_t pos = seq->index;
136 p = seq->op->next(seq, p, &seq->index);
137 if (pos == seq->index) {
138 pr_info_ratelimited("buggy seq_file .next function %ps "
139 "did not updated position index\n",
144 if (IS_ERR_OR_NULL(p))
147 /* got a valid next object, increase seq_num */
148 bpf_iter_inc_seq_num(seq);
150 if (seq->count >= size)
153 err = seq->op->show(seq, p);
155 bpf_iter_dec_seq_num(seq);
157 } else if (err < 0 || seq_has_overflowed(seq)) {
162 seq->op->stop(seq, p);
170 /* bpf program called if !p */
171 seq->op->stop(seq, p);
173 if (!seq_has_overflowed(seq)) {
174 bpf_iter_done_stop(seq);
184 n = min(seq->count, size);
185 err = copy_to_user(buf, seq->buf, n);
198 mutex_unlock(&seq->lock);
202 static int iter_open(struct inode *inode, struct file *file)
204 struct bpf_iter_link *link = inode->i_private;
206 return prepare_seq_file(file, link);
209 static int iter_release(struct inode *inode, struct file *file)
211 struct bpf_iter_priv_data *iter_priv;
212 struct seq_file *seq;
214 seq = file->private_data;
218 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
221 if (iter_priv->tinfo->reg_info->fini_seq_private)
222 iter_priv->tinfo->reg_info->fini_seq_private(seq->private);
224 bpf_prog_put(iter_priv->prog);
225 seq->private = iter_priv;
227 return seq_release_private(inode, file);
230 const struct file_operations bpf_iter_fops = {
233 .read = bpf_seq_read,
234 .release = iter_release,
237 /* The argument reg_info will be cached in bpf_iter_target_info.
238 * The common practice is to declare target reg_info as
239 * a const static variable and passed as an argument to
240 * bpf_iter_reg_target().
242 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
244 struct bpf_iter_target_info *tinfo;
246 tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);
250 tinfo->reg_info = reg_info;
251 INIT_LIST_HEAD(&tinfo->list);
253 mutex_lock(&targets_mutex);
254 list_add(&tinfo->list, &targets);
255 mutex_unlock(&targets_mutex);
260 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
262 struct bpf_iter_target_info *tinfo;
265 mutex_lock(&targets_mutex);
266 list_for_each_entry(tinfo, &targets, list) {
267 if (reg_info == tinfo->reg_info) {
268 list_del(&tinfo->list);
274 mutex_unlock(&targets_mutex);
276 WARN_ON(found == false);
279 static void cache_btf_id(struct bpf_iter_target_info *tinfo,
280 struct bpf_prog *prog)
282 tinfo->btf_id = prog->aux->attach_btf_id;
285 bool bpf_iter_prog_supported(struct bpf_prog *prog)
287 const char *attach_fname = prog->aux->attach_func_name;
288 u32 prog_btf_id = prog->aux->attach_btf_id;
289 const char *prefix = BPF_ITER_FUNC_PREFIX;
290 struct bpf_iter_target_info *tinfo;
291 int prefix_len = strlen(prefix);
292 bool supported = false;
294 if (strncmp(attach_fname, prefix, prefix_len))
297 mutex_lock(&targets_mutex);
298 list_for_each_entry(tinfo, &targets, list) {
299 if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
303 if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
304 cache_btf_id(tinfo, prog);
309 mutex_unlock(&targets_mutex);
312 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
313 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
319 static void bpf_iter_link_release(struct bpf_link *link)
323 static void bpf_iter_link_dealloc(struct bpf_link *link)
325 struct bpf_iter_link *iter_link =
326 container_of(link, struct bpf_iter_link, link);
331 static int bpf_iter_link_replace(struct bpf_link *link,
332 struct bpf_prog *new_prog,
333 struct bpf_prog *old_prog)
337 mutex_lock(&link_mutex);
338 if (old_prog && link->prog != old_prog) {
343 if (link->prog->type != new_prog->type ||
344 link->prog->expected_attach_type != new_prog->expected_attach_type ||
345 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
350 old_prog = xchg(&link->prog, new_prog);
351 bpf_prog_put(old_prog);
354 mutex_unlock(&link_mutex);
358 static const struct bpf_link_ops bpf_iter_link_lops = {
359 .release = bpf_iter_link_release,
360 .dealloc = bpf_iter_link_dealloc,
361 .update_prog = bpf_iter_link_replace,
364 bool bpf_link_is_iter(struct bpf_link *link)
366 return link->ops == &bpf_iter_link_lops;
369 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
371 struct bpf_link_primer link_primer;
372 struct bpf_iter_target_info *tinfo;
373 struct bpf_iter_link *link;
374 bool existed = false;
378 if (attr->link_create.target_fd || attr->link_create.flags)
381 prog_btf_id = prog->aux->attach_btf_id;
382 mutex_lock(&targets_mutex);
383 list_for_each_entry(tinfo, &targets, list) {
384 if (tinfo->btf_id == prog_btf_id) {
389 mutex_unlock(&targets_mutex);
393 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
397 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
400 err = bpf_link_prime(&link->link, &link_primer);
406 return bpf_link_settle(&link_primer);
409 static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
410 struct bpf_iter_target_info *tinfo,
411 struct bpf_prog *prog)
413 priv_data->tinfo = tinfo;
414 priv_data->prog = prog;
415 priv_data->session_id = atomic64_inc_return(&session_id);
416 priv_data->seq_num = 0;
417 priv_data->done_stop = false;
420 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link)
422 struct bpf_iter_priv_data *priv_data;
423 struct bpf_iter_target_info *tinfo;
424 struct bpf_prog *prog;
425 u32 total_priv_dsize;
426 struct seq_file *seq;
429 mutex_lock(&link_mutex);
430 prog = link->link.prog;
432 mutex_unlock(&link_mutex);
435 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
436 tinfo->reg_info->seq_priv_size;
437 priv_data = __seq_open_private(file, tinfo->reg_info->seq_ops,
444 if (tinfo->reg_info->init_seq_private) {
445 err = tinfo->reg_info->init_seq_private(priv_data->target_private);
447 goto release_seq_file;
450 init_seq_meta(priv_data, tinfo, prog);
451 seq = file->private_data;
452 seq->private = priv_data->target_private;
457 seq_release_private(file->f_inode, file);
458 file->private_data = NULL;
464 int bpf_iter_new_fd(struct bpf_link *link)
470 if (link->ops != &bpf_iter_link_lops)
473 flags = O_RDONLY | O_CLOEXEC;
474 fd = get_unused_fd_flags(flags);
478 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
484 err = prepare_seq_file(file,
485 container_of(link, struct bpf_iter_link, link));
489 fd_install(fd, file);
499 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
501 struct bpf_iter_priv_data *iter_priv;
502 struct seq_file *seq;
506 if (seq->file->f_op != &bpf_iter_fops)
509 seq_priv = seq->private;
510 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
513 if (in_stop && iter_priv->done_stop)
516 meta->session_id = iter_priv->session_id;
517 meta->seq_num = iter_priv->seq_num;
519 return iter_priv->prog;
522 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
528 ret = BPF_PROG_RUN(prog, ctx);
532 /* bpf program can only return 0 or 1:
534 * 1 : retry the same object
535 * The bpf_iter_run_prog() return value
536 * will be seq_ops->show() return value.
538 return ret == 0 ? 0 : -EAGAIN;