1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <net/flow_offload.h>
6 #include <net/netfilter/nf_tables.h>
7 #include <net/netfilter/nf_tables_offload.h>
8 #include <net/pkt_cls.h>
10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
12 struct nft_flow_rule *flow;
14 flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
18 flow->rule = flow_rule_alloc(num_actions);
24 flow->rule->match.dissector = &flow->match.dissector;
25 flow->rule->match.mask = &flow->match.mask;
26 flow->rule->match.key = &flow->match.key;
31 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
32 const struct nft_rule *rule)
34 struct nft_offload_ctx *ctx;
35 struct nft_flow_rule *flow;
36 int num_actions = 0, err;
37 struct nft_expr *expr;
39 expr = nft_expr_first(rule);
40 while (expr->ops && expr != nft_expr_last(rule)) {
41 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
44 expr = nft_expr_next(expr);
48 return ERR_PTR(-EOPNOTSUPP);
50 flow = nft_flow_rule_alloc(num_actions);
52 return ERR_PTR(-ENOMEM);
54 expr = nft_expr_first(rule);
56 ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
62 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
64 while (expr->ops && expr != nft_expr_last(rule)) {
65 if (!expr->ops->offload) {
69 err = expr->ops->offload(ctx, flow, expr);
73 expr = nft_expr_next(expr);
75 flow->proto = ctx->dep.l3num;
81 nft_flow_rule_destroy(flow);
86 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
88 struct flow_action_entry *entry;
91 flow_action_for_each(i, entry, &flow->rule->action) {
93 case FLOW_ACTION_REDIRECT:
94 case FLOW_ACTION_MIRRED:
105 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
106 enum nft_offload_dep_type type)
108 ctx->dep.type = type;
111 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
112 const void *data, u32 len)
114 switch (ctx->dep.type) {
115 case NFT_OFFLOAD_DEP_NETWORK:
116 WARN_ON(len != sizeof(__u16));
117 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
119 case NFT_OFFLOAD_DEP_TRANSPORT:
120 WARN_ON(len != sizeof(__u8));
121 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
126 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
129 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
130 __be16 proto, int priority,
131 struct netlink_ext_ack *extack)
133 common->protocol = proto;
134 common->prio = priority;
135 common->extack = extack;
138 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
139 struct list_head *cb_list)
141 struct flow_block_cb *block_cb;
144 list_for_each_entry(block_cb, cb_list, list) {
145 err = block_cb->cb(type, type_data, block_cb->cb_priv);
152 int nft_chain_offload_priority(struct nft_base_chain *basechain)
154 if (basechain->ops.priority <= 0 ||
155 basechain->ops.priority > USHRT_MAX)
161 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
162 const struct nft_base_chain *basechain,
163 const struct nft_rule *rule,
164 const struct nft_flow_rule *flow,
165 struct netlink_ext_ack *extack,
166 enum flow_cls_command command)
168 __be16 proto = ETH_P_ALL;
170 memset(cls_flow, 0, sizeof(*cls_flow));
175 nft_flow_offload_common_init(&cls_flow->common, proto,
176 basechain->ops.priority, extack);
177 cls_flow->command = command;
178 cls_flow->cookie = (unsigned long) rule;
180 cls_flow->rule = flow->rule;
183 static int nft_flow_offload_rule(struct nft_chain *chain,
184 struct nft_rule *rule,
185 struct nft_flow_rule *flow,
186 enum flow_cls_command command)
188 struct netlink_ext_ack extack = {};
189 struct flow_cls_offload cls_flow;
190 struct nft_base_chain *basechain;
192 if (!nft_is_base_chain(chain))
195 basechain = nft_base_chain(chain);
196 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
199 return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
200 &basechain->flow_block.cb_list);
203 static int nft_flow_offload_bind(struct flow_block_offload *bo,
204 struct nft_base_chain *basechain)
206 list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
210 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
211 struct nft_base_chain *basechain)
213 struct flow_block_cb *block_cb, *next;
214 struct flow_cls_offload cls_flow;
215 struct netlink_ext_ack extack;
216 struct nft_chain *chain;
217 struct nft_rule *rule;
219 chain = &basechain->chain;
220 list_for_each_entry(rule, &chain->rules, list) {
221 memset(&extack, 0, sizeof(extack));
222 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
223 &extack, FLOW_CLS_DESTROY);
224 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
227 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
228 list_del(&block_cb->list);
229 flow_block_cb_free(block_cb);
235 static int nft_block_setup(struct nft_base_chain *basechain,
236 struct flow_block_offload *bo,
237 enum flow_block_command cmd)
242 case FLOW_BLOCK_BIND:
243 err = nft_flow_offload_bind(bo, basechain);
245 case FLOW_BLOCK_UNBIND:
246 err = nft_flow_offload_unbind(bo, basechain);
256 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
258 enum flow_block_command cmd,
259 struct nft_base_chain *basechain,
260 struct netlink_ext_ack *extack)
262 memset(bo, 0, sizeof(*bo));
264 bo->block = &basechain->flow_block;
266 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
268 INIT_LIST_HEAD(&bo->cb_list);
271 static int nft_block_offload_cmd(struct nft_base_chain *chain,
272 struct net_device *dev,
273 enum flow_block_command cmd)
275 struct netlink_ext_ack extack = {};
276 struct flow_block_offload bo;
279 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
281 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
285 return nft_block_setup(chain, &bo, cmd);
288 static void nft_indr_block_ing_cmd(struct net_device *dev,
289 struct nft_base_chain *chain,
290 flow_indr_block_bind_cb_t *cb,
292 enum flow_block_command cmd)
294 struct netlink_ext_ack extack = {};
295 struct flow_block_offload bo;
300 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
302 cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
304 nft_block_setup(chain, &bo, cmd);
307 static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
308 struct net_device *dev,
309 enum flow_block_command cmd)
311 struct netlink_ext_ack extack = {};
312 struct flow_block_offload bo;
314 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
316 flow_indr_block_call(dev, &bo, cmd);
318 if (list_empty(&bo.cb_list))
321 return nft_block_setup(chain, &bo, cmd);
324 #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
326 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
327 struct net_device *dev,
328 enum flow_block_command cmd)
332 if (dev->netdev_ops->ndo_setup_tc)
333 err = nft_block_offload_cmd(basechain, dev, cmd);
335 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
340 static int nft_flow_block_chain(struct nft_base_chain *basechain,
341 const struct net_device *this_dev,
342 enum flow_block_command cmd)
344 struct net_device *dev;
345 struct nft_hook *hook;
348 list_for_each_entry(hook, &basechain->hook_list, list) {
350 if (this_dev && this_dev != dev)
353 err = nft_chain_offload_cmd(basechain, dev, cmd);
354 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
366 list_for_each_entry(hook, &basechain->hook_list, list) {
371 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
376 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
377 enum flow_block_command cmd)
379 struct nft_base_chain *basechain;
382 if (!nft_is_base_chain(chain))
385 basechain = nft_base_chain(chain);
386 policy = ppolicy ? *ppolicy : basechain->policy;
388 /* Only default policy to accept is supported for now. */
389 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
392 return nft_flow_block_chain(basechain, NULL, cmd);
395 static void nft_flow_rule_offload_abort(struct net *net,
396 struct nft_trans *trans)
400 list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
401 if (trans->ctx.family != NFPROTO_NETDEV)
404 switch (trans->msg_type) {
405 case NFT_MSG_NEWCHAIN:
406 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
407 nft_trans_chain_update(trans))
410 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
413 case NFT_MSG_DELCHAIN:
414 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
417 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
420 case NFT_MSG_NEWRULE:
421 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
424 err = nft_flow_offload_rule(trans->ctx.chain,
425 nft_trans_rule(trans),
426 NULL, FLOW_CLS_DESTROY);
428 case NFT_MSG_DELRULE:
429 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
432 err = nft_flow_offload_rule(trans->ctx.chain,
433 nft_trans_rule(trans),
434 nft_trans_flow_rule(trans),
439 if (WARN_ON_ONCE(err))
444 int nft_flow_rule_offload_commit(struct net *net)
446 struct nft_trans *trans;
450 list_for_each_entry(trans, &net->nft.commit_list, list) {
451 if (trans->ctx.family != NFPROTO_NETDEV)
454 switch (trans->msg_type) {
455 case NFT_MSG_NEWCHAIN:
456 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
457 nft_trans_chain_update(trans))
460 policy = nft_trans_chain_policy(trans);
461 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
464 case NFT_MSG_DELCHAIN:
465 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
468 policy = nft_trans_chain_policy(trans);
469 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
472 case NFT_MSG_NEWRULE:
473 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
476 if (trans->ctx.flags & NLM_F_REPLACE ||
477 !(trans->ctx.flags & NLM_F_APPEND)) {
481 err = nft_flow_offload_rule(trans->ctx.chain,
482 nft_trans_rule(trans),
483 nft_trans_flow_rule(trans),
486 case NFT_MSG_DELRULE:
487 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
490 err = nft_flow_offload_rule(trans->ctx.chain,
491 nft_trans_rule(trans),
492 NULL, FLOW_CLS_DESTROY);
497 nft_flow_rule_offload_abort(net, trans);
502 list_for_each_entry(trans, &net->nft.commit_list, list) {
503 if (trans->ctx.family != NFPROTO_NETDEV)
506 switch (trans->msg_type) {
507 case NFT_MSG_NEWRULE:
508 case NFT_MSG_DELRULE:
509 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
512 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
522 static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
524 struct nft_base_chain *basechain;
525 struct net *net = dev_net(dev);
526 struct nft_hook *hook, *found;
527 const struct nft_table *table;
528 struct nft_chain *chain;
530 list_for_each_entry(table, &net->nft.tables, list) {
531 if (table->family != NFPROTO_NETDEV)
534 list_for_each_entry(chain, &table->chains, list) {
535 if (!nft_is_base_chain(chain) ||
536 !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
540 basechain = nft_base_chain(chain);
541 list_for_each_entry(hook, &basechain->hook_list, list) {
542 if (hook->ops.dev != dev)
558 static void nft_indr_block_cb(struct net_device *dev,
559 flow_indr_block_bind_cb_t *cb, void *cb_priv,
560 enum flow_block_command cmd)
562 struct net *net = dev_net(dev);
563 struct nft_chain *chain;
565 mutex_lock(&net->nft.commit_mutex);
566 chain = __nft_offload_get_chain(dev);
567 if (chain && chain->flags & NFT_CHAIN_HW_OFFLOAD) {
568 struct nft_base_chain *basechain;
570 basechain = nft_base_chain(chain);
571 nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
573 mutex_unlock(&net->nft.commit_mutex);
576 static int nft_offload_netdev_event(struct notifier_block *this,
577 unsigned long event, void *ptr)
579 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
580 struct net *net = dev_net(dev);
581 struct nft_chain *chain;
583 if (event != NETDEV_UNREGISTER)
586 mutex_lock(&net->nft.commit_mutex);
587 chain = __nft_offload_get_chain(dev);
589 nft_flow_block_chain(nft_base_chain(chain), dev,
592 mutex_unlock(&net->nft.commit_mutex);
597 static struct flow_indr_block_entry block_ing_entry = {
598 .cb = nft_indr_block_cb,
599 .list = LIST_HEAD_INIT(block_ing_entry.list),
602 static struct notifier_block nft_offload_netdev_notifier = {
603 .notifier_call = nft_offload_netdev_event,
606 int nft_offload_init(void)
610 err = register_netdevice_notifier(&nft_offload_netdev_notifier);
614 flow_indr_add_block_cb(&block_ing_entry);
619 void nft_offload_exit(void)
621 flow_indr_del_block_cb(&block_ing_entry);
622 unregister_netdevice_notifier(&nft_offload_netdev_notifier);