1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <net/flow_offload.h>
6 #include <net/netfilter/nf_tables.h>
7 #include <net/netfilter/nf_tables_offload.h>
8 #include <net/pkt_cls.h>
10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
12 struct nft_flow_rule *flow;
14 flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
18 flow->rule = flow_rule_alloc(num_actions);
24 flow->rule->match.dissector = &flow->match.dissector;
25 flow->rule->match.mask = &flow->match.mask;
26 flow->rule->match.key = &flow->match.key;
31 void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
32 enum flow_dissector_key_id addr_type)
34 struct nft_flow_match *match = &flow->match;
35 struct nft_flow_key *mask = &match->mask;
36 struct nft_flow_key *key = &match->key;
38 if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL))
41 key->control.addr_type = addr_type;
42 mask->control.addr_type = 0xffff;
43 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL);
44 match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
45 offsetof(struct nft_flow_key, control);
48 struct nft_offload_ethertype {
53 static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx,
54 struct nft_flow_rule *flow)
56 struct nft_flow_match *match = &flow->match;
57 struct nft_offload_ethertype ethertype = {
58 .value = match->key.basic.n_proto,
59 .mask = match->mask.basic.n_proto,
62 if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_VLAN) &&
63 (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) ||
64 match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) {
65 match->key.basic.n_proto = match->key.cvlan.vlan_tpid;
66 match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid;
67 match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid;
68 match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid;
69 match->key.vlan.vlan_tpid = ethertype.value;
70 match->mask.vlan.vlan_tpid = ethertype.mask;
71 match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] =
72 offsetof(struct nft_flow_key, cvlan);
73 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN);
74 } else if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC) &&
75 (match->key.basic.n_proto == htons(ETH_P_8021Q) ||
76 match->key.basic.n_proto == htons(ETH_P_8021AD))) {
77 match->key.basic.n_proto = match->key.vlan.vlan_tpid;
78 match->mask.basic.n_proto = match->mask.vlan.vlan_tpid;
79 match->key.vlan.vlan_tpid = ethertype.value;
80 match->mask.vlan.vlan_tpid = ethertype.mask;
81 match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
82 offsetof(struct nft_flow_key, vlan);
83 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN);
87 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
88 const struct nft_rule *rule)
90 struct nft_offload_ctx *ctx;
91 struct nft_flow_rule *flow;
92 int num_actions = 0, err;
93 struct nft_expr *expr;
95 expr = nft_expr_first(rule);
96 while (nft_expr_more(rule, expr)) {
97 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
100 expr = nft_expr_next(expr);
103 if (num_actions == 0)
104 return ERR_PTR(-EOPNOTSUPP);
106 flow = nft_flow_rule_alloc(num_actions);
108 return ERR_PTR(-ENOMEM);
110 expr = nft_expr_first(rule);
112 ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
118 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
120 while (nft_expr_more(rule, expr)) {
121 if (!expr->ops->offload) {
125 err = expr->ops->offload(ctx, flow, expr);
129 expr = nft_expr_next(expr);
131 nft_flow_rule_transfer_vlan(ctx, flow);
133 flow->proto = ctx->dep.l3num;
139 nft_flow_rule_destroy(flow);
144 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
146 struct flow_action_entry *entry;
149 flow_action_for_each(i, entry, &flow->rule->action) {
151 case FLOW_ACTION_REDIRECT:
152 case FLOW_ACTION_MIRRED:
163 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
164 enum nft_offload_dep_type type)
166 ctx->dep.type = type;
169 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
170 const void *data, u32 len)
172 switch (ctx->dep.type) {
173 case NFT_OFFLOAD_DEP_NETWORK:
174 WARN_ON(len != sizeof(__u16));
175 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
177 case NFT_OFFLOAD_DEP_TRANSPORT:
178 WARN_ON(len != sizeof(__u8));
179 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
184 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
187 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
188 __be16 proto, int priority,
189 struct netlink_ext_ack *extack)
191 common->protocol = proto;
192 common->prio = priority;
193 common->extack = extack;
196 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
197 struct list_head *cb_list)
199 struct flow_block_cb *block_cb;
202 list_for_each_entry(block_cb, cb_list, list) {
203 err = block_cb->cb(type, type_data, block_cb->cb_priv);
210 int nft_chain_offload_priority(struct nft_base_chain *basechain)
212 if (basechain->ops.priority <= 0 ||
213 basechain->ops.priority > USHRT_MAX)
219 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
220 const struct nft_base_chain *basechain,
221 const struct nft_rule *rule,
222 const struct nft_flow_rule *flow,
223 struct netlink_ext_ack *extack,
224 enum flow_cls_command command)
226 __be16 proto = ETH_P_ALL;
228 memset(cls_flow, 0, sizeof(*cls_flow));
233 nft_flow_offload_common_init(&cls_flow->common, proto,
234 basechain->ops.priority, extack);
235 cls_flow->command = command;
236 cls_flow->cookie = (unsigned long) rule;
238 cls_flow->rule = flow->rule;
241 static int nft_flow_offload_cmd(const struct nft_chain *chain,
242 const struct nft_rule *rule,
243 struct nft_flow_rule *flow,
244 enum flow_cls_command command,
245 struct flow_cls_offload *cls_flow)
247 struct netlink_ext_ack extack = {};
248 struct nft_base_chain *basechain;
250 if (!nft_is_base_chain(chain))
253 basechain = nft_base_chain(chain);
254 nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, &extack,
257 return nft_setup_cb_call(TC_SETUP_CLSFLOWER, cls_flow,
258 &basechain->flow_block.cb_list);
261 static int nft_flow_offload_rule(const struct nft_chain *chain,
262 struct nft_rule *rule,
263 struct nft_flow_rule *flow,
264 enum flow_cls_command command)
266 struct flow_cls_offload cls_flow;
268 return nft_flow_offload_cmd(chain, rule, flow, command, &cls_flow);
271 int nft_flow_rule_stats(const struct nft_chain *chain,
272 const struct nft_rule *rule)
274 struct flow_cls_offload cls_flow = {};
275 struct nft_expr *expr, *next;
278 err = nft_flow_offload_cmd(chain, rule, NULL, FLOW_CLS_STATS,
283 nft_rule_for_each_expr(expr, next, rule) {
284 if (expr->ops->offload_stats)
285 expr->ops->offload_stats(expr, &cls_flow.stats);
291 static int nft_flow_offload_bind(struct flow_block_offload *bo,
292 struct nft_base_chain *basechain)
294 list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
298 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
299 struct nft_base_chain *basechain)
301 struct flow_block_cb *block_cb, *next;
302 struct flow_cls_offload cls_flow;
303 struct netlink_ext_ack extack;
304 struct nft_chain *chain;
305 struct nft_rule *rule;
307 chain = &basechain->chain;
308 list_for_each_entry(rule, &chain->rules, list) {
309 memset(&extack, 0, sizeof(extack));
310 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
311 &extack, FLOW_CLS_DESTROY);
312 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
315 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
316 list_del(&block_cb->list);
317 flow_block_cb_free(block_cb);
323 static int nft_block_setup(struct nft_base_chain *basechain,
324 struct flow_block_offload *bo,
325 enum flow_block_command cmd)
330 case FLOW_BLOCK_BIND:
331 err = nft_flow_offload_bind(bo, basechain);
333 case FLOW_BLOCK_UNBIND:
334 err = nft_flow_offload_unbind(bo, basechain);
344 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
346 enum flow_block_command cmd,
347 struct nft_base_chain *basechain,
348 struct netlink_ext_ack *extack)
350 memset(bo, 0, sizeof(*bo));
352 bo->block = &basechain->flow_block;
354 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
356 INIT_LIST_HEAD(&bo->cb_list);
359 static int nft_block_offload_cmd(struct nft_base_chain *chain,
360 struct net_device *dev,
361 enum flow_block_command cmd)
363 struct netlink_ext_ack extack = {};
364 struct flow_block_offload bo;
367 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
369 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
373 return nft_block_setup(chain, &bo, cmd);
376 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
378 struct nft_base_chain *basechain = block_cb->indr.data;
379 struct net_device *dev = block_cb->indr.dev;
380 struct netlink_ext_ack extack = {};
381 struct nftables_pernet *nft_net;
382 struct net *net = dev_net(dev);
383 struct flow_block_offload bo;
385 nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
387 nft_net = nft_pernet(net);
388 mutex_lock(&nft_net->commit_mutex);
389 list_del(&block_cb->driver_list);
390 list_move(&block_cb->list, &bo.cb_list);
391 nft_flow_offload_unbind(&bo, basechain);
392 mutex_unlock(&nft_net->commit_mutex);
395 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
396 struct net_device *dev,
397 enum flow_block_command cmd)
399 struct netlink_ext_ack extack = {};
400 struct flow_block_offload bo;
403 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
405 err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
406 nft_indr_block_cleanup);
410 if (list_empty(&bo.cb_list))
413 return nft_block_setup(basechain, &bo, cmd);
416 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
417 struct net_device *dev,
418 enum flow_block_command cmd)
422 if (dev->netdev_ops->ndo_setup_tc)
423 err = nft_block_offload_cmd(basechain, dev, cmd);
425 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
430 static int nft_flow_block_chain(struct nft_base_chain *basechain,
431 const struct net_device *this_dev,
432 enum flow_block_command cmd)
434 struct net_device *dev;
435 struct nft_hook *hook;
438 list_for_each_entry(hook, &basechain->hook_list, list) {
440 if (this_dev && this_dev != dev)
443 err = nft_chain_offload_cmd(basechain, dev, cmd);
444 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
456 list_for_each_entry(hook, &basechain->hook_list, list) {
461 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
466 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
467 enum flow_block_command cmd)
469 struct nft_base_chain *basechain;
472 if (!nft_is_base_chain(chain))
475 basechain = nft_base_chain(chain);
476 policy = ppolicy ? *ppolicy : basechain->policy;
478 /* Only default policy to accept is supported for now. */
479 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
482 return nft_flow_block_chain(basechain, NULL, cmd);
485 static void nft_flow_rule_offload_abort(struct net *net,
486 struct nft_trans *trans)
488 struct nftables_pernet *nft_net = nft_pernet(net);
491 list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) {
492 if (trans->ctx.family != NFPROTO_NETDEV)
495 switch (trans->msg_type) {
496 case NFT_MSG_NEWCHAIN:
497 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
498 nft_trans_chain_update(trans))
501 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
504 case NFT_MSG_DELCHAIN:
505 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
508 err = nft_flow_offload_chain(trans->ctx.chain, NULL,
511 case NFT_MSG_NEWRULE:
512 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
515 err = nft_flow_offload_rule(trans->ctx.chain,
516 nft_trans_rule(trans),
517 NULL, FLOW_CLS_DESTROY);
519 case NFT_MSG_DELRULE:
520 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
523 err = nft_flow_offload_rule(trans->ctx.chain,
524 nft_trans_rule(trans),
525 nft_trans_flow_rule(trans),
530 if (WARN_ON_ONCE(err))
535 int nft_flow_rule_offload_commit(struct net *net)
537 struct nftables_pernet *nft_net = nft_pernet(net);
538 struct nft_trans *trans;
542 list_for_each_entry(trans, &nft_net->commit_list, list) {
543 if (trans->ctx.family != NFPROTO_NETDEV)
546 switch (trans->msg_type) {
547 case NFT_MSG_NEWCHAIN:
548 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
549 nft_trans_chain_update(trans))
552 policy = nft_trans_chain_policy(trans);
553 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
556 case NFT_MSG_DELCHAIN:
557 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
560 policy = nft_trans_chain_policy(trans);
561 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
564 case NFT_MSG_NEWRULE:
565 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
568 if (trans->ctx.flags & NLM_F_REPLACE ||
569 !(trans->ctx.flags & NLM_F_APPEND)) {
573 err = nft_flow_offload_rule(trans->ctx.chain,
574 nft_trans_rule(trans),
575 nft_trans_flow_rule(trans),
578 case NFT_MSG_DELRULE:
579 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
582 err = nft_flow_offload_rule(trans->ctx.chain,
583 nft_trans_rule(trans),
584 NULL, FLOW_CLS_DESTROY);
589 nft_flow_rule_offload_abort(net, trans);
597 static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net,
598 struct net_device *dev)
600 struct nft_base_chain *basechain;
601 struct nft_hook *hook, *found;
602 const struct nft_table *table;
603 struct nft_chain *chain;
605 list_for_each_entry(table, &nft_net->tables, list) {
606 if (table->family != NFPROTO_NETDEV)
609 list_for_each_entry(chain, &table->chains, list) {
610 if (!nft_is_base_chain(chain) ||
611 !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
615 basechain = nft_base_chain(chain);
616 list_for_each_entry(hook, &basechain->hook_list, list) {
617 if (hook->ops.dev != dev)
633 static int nft_offload_netdev_event(struct notifier_block *this,
634 unsigned long event, void *ptr)
636 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
637 struct nftables_pernet *nft_net;
638 struct net *net = dev_net(dev);
639 struct nft_chain *chain;
641 if (event != NETDEV_UNREGISTER)
644 nft_net = nft_pernet(net);
645 mutex_lock(&nft_net->commit_mutex);
646 chain = __nft_offload_get_chain(nft_net, dev);
648 nft_flow_block_chain(nft_base_chain(chain), dev,
651 mutex_unlock(&nft_net->commit_mutex);
656 static struct notifier_block nft_offload_netdev_notifier = {
657 .notifier_call = nft_offload_netdev_event,
660 int nft_offload_init(void)
662 return register_netdevice_notifier(&nft_offload_netdev_notifier);
665 void nft_offload_exit(void)
667 unregister_netdevice_notifier(&nft_offload_netdev_notifier);