1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/init.h>
3 #include <linux/module.h>
4 #include <linux/netfilter.h>
5 #include <net/flow_offload.h>
6 #include <net/netfilter/nf_tables.h>
7 #include <net/netfilter/nf_tables_offload.h>
8 #include <net/pkt_cls.h>
10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
12 struct nft_flow_rule *flow;
14 flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
18 flow->rule = flow_rule_alloc(num_actions);
24 flow->rule->match.dissector = &flow->match.dissector;
25 flow->rule->match.mask = &flow->match.mask;
26 flow->rule->match.key = &flow->match.key;
31 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
32 const struct nft_rule *rule)
34 struct nft_offload_ctx *ctx;
35 struct nft_flow_rule *flow;
36 int num_actions = 0, err;
37 struct nft_expr *expr;
39 expr = nft_expr_first(rule);
40 while (expr->ops && expr != nft_expr_last(rule)) {
41 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
44 expr = nft_expr_next(expr);
47 flow = nft_flow_rule_alloc(num_actions);
49 return ERR_PTR(-ENOMEM);
51 expr = nft_expr_first(rule);
53 ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
59 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
61 while (expr->ops && expr != nft_expr_last(rule)) {
62 if (!expr->ops->offload) {
66 err = expr->ops->offload(ctx, flow, expr);
70 expr = nft_expr_next(expr);
72 flow->proto = ctx->dep.l3num;
78 nft_flow_rule_destroy(flow);
83 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
85 struct flow_action_entry *entry;
88 flow_action_for_each(i, entry, &flow->rule->action) {
90 case FLOW_ACTION_REDIRECT:
91 case FLOW_ACTION_MIRRED:
102 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
103 enum nft_offload_dep_type type)
105 ctx->dep.type = type;
108 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
109 const void *data, u32 len)
111 switch (ctx->dep.type) {
112 case NFT_OFFLOAD_DEP_NETWORK:
113 WARN_ON(len != sizeof(__u16));
114 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
116 case NFT_OFFLOAD_DEP_TRANSPORT:
117 WARN_ON(len != sizeof(__u8));
118 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
123 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
126 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
127 __be16 proto, int priority,
128 struct netlink_ext_ack *extack)
130 common->protocol = proto;
131 common->prio = priority;
132 common->extack = extack;
135 static int nft_setup_cb_call(struct nft_base_chain *basechain,
136 enum tc_setup_type type, void *type_data)
138 struct flow_block_cb *block_cb;
141 list_for_each_entry(block_cb, &basechain->flow_block.cb_list, list) {
142 err = block_cb->cb(type, type_data, block_cb->cb_priv);
149 int nft_chain_offload_priority(struct nft_base_chain *basechain)
151 if (basechain->ops.priority <= 0 ||
152 basechain->ops.priority > USHRT_MAX)
158 static int nft_flow_offload_rule(struct nft_chain *chain,
159 struct nft_rule *rule,
160 struct nft_flow_rule *flow,
161 enum flow_cls_command command)
163 struct flow_cls_offload cls_flow = {};
164 struct nft_base_chain *basechain;
165 struct netlink_ext_ack extack;
166 __be16 proto = ETH_P_ALL;
168 if (!nft_is_base_chain(chain))
171 basechain = nft_base_chain(chain);
176 nft_flow_offload_common_init(&cls_flow.common, proto,
177 basechain->ops.priority, &extack);
178 cls_flow.command = command;
179 cls_flow.cookie = (unsigned long) rule;
181 cls_flow.rule = flow->rule;
183 return nft_setup_cb_call(basechain, TC_SETUP_CLSFLOWER, &cls_flow);
186 static int nft_flow_offload_bind(struct flow_block_offload *bo,
187 struct nft_base_chain *basechain)
189 list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
193 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
194 struct nft_base_chain *basechain)
196 struct flow_block_cb *block_cb, *next;
198 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
199 list_del(&block_cb->list);
200 flow_block_cb_free(block_cb);
206 static int nft_block_setup(struct nft_base_chain *basechain,
207 struct flow_block_offload *bo,
208 enum flow_block_command cmd)
213 case FLOW_BLOCK_BIND:
214 err = nft_flow_offload_bind(bo, basechain);
216 case FLOW_BLOCK_UNBIND:
217 err = nft_flow_offload_unbind(bo, basechain);
227 static int nft_block_offload_cmd(struct nft_base_chain *chain,
228 struct net_device *dev,
229 enum flow_block_command cmd)
231 struct netlink_ext_ack extack = {};
232 struct flow_block_offload bo = {};
235 bo.net = dev_net(dev);
236 bo.block = &chain->flow_block;
238 bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
240 INIT_LIST_HEAD(&bo.cb_list);
242 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
246 return nft_block_setup(chain, &bo, cmd);
249 static void nft_indr_block_ing_cmd(struct net_device *dev,
250 struct nft_base_chain *chain,
251 flow_indr_block_bind_cb_t *cb,
253 enum flow_block_command cmd)
255 struct netlink_ext_ack extack = {};
256 struct flow_block_offload bo = {};
261 bo.net = dev_net(dev);
262 bo.block = &chain->flow_block;
264 bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
266 INIT_LIST_HEAD(&bo.cb_list);
268 cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
270 nft_block_setup(chain, &bo, cmd);
273 static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
274 struct net_device *dev,
275 enum flow_block_command cmd)
277 struct flow_block_offload bo = {};
278 struct netlink_ext_ack extack = {};
280 bo.net = dev_net(dev);
281 bo.block = &chain->flow_block;
283 bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
285 INIT_LIST_HEAD(&bo.cb_list);
287 flow_indr_block_call(dev, &bo, cmd);
289 if (list_empty(&bo.cb_list))
292 return nft_block_setup(chain, &bo, cmd);
295 #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
297 static int nft_flow_offload_chain(struct nft_chain *chain,
299 enum flow_block_command cmd)
301 struct nft_base_chain *basechain;
302 struct net_device *dev;
305 if (!nft_is_base_chain(chain))
308 basechain = nft_base_chain(chain);
309 dev = basechain->ops.dev;
313 policy = ppolicy ? *ppolicy : basechain->policy;
315 /* Only default policy to accept is supported for now. */
316 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
319 if (dev->netdev_ops->ndo_setup_tc)
320 return nft_block_offload_cmd(basechain, dev, cmd);
322 return nft_indr_block_offload_cmd(basechain, dev, cmd);
325 int nft_flow_rule_offload_commit(struct net *net)
327 struct nft_trans *trans;
331 list_for_each_entry(trans, &net->nft.commit_list, list) {
332 if (trans->ctx.family != NFPROTO_NETDEV)
335 switch (trans->msg_type) {
336 case NFT_MSG_NEWCHAIN:
337 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
338 nft_trans_chain_update(trans))
341 policy = nft_trans_chain_policy(trans);
342 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
345 case NFT_MSG_DELCHAIN:
346 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
349 policy = nft_trans_chain_policy(trans);
350 err = nft_flow_offload_chain(trans->ctx.chain, &policy,
353 case NFT_MSG_NEWRULE:
354 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
357 if (trans->ctx.flags & NLM_F_REPLACE ||
358 !(trans->ctx.flags & NLM_F_APPEND))
361 err = nft_flow_offload_rule(trans->ctx.chain,
362 nft_trans_rule(trans),
363 nft_trans_flow_rule(trans),
365 nft_flow_rule_destroy(nft_trans_flow_rule(trans));
367 case NFT_MSG_DELRULE:
368 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
371 err = nft_flow_offload_rule(trans->ctx.chain,
372 nft_trans_rule(trans),
373 nft_trans_flow_rule(trans),
385 static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
387 struct nft_base_chain *basechain;
388 struct net *net = dev_net(dev);
389 const struct nft_table *table;
390 struct nft_chain *chain;
392 list_for_each_entry(table, &net->nft.tables, list) {
393 if (table->family != NFPROTO_NETDEV)
396 list_for_each_entry(chain, &table->chains, list) {
397 if (!nft_is_base_chain(chain) ||
398 !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
401 basechain = nft_base_chain(chain);
402 if (strncmp(basechain->dev_name, dev->name, IFNAMSIZ))
412 static void nft_indr_block_cb(struct net_device *dev,
413 flow_indr_block_bind_cb_t *cb, void *cb_priv,
414 enum flow_block_command cmd)
416 struct net *net = dev_net(dev);
417 struct nft_chain *chain;
419 mutex_lock(&net->nft.commit_mutex);
420 chain = __nft_offload_get_chain(dev);
422 struct nft_base_chain *basechain;
424 basechain = nft_base_chain(chain);
425 nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
427 mutex_unlock(&net->nft.commit_mutex);
430 static void nft_offload_chain_clean(struct nft_chain *chain)
432 struct nft_rule *rule;
434 list_for_each_entry(rule, &chain->rules, list) {
435 nft_flow_offload_rule(chain, rule,
436 NULL, FLOW_CLS_DESTROY);
439 nft_flow_offload_chain(chain, NULL, FLOW_BLOCK_UNBIND);
442 static int nft_offload_netdev_event(struct notifier_block *this,
443 unsigned long event, void *ptr)
445 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
446 struct net *net = dev_net(dev);
447 struct nft_chain *chain;
449 mutex_lock(&net->nft.commit_mutex);
450 chain = __nft_offload_get_chain(dev);
452 nft_offload_chain_clean(chain);
453 mutex_unlock(&net->nft.commit_mutex);
458 static struct flow_indr_block_ing_entry block_ing_entry = {
459 .cb = nft_indr_block_cb,
460 .list = LIST_HEAD_INIT(block_ing_entry.list),
463 static struct notifier_block nft_offload_netdev_notifier = {
464 .notifier_call = nft_offload_netdev_event,
467 int nft_offload_init(void)
471 err = register_netdevice_notifier(&nft_offload_netdev_notifier);
475 flow_indr_add_block_ing_cb(&block_ing_entry);
480 void nft_offload_exit(void)
482 flow_indr_del_block_ing_cb(&block_ing_entry);
483 unregister_netdevice_notifier(&nft_offload_netdev_notifier);