1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
4 #include <linux/skbuff.h>
5 #include <net/devlink.h>
6 #include <net/pkt_cls.h>
10 #include "../nfpcore/nfp_cpp.h"
11 #include "../nfpcore/nfp_nsp.h"
12 #include "../nfp_app.h"
13 #include "../nfp_main.h"
14 #include "../nfp_net.h"
15 #include "../nfp_port.h"
17 #define NFP_FLOWER_SUPPORTED_TCPFLAGS \
18 (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
19 TCPHDR_PSH | TCPHDR_URG)
21 #define NFP_FLOWER_SUPPORTED_CTLFLAGS \
22 (FLOW_DIS_IS_FRAGMENT | \
25 #define NFP_FLOWER_WHITELIST_DISSECTOR \
26 (BIT(FLOW_DISSECTOR_KEY_CONTROL) | \
27 BIT(FLOW_DISSECTOR_KEY_BASIC) | \
28 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | \
29 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | \
30 BIT(FLOW_DISSECTOR_KEY_TCP) | \
31 BIT(FLOW_DISSECTOR_KEY_PORTS) | \
32 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | \
33 BIT(FLOW_DISSECTOR_KEY_VLAN) | \
34 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | \
35 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
36 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \
37 BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
38 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | \
39 BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | \
40 BIT(FLOW_DISSECTOR_KEY_ENC_IP) | \
41 BIT(FLOW_DISSECTOR_KEY_MPLS) | \
42 BIT(FLOW_DISSECTOR_KEY_IP))
44 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR \
45 (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
46 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | \
47 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
48 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \
49 BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | \
50 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | \
51 BIT(FLOW_DISSECTOR_KEY_ENC_IP))
53 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R \
54 (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
55 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
56 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS))
59 nfp_flower_xmit_flow(struct nfp_app *app, struct nfp_fl_payload *nfp_flow,
62 u32 meta_len, key_len, mask_len, act_len, tot_len;
66 meta_len = sizeof(struct nfp_fl_rule_metadata);
67 key_len = nfp_flow->meta.key_len;
68 mask_len = nfp_flow->meta.mask_len;
69 act_len = nfp_flow->meta.act_len;
71 tot_len = meta_len + key_len + mask_len + act_len;
73 /* Convert to long words as firmware expects
74 * lengths in units of NFP_FL_LW_SIZ.
76 nfp_flow->meta.key_len >>= NFP_FL_LW_SIZ;
77 nfp_flow->meta.mask_len >>= NFP_FL_LW_SIZ;
78 nfp_flow->meta.act_len >>= NFP_FL_LW_SIZ;
80 skb = nfp_flower_cmsg_alloc(app, tot_len, mtype, GFP_KERNEL);
84 msg = nfp_flower_cmsg_get_data(skb);
85 memcpy(msg, &nfp_flow->meta, meta_len);
86 memcpy(&msg[meta_len], nfp_flow->unmasked_data, key_len);
87 memcpy(&msg[meta_len + key_len], nfp_flow->mask_data, mask_len);
88 memcpy(&msg[meta_len + key_len + mask_len],
89 nfp_flow->action_data, act_len);
91 /* Convert back to bytes as software expects
92 * lengths in units of bytes.
94 nfp_flow->meta.key_len <<= NFP_FL_LW_SIZ;
95 nfp_flow->meta.mask_len <<= NFP_FL_LW_SIZ;
96 nfp_flow->meta.act_len <<= NFP_FL_LW_SIZ;
98 nfp_ctrl_tx(app->ctrl, skb);
103 static bool nfp_flower_check_higher_than_mac(struct tc_cls_flower_offload *f)
105 struct flow_rule *rule = tc_cls_flower_offload_flow_rule(f);
107 return flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS) ||
108 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS) ||
109 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS) ||
110 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ICMP);
114 nfp_flower_calc_opt_layer(struct flow_match_enc_opts *enc_opts,
115 u32 *key_layer_two, int *key_size)
117 if (enc_opts->key->len > NFP_FL_MAX_GENEVE_OPT_KEY)
120 if (enc_opts->key->len > 0) {
121 *key_layer_two |= NFP_FLOWER_LAYER2_GENEVE_OP;
122 *key_size += sizeof(struct nfp_flower_geneve_options);
129 nfp_flower_calculate_key_layers(struct nfp_app *app,
130 struct net_device *netdev,
131 struct nfp_fl_key_ls *ret_key_ls,
132 struct tc_cls_flower_offload *flow,
133 enum nfp_flower_tun_type *tun_type)
135 struct flow_rule *rule = tc_cls_flower_offload_flow_rule(flow);
136 struct flow_dissector *dissector = rule->match.dissector;
137 struct flow_match_basic basic = { NULL, NULL};
138 struct nfp_flower_priv *priv = app->priv;
144 if (dissector->used_keys & ~NFP_FLOWER_WHITELIST_DISSECTOR)
147 /* If any tun dissector is used then the required set must be used. */
148 if (dissector->used_keys & NFP_FLOWER_WHITELIST_TUN_DISSECTOR &&
149 (dissector->used_keys & NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R)
150 != NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R)
154 key_layer = NFP_FLOWER_LAYER_PORT;
155 key_size = sizeof(struct nfp_flower_meta_tci) +
156 sizeof(struct nfp_flower_in_port);
158 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS) ||
159 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_MPLS)) {
160 key_layer |= NFP_FLOWER_LAYER_MAC;
161 key_size += sizeof(struct nfp_flower_mac_mpls);
164 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
165 struct flow_match_vlan vlan;
167 flow_rule_match_vlan(rule, &vlan);
168 if (!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_PCP) &&
169 vlan.key->vlan_priority)
173 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
174 struct flow_match_enc_opts enc_op = { NULL, NULL };
175 struct flow_match_ipv4_addrs ipv4_addrs;
176 struct flow_match_control enc_ctl;
177 struct flow_match_ports enc_ports;
179 flow_rule_match_enc_control(rule, &enc_ctl);
181 if (enc_ctl.mask->addr_type != 0xffff ||
182 enc_ctl.key->addr_type != FLOW_DISSECTOR_KEY_IPV4_ADDRS)
185 /* These fields are already verified as used. */
186 flow_rule_match_enc_ipv4_addrs(rule, &ipv4_addrs);
187 if (ipv4_addrs.mask->dst != cpu_to_be32(~0))
190 flow_rule_match_enc_ports(rule, &enc_ports);
191 if (enc_ports.mask->dst != cpu_to_be16(~0))
194 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_OPTS))
195 flow_rule_match_enc_opts(rule, &enc_op);
197 switch (enc_ports.key->dst) {
198 case htons(NFP_FL_VXLAN_PORT):
199 *tun_type = NFP_FL_TUNNEL_VXLAN;
200 key_layer |= NFP_FLOWER_LAYER_VXLAN;
201 key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
206 case htons(NFP_FL_GENEVE_PORT):
207 if (!(priv->flower_ext_feats & NFP_FL_FEATS_GENEVE))
209 *tun_type = NFP_FL_TUNNEL_GENEVE;
210 key_layer |= NFP_FLOWER_LAYER_EXT_META;
211 key_size += sizeof(struct nfp_flower_ext_meta);
212 key_layer_two |= NFP_FLOWER_LAYER2_GENEVE;
213 key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
217 if (!(priv->flower_ext_feats & NFP_FL_FEATS_GENEVE_OPT))
219 err = nfp_flower_calc_opt_layer(&enc_op, &key_layer_two,
228 /* Ensure the ingress netdev matches the expected tun type. */
229 if (!nfp_fl_netdev_is_tunnel_type(netdev, *tun_type))
233 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC))
234 flow_rule_match_basic(rule, &basic);
236 if (basic.mask && basic.mask->n_proto) {
237 /* Ethernet type is present in the key. */
238 switch (basic.key->n_proto) {
239 case cpu_to_be16(ETH_P_IP):
240 key_layer |= NFP_FLOWER_LAYER_IPV4;
241 key_size += sizeof(struct nfp_flower_ipv4);
244 case cpu_to_be16(ETH_P_IPV6):
245 key_layer |= NFP_FLOWER_LAYER_IPV6;
246 key_size += sizeof(struct nfp_flower_ipv6);
249 /* Currently we do not offload ARP
250 * because we rely on it to get to the host.
252 case cpu_to_be16(ETH_P_ARP):
255 case cpu_to_be16(ETH_P_MPLS_UC):
256 case cpu_to_be16(ETH_P_MPLS_MC):
257 if (!(key_layer & NFP_FLOWER_LAYER_MAC)) {
258 key_layer |= NFP_FLOWER_LAYER_MAC;
259 key_size += sizeof(struct nfp_flower_mac_mpls);
263 /* Will be included in layer 2. */
264 case cpu_to_be16(ETH_P_8021Q):
268 /* Other ethtype - we need check the masks for the
269 * remainder of the key to ensure we can offload.
271 if (nfp_flower_check_higher_than_mac(flow))
277 if (basic.mask && basic.mask->ip_proto) {
278 /* Ethernet type is present in the key. */
279 switch (basic.key->ip_proto) {
285 key_layer |= NFP_FLOWER_LAYER_TP;
286 key_size += sizeof(struct nfp_flower_tp_ports);
289 /* Other ip proto - we need check the masks for the
290 * remainder of the key to ensure we can offload.
296 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
297 struct flow_match_tcp tcp;
300 flow_rule_match_tcp(rule, &tcp);
301 tcp_flags = be16_to_cpu(tcp.key->flags);
303 if (tcp_flags & ~NFP_FLOWER_SUPPORTED_TCPFLAGS)
306 /* We only support PSH and URG flags when either
307 * FIN, SYN or RST is present as well.
309 if ((tcp_flags & (TCPHDR_PSH | TCPHDR_URG)) &&
310 !(tcp_flags & (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST)))
313 /* We need to store TCP flags in the either the IPv4 or IPv6 key
314 * space, thus we need to ensure we include a IPv4/IPv6 key
315 * layer if we have not done so already.
320 if (!(key_layer & NFP_FLOWER_LAYER_IPV4) &&
321 !(key_layer & NFP_FLOWER_LAYER_IPV6)) {
322 switch (basic.key->n_proto) {
323 case cpu_to_be16(ETH_P_IP):
324 key_layer |= NFP_FLOWER_LAYER_IPV4;
325 key_size += sizeof(struct nfp_flower_ipv4);
328 case cpu_to_be16(ETH_P_IPV6):
329 key_layer |= NFP_FLOWER_LAYER_IPV6;
330 key_size += sizeof(struct nfp_flower_ipv6);
339 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
340 struct flow_match_control ctl;
342 flow_rule_match_control(rule, &ctl);
343 if (ctl.key->flags & ~NFP_FLOWER_SUPPORTED_CTLFLAGS)
347 ret_key_ls->key_layer = key_layer;
348 ret_key_ls->key_layer_two = key_layer_two;
349 ret_key_ls->key_size = key_size;
354 static struct nfp_fl_payload *
355 nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer)
357 struct nfp_fl_payload *flow_pay;
359 flow_pay = kmalloc(sizeof(*flow_pay), GFP_KERNEL);
363 flow_pay->meta.key_len = key_layer->key_size;
364 flow_pay->unmasked_data = kmalloc(key_layer->key_size, GFP_KERNEL);
365 if (!flow_pay->unmasked_data)
368 flow_pay->meta.mask_len = key_layer->key_size;
369 flow_pay->mask_data = kmalloc(key_layer->key_size, GFP_KERNEL);
370 if (!flow_pay->mask_data)
371 goto err_free_unmasked;
373 flow_pay->action_data = kmalloc(NFP_FL_MAX_A_SIZ, GFP_KERNEL);
374 if (!flow_pay->action_data)
377 flow_pay->nfp_tun_ipv4_addr = 0;
378 flow_pay->meta.flags = 0;
383 kfree(flow_pay->mask_data);
385 kfree(flow_pay->unmasked_data);
392 * nfp_flower_add_offload() - Adds a new flow to hardware.
393 * @app: Pointer to the APP handle
394 * @netdev: netdev structure.
395 * @flow: TC flower classifier offload structure.
397 * Adds a new flow to the repeated hash structure and action payload.
399 * Return: negative value on error, 0 if configured successfully.
402 nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
403 struct tc_cls_flower_offload *flow)
405 enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
406 struct nfp_flower_priv *priv = app->priv;
407 struct nfp_fl_payload *flow_pay;
408 struct nfp_fl_key_ls *key_layer;
409 struct nfp_port *port = NULL;
412 if (nfp_netdev_is_nfp_repr(netdev))
413 port = nfp_port_from_netdev(netdev);
415 key_layer = kmalloc(sizeof(*key_layer), GFP_KERNEL);
419 err = nfp_flower_calculate_key_layers(app, netdev, key_layer, flow,
422 goto err_free_key_ls;
424 flow_pay = nfp_flower_allocate_new(key_layer);
427 goto err_free_key_ls;
430 err = nfp_flower_compile_flow_match(app, flow, key_layer, netdev,
433 goto err_destroy_flow;
435 err = nfp_flower_compile_action(app, flow, netdev, flow_pay);
437 goto err_destroy_flow;
439 err = nfp_compile_flow_metadata(app, flow, flow_pay, netdev);
441 goto err_destroy_flow;
443 flow_pay->tc_flower_cookie = flow->cookie;
444 err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node,
445 nfp_flower_table_params);
447 goto err_release_metadata;
449 err = nfp_flower_xmit_flow(app, flow_pay,
450 NFP_FLOWER_CMSG_TYPE_FLOW_ADD);
452 goto err_remove_rhash;
455 port->tc_offload_cnt++;
457 /* Deallocate flow payload when flower rule has been destroyed. */
463 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
465 nfp_flower_table_params));
466 err_release_metadata:
467 nfp_modify_flow_metadata(app, flow_pay);
469 kfree(flow_pay->action_data);
470 kfree(flow_pay->mask_data);
471 kfree(flow_pay->unmasked_data);
479 * nfp_flower_del_offload() - Removes a flow from hardware.
480 * @app: Pointer to the APP handle
481 * @netdev: netdev structure.
482 * @flow: TC flower classifier offload structure
484 * Removes a flow from the repeated hash structure and clears the
487 * Return: negative value on error, 0 if removed successfully.
490 nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
491 struct tc_cls_flower_offload *flow)
493 struct nfp_flower_priv *priv = app->priv;
494 struct nfp_fl_payload *nfp_flow;
495 struct nfp_port *port = NULL;
498 if (nfp_netdev_is_nfp_repr(netdev))
499 port = nfp_port_from_netdev(netdev);
501 nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev);
505 err = nfp_modify_flow_metadata(app, nfp_flow);
509 if (nfp_flow->nfp_tun_ipv4_addr)
510 nfp_tunnel_del_ipv4_off(app, nfp_flow->nfp_tun_ipv4_addr);
512 err = nfp_flower_xmit_flow(app, nfp_flow,
513 NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
519 port->tc_offload_cnt--;
520 kfree(nfp_flow->action_data);
521 kfree(nfp_flow->mask_data);
522 kfree(nfp_flow->unmasked_data);
523 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
525 nfp_flower_table_params));
526 kfree_rcu(nfp_flow, rcu);
531 * nfp_flower_get_stats() - Populates flow stats obtained from hardware.
532 * @app: Pointer to the APP handle
533 * @netdev: Netdev structure.
534 * @flow: TC flower classifier offload structure
536 * Populates a flow statistics structure which which corresponds to a
539 * Return: negative value on error, 0 if stats populated successfully.
542 nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev,
543 struct tc_cls_flower_offload *flow)
545 struct nfp_flower_priv *priv = app->priv;
546 struct nfp_fl_payload *nfp_flow;
549 nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev);
553 ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
555 spin_lock_bh(&priv->stats_lock);
556 flow_stats_update(&flow->stats, priv->stats[ctx_id].bytes,
557 priv->stats[ctx_id].pkts, priv->stats[ctx_id].used);
559 priv->stats[ctx_id].pkts = 0;
560 priv->stats[ctx_id].bytes = 0;
561 spin_unlock_bh(&priv->stats_lock);
567 nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev,
568 struct tc_cls_flower_offload *flower)
570 if (!eth_proto_is_802_3(flower->common.protocol))
573 switch (flower->command) {
574 case TC_CLSFLOWER_REPLACE:
575 return nfp_flower_add_offload(app, netdev, flower);
576 case TC_CLSFLOWER_DESTROY:
577 return nfp_flower_del_offload(app, netdev, flower);
578 case TC_CLSFLOWER_STATS:
579 return nfp_flower_get_stats(app, netdev, flower);
585 static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type,
586 void *type_data, void *cb_priv)
588 struct nfp_repr *repr = cb_priv;
590 if (!tc_cls_can_offload_and_chain0(repr->netdev, type_data))
594 case TC_SETUP_CLSFLOWER:
595 return nfp_flower_repr_offload(repr->app, repr->netdev,
602 static int nfp_flower_setup_tc_block(struct net_device *netdev,
603 struct tc_block_offload *f)
605 struct nfp_repr *repr = netdev_priv(netdev);
607 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
610 switch (f->command) {
612 return tcf_block_cb_register(f->block,
613 nfp_flower_setup_tc_block_cb,
614 repr, repr, f->extack);
615 case TC_BLOCK_UNBIND:
616 tcf_block_cb_unregister(f->block,
617 nfp_flower_setup_tc_block_cb,
625 int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev,
626 enum tc_setup_type type, void *type_data)
630 return nfp_flower_setup_tc_block(netdev, type_data);
636 struct nfp_flower_indr_block_cb_priv {
637 struct net_device *netdev;
639 struct list_head list;
642 static struct nfp_flower_indr_block_cb_priv *
643 nfp_flower_indr_block_cb_priv_lookup(struct nfp_app *app,
644 struct net_device *netdev)
646 struct nfp_flower_indr_block_cb_priv *cb_priv;
647 struct nfp_flower_priv *priv = app->priv;
649 /* All callback list access should be protected by RTNL. */
652 list_for_each_entry(cb_priv, &priv->indr_block_cb_priv, list)
653 if (cb_priv->netdev == netdev)
659 static int nfp_flower_setup_indr_block_cb(enum tc_setup_type type,
660 void *type_data, void *cb_priv)
662 struct nfp_flower_indr_block_cb_priv *priv = cb_priv;
663 struct tc_cls_flower_offload *flower = type_data;
665 if (flower->common.chain_index)
669 case TC_SETUP_CLSFLOWER:
670 return nfp_flower_repr_offload(priv->app, priv->netdev,
678 nfp_flower_setup_indr_tc_block(struct net_device *netdev, struct nfp_app *app,
679 struct tc_block_offload *f)
681 struct nfp_flower_indr_block_cb_priv *cb_priv;
682 struct nfp_flower_priv *priv = app->priv;
685 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
688 switch (f->command) {
690 cb_priv = kmalloc(sizeof(*cb_priv), GFP_KERNEL);
694 cb_priv->netdev = netdev;
696 list_add(&cb_priv->list, &priv->indr_block_cb_priv);
698 err = tcf_block_cb_register(f->block,
699 nfp_flower_setup_indr_block_cb,
700 cb_priv, cb_priv, f->extack);
702 list_del(&cb_priv->list);
707 case TC_BLOCK_UNBIND:
708 cb_priv = nfp_flower_indr_block_cb_priv_lookup(app, netdev);
712 tcf_block_cb_unregister(f->block,
713 nfp_flower_setup_indr_block_cb,
715 list_del(&cb_priv->list);
726 nfp_flower_indr_setup_tc_cb(struct net_device *netdev, void *cb_priv,
727 enum tc_setup_type type, void *type_data)
731 return nfp_flower_setup_indr_tc_block(netdev, cb_priv,
738 int nfp_flower_reg_indir_block_handler(struct nfp_app *app,
739 struct net_device *netdev,
744 if (!nfp_fl_is_netdev_to_offload(netdev))
747 if (event == NETDEV_REGISTER) {
748 err = __tc_indr_block_cb_register(netdev, app,
749 nfp_flower_indr_setup_tc_cb,
752 nfp_flower_cmsg_warn(app,
753 "Indirect block reg failed - %s\n",
755 } else if (event == NETDEV_UNREGISTER) {
756 __tc_indr_block_cb_unregister(netdev,
757 nfp_flower_indr_setup_tc_cb, app);