1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2021 Corigine, Inc. */
6 const struct rhashtable_params nfp_tc_ct_merge_params = {
7 .head_offset = offsetof(struct nfp_fl_ct_tc_merge,
9 .key_len = sizeof(unsigned long) * 2,
10 .key_offset = offsetof(struct nfp_fl_ct_tc_merge, cookie),
11 .automatic_shrinking = true,
14 const struct rhashtable_params nfp_nft_ct_merge_params = {
15 .head_offset = offsetof(struct nfp_fl_nft_tc_merge,
17 .key_len = sizeof(unsigned long) * 3,
18 .key_offset = offsetof(struct nfp_fl_nft_tc_merge, cookie),
19 .automatic_shrinking = true,
22 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
23 enum flow_action_id act_id);
26 * get_hashentry() - Wrapper around hashtable lookup.
27 * @ht: hashtable where entry could be found
29 * @params: hashtable params
30 * @size: size of entry to allocate if not in table
32 * Returns an entry from a hashtable. If entry does not exist
33 * yet allocate the memory for it and return the new entry.
35 static void *get_hashentry(struct rhashtable *ht, void *key,
36 const struct rhashtable_params params, size_t size)
40 result = rhashtable_lookup_fast(ht, key, params);
45 result = kzalloc(size, GFP_KERNEL);
47 return ERR_PTR(-ENOMEM);
52 bool is_pre_ct_flow(struct flow_cls_offload *flow)
54 struct flow_action_entry *act;
57 flow_action_for_each(i, act, &flow->rule->action) {
58 if (act->id == FLOW_ACTION_CT && !act->ct.action)
64 bool is_post_ct_flow(struct flow_cls_offload *flow)
66 struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
67 struct flow_dissector *dissector = rule->match.dissector;
68 struct flow_match_ct ct;
70 if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
71 flow_rule_match_ct(rule, &ct);
72 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
78 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
79 struct nfp_fl_ct_flow_entry *entry2)
81 unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys &
82 entry2->rule->match.dissector->used_keys;
85 /* check the overlapped fields one by one, the unmasked part
86 * should not conflict with each other.
88 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) {
89 struct flow_match_control match1, match2;
91 flow_rule_match_control(entry1->rule, &match1);
92 flow_rule_match_control(entry2->rule, &match2);
93 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
98 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) {
99 struct flow_match_basic match1, match2;
101 flow_rule_match_basic(entry1->rule, &match1);
102 flow_rule_match_basic(entry2->rule, &match2);
103 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
108 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
109 struct flow_match_ipv4_addrs match1, match2;
111 flow_rule_match_ipv4_addrs(entry1->rule, &match1);
112 flow_rule_match_ipv4_addrs(entry2->rule, &match2);
113 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
118 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
119 struct flow_match_ipv6_addrs match1, match2;
121 flow_rule_match_ipv6_addrs(entry1->rule, &match1);
122 flow_rule_match_ipv6_addrs(entry2->rule, &match2);
123 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
128 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) {
129 struct flow_match_ports match1, match2;
131 flow_rule_match_ports(entry1->rule, &match1);
132 flow_rule_match_ports(entry2->rule, &match2);
133 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
138 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
139 struct flow_match_eth_addrs match1, match2;
141 flow_rule_match_eth_addrs(entry1->rule, &match1);
142 flow_rule_match_eth_addrs(entry2->rule, &match2);
143 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
148 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) {
149 struct flow_match_vlan match1, match2;
151 flow_rule_match_vlan(entry1->rule, &match1);
152 flow_rule_match_vlan(entry2->rule, &match2);
153 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
158 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) {
159 struct flow_match_mpls match1, match2;
161 flow_rule_match_mpls(entry1->rule, &match1);
162 flow_rule_match_mpls(entry2->rule, &match2);
163 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
168 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) {
169 struct flow_match_tcp match1, match2;
171 flow_rule_match_tcp(entry1->rule, &match1);
172 flow_rule_match_tcp(entry2->rule, &match2);
173 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
178 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) {
179 struct flow_match_ip match1, match2;
181 flow_rule_match_ip(entry1->rule, &match1);
182 flow_rule_match_ip(entry2->rule, &match2);
183 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
188 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
189 struct flow_match_enc_keyid match1, match2;
191 flow_rule_match_enc_keyid(entry1->rule, &match1);
192 flow_rule_match_enc_keyid(entry2->rule, &match2);
193 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
198 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
199 struct flow_match_ipv4_addrs match1, match2;
201 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
202 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
203 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
208 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
209 struct flow_match_ipv6_addrs match1, match2;
211 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
212 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
213 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
218 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
219 struct flow_match_control match1, match2;
221 flow_rule_match_enc_control(entry1->rule, &match1);
222 flow_rule_match_enc_control(entry2->rule, &match2);
223 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
228 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) {
229 struct flow_match_ip match1, match2;
231 flow_rule_match_enc_ip(entry1->rule, &match1);
232 flow_rule_match_enc_ip(entry2->rule, &match2);
233 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
238 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
239 struct flow_match_enc_opts match1, match2;
241 flow_rule_match_enc_opts(entry1->rule, &match1);
242 flow_rule_match_enc_opts(entry2->rule, &match2);
243 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
254 static int nfp_ct_check_mangle_merge(struct flow_action_entry *a_in,
255 struct flow_rule *rule)
257 enum flow_action_mangle_base htype = a_in->mangle.htype;
258 u32 offset = a_in->mangle.offset;
261 case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
262 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS))
265 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
266 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
267 struct flow_match_ip match;
269 flow_rule_match_ip(rule, &match);
270 if (offset == offsetof(struct iphdr, ttl) &&
273 if (offset == round_down(offsetof(struct iphdr, tos), 4) &&
277 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
278 struct flow_match_ipv4_addrs match;
280 flow_rule_match_ipv4_addrs(rule, &match);
281 if (offset == offsetof(struct iphdr, saddr) &&
284 if (offset == offsetof(struct iphdr, daddr) &&
289 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
290 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
291 struct flow_match_ip match;
293 flow_rule_match_ip(rule, &match);
294 if (offset == round_down(offsetof(struct ipv6hdr, hop_limit), 4) &&
297 /* for ipv6, tos and flow_lbl are in the same word */
298 if (offset == round_down(offsetof(struct ipv6hdr, flow_lbl), 4) &&
302 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
303 struct flow_match_ipv6_addrs match;
305 flow_rule_match_ipv6_addrs(rule, &match);
306 if (offset >= offsetof(struct ipv6hdr, saddr) &&
307 offset < offsetof(struct ipv6hdr, daddr) &&
308 memchr_inv(&match.mask->src, 0, sizeof(match.mask->src)))
310 if (offset >= offsetof(struct ipv6hdr, daddr) &&
311 offset < sizeof(struct ipv6hdr) &&
312 memchr_inv(&match.mask->dst, 0, sizeof(match.mask->dst)))
316 case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
317 case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
318 /* currently only can modify ports */
319 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS))
328 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
329 struct nfp_fl_ct_flow_entry *post_ct_entry,
330 struct nfp_fl_ct_flow_entry *nft_entry)
332 struct flow_action_entry *act;
335 /* Check for pre_ct->action conflicts */
336 flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
338 case FLOW_ACTION_MANGLE:
339 err = nfp_ct_check_mangle_merge(act, nft_entry->rule);
342 err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
346 case FLOW_ACTION_VLAN_PUSH:
347 case FLOW_ACTION_VLAN_POP:
348 case FLOW_ACTION_VLAN_MANGLE:
349 case FLOW_ACTION_MPLS_PUSH:
350 case FLOW_ACTION_MPLS_POP:
351 case FLOW_ACTION_MPLS_MANGLE:
358 /* Check for nft->action conflicts */
359 flow_action_for_each(i, act, &nft_entry->rule->action) {
361 case FLOW_ACTION_MANGLE:
362 err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
366 case FLOW_ACTION_VLAN_PUSH:
367 case FLOW_ACTION_VLAN_POP:
368 case FLOW_ACTION_VLAN_MANGLE:
369 case FLOW_ACTION_MPLS_PUSH:
370 case FLOW_ACTION_MPLS_POP:
371 case FLOW_ACTION_MPLS_MANGLE:
380 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
381 struct nfp_fl_ct_flow_entry *nft_entry)
383 struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
384 struct flow_action_entry *ct_met;
385 struct flow_match_ct ct;
388 ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
389 if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) {
392 act_lbl = ct_met->ct_metadata.labels;
393 flow_rule_match_ct(post_ct_entry->rule, &ct);
394 for (i = 0; i < 4; i++) {
395 if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
396 (act_lbl[i] & ct.mask->ct_labels[i]))
400 if ((ct.key->ct_mark & ct.mask->ct_mark) ^
401 (ct_met->ct_metadata.mark & ct.mask->ct_mark))
410 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
415 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
416 struct net_device *netdev)
421 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
422 struct nfp_fl_ct_flow_entry *nft_entry,
423 struct nfp_fl_ct_tc_merge *tc_m_entry)
425 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
426 struct nfp_fl_nft_tc_merge *nft_m_entry;
427 unsigned long new_cookie[3];
430 pre_ct_entry = tc_m_entry->pre_ct_parent;
431 post_ct_entry = tc_m_entry->post_ct_parent;
433 err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
437 /* Check that the two tc flows are also compatible with
438 * the nft entry. No need to check the pre_ct and post_ct
439 * entries as that was already done during pre_merge.
440 * The nft entry does not have a netdev or chain populated, so
443 err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
446 err = nfp_ct_merge_check(post_ct_entry, nft_entry);
449 err = nfp_ct_check_meta(post_ct_entry, nft_entry);
453 /* Combine tc_merge and nft cookies for this cookie. */
454 new_cookie[0] = tc_m_entry->cookie[0];
455 new_cookie[1] = tc_m_entry->cookie[1];
456 new_cookie[2] = nft_entry->cookie;
457 nft_m_entry = get_hashentry(&zt->nft_merge_tb,
459 nfp_nft_ct_merge_params,
460 sizeof(*nft_m_entry));
462 if (IS_ERR(nft_m_entry))
463 return PTR_ERR(nft_m_entry);
465 /* nft_m_entry already present, not merging again */
466 if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
469 memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
470 nft_m_entry->zt = zt;
471 nft_m_entry->tc_m_parent = tc_m_entry;
472 nft_m_entry->nft_parent = nft_entry;
473 nft_m_entry->tc_flower_cookie = 0;
474 /* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created
475 * it only combined them if the netdevs were the same, so can use any of them.
477 nft_m_entry->netdev = pre_ct_entry->netdev;
479 /* Add this entry to the tc_m_list and nft_flow lists */
480 list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
481 list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
483 /* Generate offload structure and send to nfp */
484 err = nfp_fl_ct_add_offload(nft_m_entry);
486 goto err_nft_ct_offload;
488 err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
489 nfp_nft_ct_merge_params);
491 goto err_nft_ct_merge_insert;
493 zt->nft_merge_count++;
497 err_nft_ct_merge_insert:
498 nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
499 nft_m_entry->netdev);
501 list_del(&nft_m_entry->tc_merge_list);
502 list_del(&nft_m_entry->nft_flow_list);
507 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
508 struct nfp_fl_ct_flow_entry *ct_entry1,
509 struct nfp_fl_ct_flow_entry *ct_entry2)
511 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
512 struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
513 struct nfp_fl_ct_tc_merge *m_entry;
514 unsigned long new_cookie[2];
517 if (ct_entry1->type == CT_TYPE_PRE_CT) {
518 pre_ct_entry = ct_entry1;
519 post_ct_entry = ct_entry2;
521 post_ct_entry = ct_entry1;
522 pre_ct_entry = ct_entry2;
525 if (post_ct_entry->netdev != pre_ct_entry->netdev)
527 /* Checks that the chain_index of the filter matches the
528 * chain_index of the GOTO action.
530 if (post_ct_entry->chain_index != pre_ct_entry->chain_index)
533 err = nfp_ct_merge_check(post_ct_entry, pre_ct_entry);
537 new_cookie[0] = pre_ct_entry->cookie;
538 new_cookie[1] = post_ct_entry->cookie;
539 m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
540 nfp_tc_ct_merge_params, sizeof(*m_entry));
542 return PTR_ERR(m_entry);
544 /* m_entry already present, not merging again */
545 if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
548 memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
550 m_entry->post_ct_parent = post_ct_entry;
551 m_entry->pre_ct_parent = pre_ct_entry;
553 /* Add this entry to the pre_ct and post_ct lists */
554 list_add(&m_entry->post_ct_list, &post_ct_entry->children);
555 list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
556 INIT_LIST_HEAD(&m_entry->children);
558 err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
559 nfp_tc_ct_merge_params);
561 goto err_ct_tc_merge_insert;
562 zt->tc_merge_count++;
564 /* Merge with existing nft flows */
565 list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
567 nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
572 err_ct_tc_merge_insert:
573 list_del(&m_entry->post_ct_list);
574 list_del(&m_entry->pre_ct_list);
580 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
581 u16 zone, bool wildcarded)
583 struct nfp_fl_ct_zone_entry *zt;
586 if (wildcarded && priv->ct_zone_wc)
587 return priv->ct_zone_wc;
590 zt = get_hashentry(&priv->ct_zone_table, &zone,
591 nfp_zone_table_params, sizeof(*zt));
593 /* If priv is set this is an existing entry, just return it */
594 if (IS_ERR(zt) || zt->priv)
597 zt = kzalloc(sizeof(*zt), GFP_KERNEL);
599 return ERR_PTR(-ENOMEM);
606 /* init the various hash tables and lists*/
607 INIT_LIST_HEAD(&zt->pre_ct_list);
608 INIT_LIST_HEAD(&zt->post_ct_list);
609 INIT_LIST_HEAD(&zt->nft_flows_list);
611 err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
613 goto err_tc_merge_tb_init;
615 err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
617 goto err_nft_merge_tb_init;
620 priv->ct_zone_wc = zt;
622 err = rhashtable_insert_fast(&priv->ct_zone_table,
624 nfp_zone_table_params);
626 goto err_zone_insert;
632 rhashtable_destroy(&zt->nft_merge_tb);
633 err_nft_merge_tb_init:
634 rhashtable_destroy(&zt->tc_merge_tb);
635 err_tc_merge_tb_init:
641 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
642 struct net_device *netdev,
643 struct flow_cls_offload *flow,
644 bool is_nft, struct netlink_ext_ack *extack)
646 struct nf_flow_match *nft_match = NULL;
647 struct nfp_fl_ct_flow_entry *entry;
648 struct nfp_fl_ct_map_entry *map;
649 struct flow_action_entry *act;
652 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
654 return ERR_PTR(-ENOMEM);
656 entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
659 goto err_pre_ct_rule;
662 /* nft flows gets destroyed after callback return, so need
663 * to do a full copy instead of just a reference.
666 nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
671 memcpy(&nft_match->dissector, flow->rule->match.dissector,
672 sizeof(nft_match->dissector));
673 memcpy(&nft_match->mask, flow->rule->match.mask,
674 sizeof(nft_match->mask));
675 memcpy(&nft_match->key, flow->rule->match.key,
676 sizeof(nft_match->key));
677 entry->rule->match.dissector = &nft_match->dissector;
678 entry->rule->match.mask = &nft_match->mask;
679 entry->rule->match.key = &nft_match->key;
681 entry->rule->match.dissector = flow->rule->match.dissector;
682 entry->rule->match.mask = flow->rule->match.mask;
683 entry->rule->match.key = flow->rule->match.key;
687 entry->netdev = netdev;
688 entry->cookie = flow->cookie;
689 entry->chain_index = flow->common.chain_index;
690 entry->tun_offset = NFP_FL_CT_NO_TUN;
692 /* Copy over action data. Unfortunately we do not get a handle to the
693 * original tcf_action data, and the flow objects gets destroyed, so we
694 * cannot just save a pointer to this either, so need to copy over the
695 * data unfortunately.
697 entry->rule->action.num_entries = flow->rule->action.num_entries;
698 flow_action_for_each(i, act, &flow->rule->action) {
699 struct flow_action_entry *new_act;
701 new_act = &entry->rule->action.entries[i];
702 memcpy(new_act, act, sizeof(struct flow_action_entry));
703 /* Entunnel is a special case, need to allocate and copy
706 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
707 struct ip_tunnel_info *tun = act->tunnel;
708 size_t tun_size = sizeof(*tun) + tun->options_len;
710 new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
711 if (!new_act->tunnel) {
713 goto err_pre_ct_tun_cp;
715 entry->tun_offset = i;
719 INIT_LIST_HEAD(&entry->children);
721 /* Now add a ct map entry to flower-priv */
722 map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
723 nfp_ct_map_params, sizeof(*map));
725 NL_SET_ERR_MSG_MOD(extack,
726 "offload error: ct map entry creation failed");
728 goto err_ct_flow_insert;
730 map->cookie = flow->cookie;
731 map->ct_entry = entry;
732 err = rhashtable_insert_fast(&zt->priv->ct_map_table,
736 NL_SET_ERR_MSG_MOD(extack,
737 "offload error: ct map entry table add failed");
746 if (entry->tun_offset != NFP_FL_CT_NO_TUN)
747 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
757 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
759 struct nfp_fl_ct_zone_entry *zt;
764 /* Flow is in HW, need to delete */
765 if (m_entry->tc_flower_cookie) {
766 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
772 WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
774 nfp_nft_ct_merge_params));
775 zt->nft_merge_count--;
776 list_del(&m_entry->tc_merge_list);
777 list_del(&m_entry->nft_flow_list);
782 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
784 struct nfp_fl_nft_tc_merge *m_entry, *tmp;
786 /* These post entries are parts of two lists, one is a list of nft_entries
787 * and the other is of from a list of tc_merge structures. Iterate
788 * through the relevant list and cleanup the entries.
792 /* Need to iterate through list of nft_flow entries*/
793 struct nfp_fl_ct_flow_entry *ct_entry = entry;
795 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
797 cleanup_nft_merge_entry(m_entry);
800 /* Need to iterate through list of tc_merged_flow entries*/
801 struct nfp_fl_ct_tc_merge *ct_entry = entry;
803 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
805 cleanup_nft_merge_entry(m_entry);
810 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
812 struct nfp_fl_ct_zone_entry *zt;
816 err = rhashtable_remove_fast(&zt->tc_merge_tb,
818 nfp_tc_ct_merge_params);
820 pr_warn("WARNING: could not remove merge_entry from hashtable\n");
821 zt->tc_merge_count--;
822 list_del(&m_ent->post_ct_list);
823 list_del(&m_ent->pre_ct_list);
825 if (!list_empty(&m_ent->children))
826 nfp_free_nft_merge_children(m_ent, false);
830 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
832 struct nfp_fl_ct_tc_merge *m_ent, *tmp;
834 switch (entry->type) {
836 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
837 nfp_del_tc_merge_entry(m_ent);
840 case CT_TYPE_POST_CT:
841 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
842 nfp_del_tc_merge_entry(m_ent);
850 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
852 list_del(&entry->list_node);
854 if (!list_empty(&entry->children)) {
855 if (entry->type == CT_TYPE_NFT)
856 nfp_free_nft_merge_children(entry, true);
858 nfp_free_tc_merge_children(entry);
861 if (entry->tun_offset != NFP_FL_CT_NO_TUN)
862 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
864 if (entry->type == CT_TYPE_NFT) {
865 struct nf_flow_match *nft_match;
867 nft_match = container_of(entry->rule->match.dissector,
868 struct nf_flow_match, dissector);
876 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
877 enum flow_action_id act_id)
879 struct flow_action_entry *act = NULL;
882 flow_action_for_each(i, act, &rule->action) {
883 if (act->id == act_id)
890 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
891 struct nfp_fl_ct_zone_entry *zt_src,
892 struct nfp_fl_ct_zone_entry *zt_dst)
894 struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
895 struct list_head *ct_list;
897 if (ct_entry1->type == CT_TYPE_PRE_CT)
898 ct_list = &zt_src->post_ct_list;
899 else if (ct_entry1->type == CT_TYPE_POST_CT)
900 ct_list = &zt_src->pre_ct_list;
904 list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
906 nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
911 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
912 struct nfp_fl_ct_zone_entry *zt)
914 struct nfp_fl_ct_tc_merge *tc_merge_entry;
915 struct rhashtable_iter iter;
917 rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
918 rhashtable_walk_start(&iter);
919 while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
920 if (IS_ERR(tc_merge_entry))
922 rhashtable_walk_stop(&iter);
923 nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
924 rhashtable_walk_start(&iter);
926 rhashtable_walk_stop(&iter);
927 rhashtable_walk_exit(&iter);
930 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
931 struct net_device *netdev,
932 struct flow_cls_offload *flow,
933 struct netlink_ext_ack *extack)
935 struct flow_action_entry *ct_act, *ct_goto;
936 struct nfp_fl_ct_flow_entry *ct_entry;
937 struct nfp_fl_ct_zone_entry *zt;
940 ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT);
942 NL_SET_ERR_MSG_MOD(extack,
943 "unsupported offload: Conntrack action empty in conntrack offload");
947 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
949 NL_SET_ERR_MSG_MOD(extack,
950 "unsupported offload: Conntrack requires ACTION_GOTO");
954 zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
956 NL_SET_ERR_MSG_MOD(extack,
957 "offload error: Could not create zone table entry");
962 zt->nft = ct_act->ct.flow_table;
963 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
965 NL_SET_ERR_MSG_MOD(extack,
966 "offload error: Could not register nft_callback");
971 /* Add entry to pre_ct_list */
972 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
973 if (IS_ERR(ct_entry))
974 return PTR_ERR(ct_entry);
975 ct_entry->type = CT_TYPE_PRE_CT;
976 ct_entry->chain_index = ct_goto->chain_index;
977 list_add(&ct_entry->list_node, &zt->pre_ct_list);
980 nfp_ct_merge_tc_entries(ct_entry, zt, zt);
982 /* Need to check and merge with tables in the wc_zone as well */
983 if (priv->ct_zone_wc)
984 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);
989 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
990 struct net_device *netdev,
991 struct flow_cls_offload *flow,
992 struct netlink_ext_ack *extack)
994 struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
995 struct nfp_fl_ct_flow_entry *ct_entry;
996 struct nfp_fl_ct_zone_entry *zt;
997 bool wildcarded = false;
998 struct flow_match_ct ct;
1000 flow_rule_match_ct(rule, &ct);
1001 if (!ct.mask->ct_zone) {
1003 } else if (ct.mask->ct_zone != U16_MAX) {
1004 NL_SET_ERR_MSG_MOD(extack,
1005 "unsupported offload: partially wildcarded ct_zone is not supported");
1009 zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
1011 NL_SET_ERR_MSG_MOD(extack,
1012 "offload error: Could not create zone table entry");
1016 /* Add entry to post_ct_list */
1017 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1018 if (IS_ERR(ct_entry))
1019 return PTR_ERR(ct_entry);
1021 ct_entry->type = CT_TYPE_POST_CT;
1022 ct_entry->chain_index = flow->common.chain_index;
1023 list_add(&ct_entry->list_node, &zt->post_ct_list);
1024 zt->post_ct_count++;
1027 /* Iterate through all zone tables if not empty, look for merges with
1028 * pre_ct entries and merge them.
1030 struct rhashtable_iter iter;
1031 struct nfp_fl_ct_zone_entry *zone_table;
1033 rhashtable_walk_enter(&priv->ct_zone_table, &iter);
1034 rhashtable_walk_start(&iter);
1035 while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
1036 if (IS_ERR(zone_table))
1038 rhashtable_walk_stop(&iter);
1039 nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
1040 rhashtable_walk_start(&iter);
1042 rhashtable_walk_stop(&iter);
1043 rhashtable_walk_exit(&iter);
1045 nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1052 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
1054 struct nfp_fl_ct_map_entry *ct_map_ent;
1055 struct nfp_fl_ct_flow_entry *ct_entry;
1056 struct netlink_ext_ack *extack = NULL;
1060 extack = flow->common.extack;
1061 switch (flow->command) {
1062 case FLOW_CLS_REPLACE:
1063 /* Netfilter can request offload multiple times for the same
1064 * flow - protect against adding duplicates.
1066 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1069 ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
1070 if (IS_ERR(ct_entry))
1071 return PTR_ERR(ct_entry);
1072 ct_entry->type = CT_TYPE_NFT;
1073 list_add(&ct_entry->list_node, &zt->nft_flows_list);
1074 zt->nft_flows_count++;
1075 nfp_ct_merge_nft_with_tc(ct_entry, zt);
1078 case FLOW_CLS_DESTROY:
1079 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1081 return nfp_fl_ct_del_flow(ct_map_ent);
1082 case FLOW_CLS_STATS:
1090 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
1092 struct flow_cls_offload *flow = type_data;
1093 struct nfp_fl_ct_zone_entry *zt = cb_priv;
1094 int err = -EOPNOTSUPP;
1097 case TC_SETUP_CLSFLOWER:
1099 err = nfp_fl_ct_offload_nft_flow(zt, flow);
1109 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
1111 struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
1112 struct nfp_fl_ct_map_entry *ct_map_ent;
1114 list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
1116 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
1119 nfp_fl_ct_del_flow(ct_map_ent);
1123 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
1125 struct nfp_fl_ct_flow_entry *ct_entry;
1126 struct nfp_fl_ct_zone_entry *zt;
1127 struct rhashtable *m_table;
1132 zt = ct_map_ent->ct_entry->zt;
1133 ct_entry = ct_map_ent->ct_entry;
1134 m_table = &zt->priv->ct_map_table;
1136 switch (ct_entry->type) {
1137 case CT_TYPE_PRE_CT:
1139 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1141 nfp_fl_ct_clean_flow_entry(ct_entry);
1144 /* If this is the last pre_ct_rule it means that it is
1145 * very likely that the nft table will be cleaned up next,
1146 * as this happens on the removal of the last act_ct flow.
1147 * However we cannot deregister the callback on the removal
1148 * of the last nft flow as this runs into a deadlock situation.
1149 * So deregister the callback on removal of the last pre_ct flow
1150 * and remove any remaining nft flow entries. We also cannot
1151 * save this state and delete the callback later since the
1152 * nft table would already have been freed at that time.
1154 if (!zt->pre_ct_count) {
1155 nf_flow_table_offload_del_cb(zt->nft,
1156 nfp_fl_ct_handle_nft_flow,
1159 nfp_fl_ct_clean_nft_entries(zt);
1162 case CT_TYPE_POST_CT:
1163 zt->post_ct_count--;
1164 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1166 nfp_fl_ct_clean_flow_entry(ct_entry);
1170 zt->nft_flows_count--;
1171 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1173 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);