drivers/net/ethernet/netronome/nfp/flower/conntrack.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2021 Corigine, Inc. */
   3
   4 #include "conntrack.h"
   5
   6 const struct rhashtable_params nfp_tc_ct_merge_params = {
   7         .head_offset            = offsetof(struct nfp_fl_ct_tc_merge,
   8                                            hash_node),
   9         .key_len                = sizeof(unsigned long) * 2,
  10         .key_offset             = offsetof(struct nfp_fl_ct_tc_merge, cookie),
  11         .automatic_shrinking    = true,
  12 };
  13
  14 const struct rhashtable_params nfp_nft_ct_merge_params = {
  15         .head_offset            = offsetof(struct nfp_fl_nft_tc_merge,
  16                                            hash_node),
  17         .key_len                = sizeof(unsigned long) * 3,
  18         .key_offset             = offsetof(struct nfp_fl_nft_tc_merge, cookie),
  19         .automatic_shrinking    = true,
  20 };
  21
  22 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
  23                                               enum flow_action_id act_id);
  24
  25 /**
  26  * get_hashentry() - Wrapper around hashtable lookup.
  27  * @ht:         hashtable where entry could be found
  28  * @key:        key to lookup
  29  * @params:     hashtable params
  30  * @size:       size of entry to allocate if not in table
  31  *
  32  * Returns an entry from a hashtable. If entry does not exist
  33  * yet allocate the memory for it and return the new entry.
  34  */
  35 static void *get_hashentry(struct rhashtable *ht, void *key,
  36                            const struct rhashtable_params params, size_t size)
  37 {
  38         void *result;
  39
  40         result = rhashtable_lookup_fast(ht, key, params);
  41
  42         if (result)
  43                 return result;
  44
  45         result = kzalloc(size, GFP_KERNEL);
  46         if (!result)
  47                 return ERR_PTR(-ENOMEM);
  48
  49         return result;
  50 }
  51
  52 bool is_pre_ct_flow(struct flow_cls_offload *flow)
  53 {
  54         struct flow_action_entry *act;
  55         int i;
  56
  57         flow_action_for_each(i, act, &flow->rule->action) {
  58                 if (act->id == FLOW_ACTION_CT && !act->ct.action)
  59                         return true;
  60         }
  61         return false;
  62 }
  63
  64 bool is_post_ct_flow(struct flow_cls_offload *flow)
  65 {
  66         struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
  67         struct flow_dissector *dissector = rule->match.dissector;
  68         struct flow_match_ct ct;
  69
  70         if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
  71                 flow_rule_match_ct(rule, &ct);
  72                 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
  73                         return true;
  74         }
  75         return false;
  76 }
  77
  78 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
  79                               struct nfp_fl_ct_flow_entry *entry2)
  80 {
  81         unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys &
  82                                  entry2->rule->match.dissector->used_keys;
  83         bool out;
  84
  85         /* check the overlapped fields one by one, the unmasked part
  86          * should not conflict with each other.
  87          */
  88         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) {
  89                 struct flow_match_control match1, match2;
  90
  91                 flow_rule_match_control(entry1->rule, &match1);
  92                 flow_rule_match_control(entry2->rule, &match2);
  93                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
  94                 if (out)
  95                         goto check_failed;
  96         }
  97
  98         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) {
  99                 struct flow_match_basic match1, match2;
 100
 101                 flow_rule_match_basic(entry1->rule, &match1);
 102                 flow_rule_match_basic(entry2->rule, &match2);
 103                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 104                 if (out)
 105                         goto check_failed;
 106         }
 107
 108         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 109                 struct flow_match_ipv4_addrs match1, match2;
 110
 111                 flow_rule_match_ipv4_addrs(entry1->rule, &match1);
 112                 flow_rule_match_ipv4_addrs(entry2->rule, &match2);
 113                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 114                 if (out)
 115                         goto check_failed;
 116         }
 117
 118         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 119                 struct flow_match_ipv6_addrs match1, match2;
 120
 121                 flow_rule_match_ipv6_addrs(entry1->rule, &match1);
 122                 flow_rule_match_ipv6_addrs(entry2->rule, &match2);
 123                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 124                 if (out)
 125                         goto check_failed;
 126         }
 127
 128         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) {
 129                 struct flow_match_ports match1, match2;
 130
 131                 flow_rule_match_ports(entry1->rule, &match1);
 132                 flow_rule_match_ports(entry2->rule, &match2);
 133                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 134                 if (out)
 135                         goto check_failed;
 136         }
 137
 138         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 139                 struct flow_match_eth_addrs match1, match2;
 140
 141                 flow_rule_match_eth_addrs(entry1->rule, &match1);
 142                 flow_rule_match_eth_addrs(entry2->rule, &match2);
 143                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 144                 if (out)
 145                         goto check_failed;
 146         }
 147
 148         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) {
 149                 struct flow_match_vlan match1, match2;
 150
 151                 flow_rule_match_vlan(entry1->rule, &match1);
 152                 flow_rule_match_vlan(entry2->rule, &match2);
 153                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 154                 if (out)
 155                         goto check_failed;
 156         }
 157
 158         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) {
 159                 struct flow_match_mpls match1, match2;
 160
 161                 flow_rule_match_mpls(entry1->rule, &match1);
 162                 flow_rule_match_mpls(entry2->rule, &match2);
 163                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 164                 if (out)
 165                         goto check_failed;
 166         }
 167
 168         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) {
 169                 struct flow_match_tcp match1, match2;
 170
 171                 flow_rule_match_tcp(entry1->rule, &match1);
 172                 flow_rule_match_tcp(entry2->rule, &match2);
 173                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 174                 if (out)
 175                         goto check_failed;
 176         }
 177
 178         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) {
 179                 struct flow_match_ip match1, match2;
 180
 181                 flow_rule_match_ip(entry1->rule, &match1);
 182                 flow_rule_match_ip(entry2->rule, &match2);
 183                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 184                 if (out)
 185                         goto check_failed;
 186         }
 187
 188         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 189                 struct flow_match_enc_keyid match1, match2;
 190
 191                 flow_rule_match_enc_keyid(entry1->rule, &match1);
 192                 flow_rule_match_enc_keyid(entry2->rule, &match2);
 193                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 194                 if (out)
 195                         goto check_failed;
 196         }
 197
 198         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
 199                 struct flow_match_ipv4_addrs match1, match2;
 200
 201                 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1);
 202                 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2);
 203                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 204                 if (out)
 205                         goto check_failed;
 206         }
 207
 208         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
 209                 struct flow_match_ipv6_addrs match1, match2;
 210
 211                 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1);
 212                 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2);
 213                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 214                 if (out)
 215                         goto check_failed;
 216         }
 217
 218         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
 219                 struct flow_match_control match1, match2;
 220
 221                 flow_rule_match_enc_control(entry1->rule, &match1);
 222                 flow_rule_match_enc_control(entry2->rule, &match2);
 223                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 224                 if (out)
 225                         goto check_failed;
 226         }
 227
 228         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) {
 229                 struct flow_match_ip match1, match2;
 230
 231                 flow_rule_match_enc_ip(entry1->rule, &match1);
 232                 flow_rule_match_enc_ip(entry2->rule, &match2);
 233                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 234                 if (out)
 235                         goto check_failed;
 236         }
 237
 238         if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
 239                 struct flow_match_enc_opts match1, match2;
 240
 241                 flow_rule_match_enc_opts(entry1->rule, &match1);
 242                 flow_rule_match_enc_opts(entry2->rule, &match2);
 243                 COMPARE_UNMASKED_FIELDS(match1, match2, &out);
 244                 if (out)
 245                         goto check_failed;
 246         }
 247
 248         return 0;
 249
 250 check_failed:
 251         return -EINVAL;
 252 }
 253
 254 static int nfp_ct_check_mangle_merge(struct flow_action_entry *a_in,
 255                                      struct flow_rule *rule)
 256 {
 257         enum flow_action_mangle_base htype = a_in->mangle.htype;
 258         u32 offset = a_in->mangle.offset;
 259
 260         switch (htype) {
 261         case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
 262                 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS))
 263                         return -EOPNOTSUPP;
 264                 break;
 265         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 266                 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
 267                         struct flow_match_ip match;
 268
 269                         flow_rule_match_ip(rule, &match);
 270                         if (offset == offsetof(struct iphdr, ttl) &&
 271                             match.mask->ttl)
 272                                 return -EOPNOTSUPP;
 273                         if (offset == round_down(offsetof(struct iphdr, tos), 4) &&
 274                             match.mask->tos)
 275                                 return -EOPNOTSUPP;
 276                 }
 277                 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 278                         struct flow_match_ipv4_addrs match;
 279
 280                         flow_rule_match_ipv4_addrs(rule, &match);
 281                         if (offset == offsetof(struct iphdr, saddr) &&
 282                             match.mask->src)
 283                                 return -EOPNOTSUPP;
 284                         if (offset == offsetof(struct iphdr, daddr) &&
 285                             match.mask->dst)
 286                                 return -EOPNOTSUPP;
 287                 }
 288                 break;
 289         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 290                 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
 291                         struct flow_match_ip match;
 292
 293                         flow_rule_match_ip(rule, &match);
 294                         if (offset == round_down(offsetof(struct ipv6hdr, hop_limit), 4) &&
 295                             match.mask->ttl)
 296                                 return -EOPNOTSUPP;
 297                         /* for ipv6, tos and flow_lbl are in the same word */
 298                         if (offset == round_down(offsetof(struct ipv6hdr, flow_lbl), 4) &&
 299                             match.mask->tos)
 300                                 return -EOPNOTSUPP;
 301                 }
 302                 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 303                         struct flow_match_ipv6_addrs match;
 304
 305                         flow_rule_match_ipv6_addrs(rule, &match);
 306                         if (offset >= offsetof(struct ipv6hdr, saddr) &&
 307                             offset < offsetof(struct ipv6hdr, daddr) &&
 308                             memchr_inv(&match.mask->src, 0, sizeof(match.mask->src)))
 309                                 return -EOPNOTSUPP;
 310                         if (offset >= offsetof(struct ipv6hdr, daddr) &&
 311                             offset < sizeof(struct ipv6hdr) &&
 312                             memchr_inv(&match.mask->dst, 0, sizeof(match.mask->dst)))
 313                                 return -EOPNOTSUPP;
 314                 }
 315                 break;
 316         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 317         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 318                 /* currently only can modify ports */
 319                 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS))
 320                         return -EOPNOTSUPP;
 321                 break;
 322         default:
 323                 break;
 324         }
 325         return 0;
 326 }
 327
 328 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
 329                                   struct nfp_fl_ct_flow_entry *post_ct_entry,
 330                                   struct nfp_fl_ct_flow_entry *nft_entry)
 331 {
 332         struct flow_action_entry *act;
 333         int err, i;
 334
 335         /* Check for pre_ct->action conflicts */
 336         flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
 337                 switch (act->id) {
 338                 case FLOW_ACTION_MANGLE:
 339                         err = nfp_ct_check_mangle_merge(act, nft_entry->rule);
 340                         if (err)
 341                                 return err;
 342                         err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
 343                         if (err)
 344                                 return err;
 345                         break;
 346                 case FLOW_ACTION_VLAN_PUSH:
 347                 case FLOW_ACTION_VLAN_POP:
 348                 case FLOW_ACTION_VLAN_MANGLE:
 349                 case FLOW_ACTION_MPLS_PUSH:
 350                 case FLOW_ACTION_MPLS_POP:
 351                 case FLOW_ACTION_MPLS_MANGLE:
 352                         return -EOPNOTSUPP;
 353                 default:
 354                         break;
 355                 }
 356         }
 357
 358         /* Check for nft->action conflicts */
 359         flow_action_for_each(i, act, &nft_entry->rule->action) {
 360                 switch (act->id) {
 361                 case FLOW_ACTION_MANGLE:
 362                         err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule);
 363                         if (err)
 364                                 return err;
 365                         break;
 366                 case FLOW_ACTION_VLAN_PUSH:
 367                 case FLOW_ACTION_VLAN_POP:
 368                 case FLOW_ACTION_VLAN_MANGLE:
 369                 case FLOW_ACTION_MPLS_PUSH:
 370                 case FLOW_ACTION_MPLS_POP:
 371                 case FLOW_ACTION_MPLS_MANGLE:
 372                         return -EOPNOTSUPP;
 373                 default:
 374                         break;
 375                 }
 376         }
 377         return 0;
 378 }
 379
 380 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
 381                              struct nfp_fl_ct_flow_entry *nft_entry)
 382 {
 383         struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
 384         struct flow_action_entry *ct_met;
 385         struct flow_match_ct ct;
 386         int i;
 387
 388         ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA);
 389         if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) {
 390                 u32 *act_lbl;
 391
 392                 act_lbl = ct_met->ct_metadata.labels;
 393                 flow_rule_match_ct(post_ct_entry->rule, &ct);
 394                 for (i = 0; i < 4; i++) {
 395                         if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
 396                             (act_lbl[i] & ct.mask->ct_labels[i]))
 397                                 return -EINVAL;
 398                 }
 399
 400                 if ((ct.key->ct_mark & ct.mask->ct_mark) ^
 401                     (ct_met->ct_metadata.mark & ct.mask->ct_mark))
 402                         return -EINVAL;
 403
 404                 return 0;
 405         }
 406
 407         return -EINVAL;
 408 }
 409
 410 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
 411 {
 412         return 0;
 413 }
 414
 415 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
 416                                  struct net_device *netdev)
 417 {
 418         return 0;
 419 }
 420
 421 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
 422                                struct nfp_fl_ct_flow_entry *nft_entry,
 423                                struct nfp_fl_ct_tc_merge *tc_m_entry)
 424 {
 425         struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
 426         struct nfp_fl_nft_tc_merge *nft_m_entry;
 427         unsigned long new_cookie[3];
 428         int err;
 429
 430         pre_ct_entry = tc_m_entry->pre_ct_parent;
 431         post_ct_entry = tc_m_entry->post_ct_parent;
 432
 433         err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
 434         if (err)
 435                 return err;
 436
 437         /* Check that the two tc flows are also compatible with
 438          * the nft entry. No need to check the pre_ct and post_ct
 439          * entries as that was already done during pre_merge.
 440          * The nft entry does not have a netdev or chain populated, so
 441          * skip this check.
 442          */
 443         err = nfp_ct_merge_check(pre_ct_entry, nft_entry);
 444         if (err)
 445                 return err;
 446         err = nfp_ct_merge_check(post_ct_entry, nft_entry);
 447         if (err)
 448                 return err;
 449         err = nfp_ct_check_meta(post_ct_entry, nft_entry);
 450         if (err)
 451                 return err;
 452
 453         /* Combine tc_merge and nft cookies for this cookie. */
 454         new_cookie[0] = tc_m_entry->cookie[0];
 455         new_cookie[1] = tc_m_entry->cookie[1];
 456         new_cookie[2] = nft_entry->cookie;
 457         nft_m_entry = get_hashentry(&zt->nft_merge_tb,
 458                                     &new_cookie,
 459                                     nfp_nft_ct_merge_params,
 460                                     sizeof(*nft_m_entry));
 461
 462         if (IS_ERR(nft_m_entry))
 463                 return PTR_ERR(nft_m_entry);
 464
 465         /* nft_m_entry already present, not merging again */
 466         if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie)))
 467                 return 0;
 468
 469         memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
 470         nft_m_entry->zt = zt;
 471         nft_m_entry->tc_m_parent = tc_m_entry;
 472         nft_m_entry->nft_parent = nft_entry;
 473         nft_m_entry->tc_flower_cookie = 0;
 474         /* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created
 475          * it only combined them if the netdevs were the same, so can use any of them.
 476          */
 477         nft_m_entry->netdev = pre_ct_entry->netdev;
 478
 479         /* Add this entry to the tc_m_list and nft_flow lists */
 480         list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children);
 481         list_add(&nft_m_entry->nft_flow_list, &nft_entry->children);
 482
 483         /* Generate offload structure and send to nfp */
 484         err = nfp_fl_ct_add_offload(nft_m_entry);
 485         if (err)
 486                 goto err_nft_ct_offload;
 487
 488         err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,
 489                                      nfp_nft_ct_merge_params);
 490         if (err)
 491                 goto err_nft_ct_merge_insert;
 492
 493         zt->nft_merge_count++;
 494
 495         return err;
 496
 497 err_nft_ct_merge_insert:
 498         nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,
 499                               nft_m_entry->netdev);
 500 err_nft_ct_offload:
 501         list_del(&nft_m_entry->tc_merge_list);
 502         list_del(&nft_m_entry->nft_flow_list);
 503         kfree(nft_m_entry);
 504         return err;
 505 }
 506
 507 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
 508                               struct nfp_fl_ct_flow_entry *ct_entry1,
 509                               struct nfp_fl_ct_flow_entry *ct_entry2)
 510 {
 511         struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
 512         struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
 513         struct nfp_fl_ct_tc_merge *m_entry;
 514         unsigned long new_cookie[2];
 515         int err;
 516
 517         if (ct_entry1->type == CT_TYPE_PRE_CT) {
 518                 pre_ct_entry = ct_entry1;
 519                 post_ct_entry = ct_entry2;
 520         } else {
 521                 post_ct_entry = ct_entry1;
 522                 pre_ct_entry = ct_entry2;
 523         }
 524
 525         if (post_ct_entry->netdev != pre_ct_entry->netdev)
 526                 return -EINVAL;
 527         /* Checks that the chain_index of the filter matches the
 528          * chain_index of the GOTO action.
 529          */
 530         if (post_ct_entry->chain_index != pre_ct_entry->chain_index)
 531                 return -EINVAL;
 532
 533         err = nfp_ct_merge_check(post_ct_entry, pre_ct_entry);
 534         if (err)
 535                 return err;
 536
 537         new_cookie[0] = pre_ct_entry->cookie;
 538         new_cookie[1] = post_ct_entry->cookie;
 539         m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,
 540                                 nfp_tc_ct_merge_params, sizeof(*m_entry));
 541         if (IS_ERR(m_entry))
 542                 return PTR_ERR(m_entry);
 543
 544         /* m_entry already present, not merging again */
 545         if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie)))
 546                 return 0;
 547
 548         memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
 549         m_entry->zt = zt;
 550         m_entry->post_ct_parent = post_ct_entry;
 551         m_entry->pre_ct_parent = pre_ct_entry;
 552
 553         /* Add this entry to the pre_ct and post_ct lists */
 554         list_add(&m_entry->post_ct_list, &post_ct_entry->children);
 555         list_add(&m_entry->pre_ct_list, &pre_ct_entry->children);
 556         INIT_LIST_HEAD(&m_entry->children);
 557
 558         err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,
 559                                      nfp_tc_ct_merge_params);
 560         if (err)
 561                 goto err_ct_tc_merge_insert;
 562         zt->tc_merge_count++;
 563
 564         /* Merge with existing nft flows */
 565         list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
 566                                  list_node) {
 567                 nfp_ct_do_nft_merge(zt, nft_entry, m_entry);
 568         }
 569
 570         return 0;
 571
 572 err_ct_tc_merge_insert:
 573         list_del(&m_entry->post_ct_list);
 574         list_del(&m_entry->pre_ct_list);
 575         kfree(m_entry);
 576         return err;
 577 }
 578
 579 static struct
 580 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
 581                                          u16 zone, bool wildcarded)
 582 {
 583         struct nfp_fl_ct_zone_entry *zt;
 584         int err;
 585
 586         if (wildcarded && priv->ct_zone_wc)
 587                 return priv->ct_zone_wc;
 588
 589         if (!wildcarded) {
 590                 zt = get_hashentry(&priv->ct_zone_table, &zone,
 591                                    nfp_zone_table_params, sizeof(*zt));
 592
 593                 /* If priv is set this is an existing entry, just return it */
 594                 if (IS_ERR(zt) || zt->priv)
 595                         return zt;
 596         } else {
 597                 zt = kzalloc(sizeof(*zt), GFP_KERNEL);
 598                 if (!zt)
 599                         return ERR_PTR(-ENOMEM);
 600         }
 601
 602         zt->zone = zone;
 603         zt->priv = priv;
 604         zt->nft = NULL;
 605
 606         /* init the various hash tables and lists*/
 607         INIT_LIST_HEAD(&zt->pre_ct_list);
 608         INIT_LIST_HEAD(&zt->post_ct_list);
 609         INIT_LIST_HEAD(&zt->nft_flows_list);
 610
 611         err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);
 612         if (err)
 613                 goto err_tc_merge_tb_init;
 614
 615         err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);
 616         if (err)
 617                 goto err_nft_merge_tb_init;
 618
 619         if (wildcarded) {
 620                 priv->ct_zone_wc = zt;
 621         } else {
 622                 err = rhashtable_insert_fast(&priv->ct_zone_table,
 623                                              &zt->hash_node,
 624                                              nfp_zone_table_params);
 625                 if (err)
 626                         goto err_zone_insert;
 627         }
 628
 629         return zt;
 630
 631 err_zone_insert:
 632         rhashtable_destroy(&zt->nft_merge_tb);
 633 err_nft_merge_tb_init:
 634         rhashtable_destroy(&zt->tc_merge_tb);
 635 err_tc_merge_tb_init:
 636         kfree(zt);
 637         return ERR_PTR(err);
 638 }
 639
 640 static struct
 641 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
 642                                          struct net_device *netdev,
 643                                          struct flow_cls_offload *flow,
 644                                          bool is_nft, struct netlink_ext_ack *extack)
 645 {
 646         struct nf_flow_match *nft_match = NULL;
 647         struct nfp_fl_ct_flow_entry *entry;
 648         struct nfp_fl_ct_map_entry *map;
 649         struct flow_action_entry *act;
 650         int err, i;
 651
 652         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 653         if (!entry)
 654                 return ERR_PTR(-ENOMEM);
 655
 656         entry->rule = flow_rule_alloc(flow->rule->action.num_entries);
 657         if (!entry->rule) {
 658                 err = -ENOMEM;
 659                 goto err_pre_ct_rule;
 660         }
 661
 662         /* nft flows gets destroyed after callback return, so need
 663          * to do a full copy instead of just a reference.
 664          */
 665         if (is_nft) {
 666                 nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL);
 667                 if (!nft_match) {
 668                         err = -ENOMEM;
 669                         goto err_pre_ct_act;
 670                 }
 671                 memcpy(&nft_match->dissector, flow->rule->match.dissector,
 672                        sizeof(nft_match->dissector));
 673                 memcpy(&nft_match->mask, flow->rule->match.mask,
 674                        sizeof(nft_match->mask));
 675                 memcpy(&nft_match->key, flow->rule->match.key,
 676                        sizeof(nft_match->key));
 677                 entry->rule->match.dissector = &nft_match->dissector;
 678                 entry->rule->match.mask = &nft_match->mask;
 679                 entry->rule->match.key = &nft_match->key;
 680         } else {
 681                 entry->rule->match.dissector = flow->rule->match.dissector;
 682                 entry->rule->match.mask = flow->rule->match.mask;
 683                 entry->rule->match.key = flow->rule->match.key;
 684         }
 685
 686         entry->zt = zt;
 687         entry->netdev = netdev;
 688         entry->cookie = flow->cookie;
 689         entry->chain_index = flow->common.chain_index;
 690         entry->tun_offset = NFP_FL_CT_NO_TUN;
 691
 692         /* Copy over action data. Unfortunately we do not get a handle to the
 693          * original tcf_action data, and the flow objects gets destroyed, so we
 694          * cannot just save a pointer to this either, so need to copy over the
 695          * data unfortunately.
 696          */
 697         entry->rule->action.num_entries = flow->rule->action.num_entries;
 698         flow_action_for_each(i, act, &flow->rule->action) {
 699                 struct flow_action_entry *new_act;
 700
 701                 new_act = &entry->rule->action.entries[i];
 702                 memcpy(new_act, act, sizeof(struct flow_action_entry));
 703                 /* Entunnel is a special case, need to allocate and copy
 704                  * tunnel info.
 705                  */
 706                 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
 707                         struct ip_tunnel_info *tun = act->tunnel;
 708                         size_t tun_size = sizeof(*tun) + tun->options_len;
 709
 710                         new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC);
 711                         if (!new_act->tunnel) {
 712                                 err = -ENOMEM;
 713                                 goto err_pre_ct_tun_cp;
 714                         }
 715                         entry->tun_offset = i;
 716                 }
 717         }
 718
 719         INIT_LIST_HEAD(&entry->children);
 720
 721         /* Now add a ct map entry to flower-priv */
 722         map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,
 723                             nfp_ct_map_params, sizeof(*map));
 724         if (IS_ERR(map)) {
 725                 NL_SET_ERR_MSG_MOD(extack,
 726                                    "offload error: ct map entry creation failed");
 727                 err = -ENOMEM;
 728                 goto err_ct_flow_insert;
 729         }
 730         map->cookie = flow->cookie;
 731         map->ct_entry = entry;
 732         err = rhashtable_insert_fast(&zt->priv->ct_map_table,
 733                                      &map->hash_node,
 734                                      nfp_ct_map_params);
 735         if (err) {
 736                 NL_SET_ERR_MSG_MOD(extack,
 737                                    "offload error: ct map entry table add failed");
 738                 goto err_map_insert;
 739         }
 740
 741         return entry;
 742
 743 err_map_insert:
 744         kfree(map);
 745 err_ct_flow_insert:
 746         if (entry->tun_offset != NFP_FL_CT_NO_TUN)
 747                 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
 748 err_pre_ct_tun_cp:
 749         kfree(nft_match);
 750 err_pre_ct_act:
 751         kfree(entry->rule);
 752 err_pre_ct_rule:
 753         kfree(entry);
 754         return ERR_PTR(err);
 755 }
 756
 757 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
 758 {
 759         struct nfp_fl_ct_zone_entry *zt;
 760         int err;
 761
 762         zt = m_entry->zt;
 763
 764         /* Flow is in HW, need to delete */
 765         if (m_entry->tc_flower_cookie) {
 766                 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,
 767                                             m_entry->netdev);
 768                 if (err)
 769                         return;
 770         }
 771
 772         WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
 773                                             &m_entry->hash_node,
 774                                             nfp_nft_ct_merge_params));
 775         zt->nft_merge_count--;
 776         list_del(&m_entry->tc_merge_list);
 777         list_del(&m_entry->nft_flow_list);
 778
 779         kfree(m_entry);
 780 }
 781
 782 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
 783 {
 784         struct nfp_fl_nft_tc_merge *m_entry, *tmp;
 785
 786         /* These post entries are parts of two lists, one is a list of nft_entries
 787          * and the other is of from a list of tc_merge structures. Iterate
 788          * through the relevant list and cleanup the entries.
 789          */
 790
 791         if (is_nft_flow) {
 792                 /* Need to iterate through list of nft_flow entries*/
 793                 struct nfp_fl_ct_flow_entry *ct_entry = entry;
 794
 795                 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
 796                                          nft_flow_list) {
 797                         cleanup_nft_merge_entry(m_entry);
 798                 }
 799         } else {
 800                 /* Need to iterate through list of tc_merged_flow entries*/
 801                 struct nfp_fl_ct_tc_merge *ct_entry = entry;
 802
 803                 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
 804                                          tc_merge_list) {
 805                         cleanup_nft_merge_entry(m_entry);
 806                 }
 807         }
 808 }
 809
 810 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
 811 {
 812         struct nfp_fl_ct_zone_entry *zt;
 813         int err;
 814
 815         zt = m_ent->zt;
 816         err = rhashtable_remove_fast(&zt->tc_merge_tb,
 817                                      &m_ent->hash_node,
 818                                      nfp_tc_ct_merge_params);
 819         if (err)
 820                 pr_warn("WARNING: could not remove merge_entry from hashtable\n");
 821         zt->tc_merge_count--;
 822         list_del(&m_ent->post_ct_list);
 823         list_del(&m_ent->pre_ct_list);
 824
 825         if (!list_empty(&m_ent->children))
 826                 nfp_free_nft_merge_children(m_ent, false);
 827         kfree(m_ent);
 828 }
 829
 830 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
 831 {
 832         struct nfp_fl_ct_tc_merge *m_ent, *tmp;
 833
 834         switch (entry->type) {
 835         case CT_TYPE_PRE_CT:
 836                 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
 837                         nfp_del_tc_merge_entry(m_ent);
 838                 }
 839                 break;
 840         case CT_TYPE_POST_CT:
 841                 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
 842                         nfp_del_tc_merge_entry(m_ent);
 843                 }
 844                 break;
 845         default:
 846                 break;
 847         }
 848 }
 849
 850 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
 851 {
 852         list_del(&entry->list_node);
 853
 854         if (!list_empty(&entry->children)) {
 855                 if (entry->type == CT_TYPE_NFT)
 856                         nfp_free_nft_merge_children(entry, true);
 857                 else
 858                         nfp_free_tc_merge_children(entry);
 859         }
 860
 861         if (entry->tun_offset != NFP_FL_CT_NO_TUN)
 862                 kfree(entry->rule->action.entries[entry->tun_offset].tunnel);
 863
 864         if (entry->type == CT_TYPE_NFT) {
 865                 struct nf_flow_match *nft_match;
 866
 867                 nft_match = container_of(entry->rule->match.dissector,
 868                                          struct nf_flow_match, dissector);
 869                 kfree(nft_match);
 870         }
 871
 872         kfree(entry->rule);
 873         kfree(entry);
 874 }
 875
 876 static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
 877                                               enum flow_action_id act_id)
 878 {
 879         struct flow_action_entry *act = NULL;
 880         int i;
 881
 882         flow_action_for_each(i, act, &rule->action) {
 883                 if (act->id == act_id)
 884                         return act;
 885         }
 886         return NULL;
 887 }
 888
 889 static void
 890 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
 891                         struct nfp_fl_ct_zone_entry *zt_src,
 892                         struct nfp_fl_ct_zone_entry *zt_dst)
 893 {
 894         struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
 895         struct list_head *ct_list;
 896
 897         if (ct_entry1->type == CT_TYPE_PRE_CT)
 898                 ct_list = &zt_src->post_ct_list;
 899         else if (ct_entry1->type == CT_TYPE_POST_CT)
 900                 ct_list = &zt_src->pre_ct_list;
 901         else
 902                 return;
 903
 904         list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
 905                                  list_node) {
 906                 nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1);
 907         }
 908 }
 909
 910 static void
 911 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
 912                          struct nfp_fl_ct_zone_entry *zt)
 913 {
 914         struct nfp_fl_ct_tc_merge *tc_merge_entry;
 915         struct rhashtable_iter iter;
 916
 917         rhashtable_walk_enter(&zt->tc_merge_tb, &iter);
 918         rhashtable_walk_start(&iter);
 919         while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) {
 920                 if (IS_ERR(tc_merge_entry))
 921                         continue;
 922                 rhashtable_walk_stop(&iter);
 923                 nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);
 924                 rhashtable_walk_start(&iter);
 925         }
 926         rhashtable_walk_stop(&iter);
 927         rhashtable_walk_exit(&iter);
 928 }
 929
 930 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
 931                             struct net_device *netdev,
 932                             struct flow_cls_offload *flow,
 933                             struct netlink_ext_ack *extack)
 934 {
 935         struct flow_action_entry *ct_act, *ct_goto;
 936         struct nfp_fl_ct_flow_entry *ct_entry;
 937         struct nfp_fl_ct_zone_entry *zt;
 938         int err;
 939
 940         ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT);
 941         if (!ct_act) {
 942                 NL_SET_ERR_MSG_MOD(extack,
 943                                    "unsupported offload: Conntrack action empty in conntrack offload");
 944                 return -EOPNOTSUPP;
 945         }
 946
 947         ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO);
 948         if (!ct_goto) {
 949                 NL_SET_ERR_MSG_MOD(extack,
 950                                    "unsupported offload: Conntrack requires ACTION_GOTO");
 951                 return -EOPNOTSUPP;
 952         }
 953
 954         zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);
 955         if (IS_ERR(zt)) {
 956                 NL_SET_ERR_MSG_MOD(extack,
 957                                    "offload error: Could not create zone table entry");
 958                 return PTR_ERR(zt);
 959         }
 960
 961         if (!zt->nft) {
 962                 zt->nft = ct_act->ct.flow_table;
 963                 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);
 964                 if (err) {
 965                         NL_SET_ERR_MSG_MOD(extack,
 966                                            "offload error: Could not register nft_callback");
 967                         return err;
 968                 }
 969         }
 970
 971         /* Add entry to pre_ct_list */
 972         ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
 973         if (IS_ERR(ct_entry))
 974                 return PTR_ERR(ct_entry);
 975         ct_entry->type = CT_TYPE_PRE_CT;
 976         ct_entry->chain_index = ct_goto->chain_index;
 977         list_add(&ct_entry->list_node, &zt->pre_ct_list);
 978         zt->pre_ct_count++;
 979
 980         nfp_ct_merge_tc_entries(ct_entry, zt, zt);
 981
 982         /* Need to check and merge with tables in the wc_zone as well */
 983         if (priv->ct_zone_wc)
 984                 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);
 985
 986         return 0;
 987 }
 988
 989 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
 990                              struct net_device *netdev,
 991                              struct flow_cls_offload *flow,
 992                              struct netlink_ext_ack *extack)
 993 {
 994         struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
 995         struct nfp_fl_ct_flow_entry *ct_entry;
 996         struct nfp_fl_ct_zone_entry *zt;
 997         bool wildcarded = false;
 998         struct flow_match_ct ct;
 999
1000         flow_rule_match_ct(rule, &ct);
1001         if (!ct.mask->ct_zone) {
1002                 wildcarded = true;
1003         } else if (ct.mask->ct_zone != U16_MAX) {
1004                 NL_SET_ERR_MSG_MOD(extack,
1005                                    "unsupported offload: partially wildcarded ct_zone is not supported");
1006                 return -EOPNOTSUPP;
1007         }
1008
1009         zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);
1010         if (IS_ERR(zt)) {
1011                 NL_SET_ERR_MSG_MOD(extack,
1012                                    "offload error: Could not create zone table entry");
1013                 return PTR_ERR(zt);
1014         }
1015
1016         /* Add entry to post_ct_list */
1017         ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);
1018         if (IS_ERR(ct_entry))
1019                 return PTR_ERR(ct_entry);
1020
1021         ct_entry->type = CT_TYPE_POST_CT;
1022         ct_entry->chain_index = flow->common.chain_index;
1023         list_add(&ct_entry->list_node, &zt->post_ct_list);
1024         zt->post_ct_count++;
1025
1026         if (wildcarded) {
1027                 /* Iterate through all zone tables if not empty, look for merges with
1028                  * pre_ct entries and merge them.
1029                  */
1030                 struct rhashtable_iter iter;
1031                 struct nfp_fl_ct_zone_entry *zone_table;
1032
1033                 rhashtable_walk_enter(&priv->ct_zone_table, &iter);
1034                 rhashtable_walk_start(&iter);
1035                 while ((zone_table = rhashtable_walk_next(&iter)) != NULL) {
1036                         if (IS_ERR(zone_table))
1037                                 continue;
1038                         rhashtable_walk_stop(&iter);
1039                         nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table);
1040                         rhashtable_walk_start(&iter);
1041                 }
1042                 rhashtable_walk_stop(&iter);
1043                 rhashtable_walk_exit(&iter);
1044         } else {
1045                 nfp_ct_merge_tc_entries(ct_entry, zt, zt);
1046         }
1047
1048         return 0;
1049 }
1050
1051 static int
1052 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
1053 {
1054         struct nfp_fl_ct_map_entry *ct_map_ent;
1055         struct nfp_fl_ct_flow_entry *ct_entry;
1056         struct netlink_ext_ack *extack = NULL;
1057
1058         ASSERT_RTNL();
1059
1060         extack = flow->common.extack;
1061         switch (flow->command) {
1062         case FLOW_CLS_REPLACE:
1063                 /* Netfilter can request offload multiple times for the same
1064                  * flow - protect against adding duplicates.
1065                  */
1066                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1067                                                     nfp_ct_map_params);
1068                 if (!ct_map_ent) {
1069                         ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);
1070                         if (IS_ERR(ct_entry))
1071                                 return PTR_ERR(ct_entry);
1072                         ct_entry->type = CT_TYPE_NFT;
1073                         list_add(&ct_entry->list_node, &zt->nft_flows_list);
1074                         zt->nft_flows_count++;
1075                         nfp_ct_merge_nft_with_tc(ct_entry, zt);
1076                 }
1077                 return 0;
1078         case FLOW_CLS_DESTROY:
1079                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,
1080                                                     nfp_ct_map_params);
1081                 return nfp_fl_ct_del_flow(ct_map_ent);
1082         case FLOW_CLS_STATS:
1083                 return 0;
1084         default:
1085                 break;
1086         }
1087         return -EINVAL;
1088 }
1089
1090 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
1091 {
1092         struct flow_cls_offload *flow = type_data;
1093         struct nfp_fl_ct_zone_entry *zt = cb_priv;
1094         int err = -EOPNOTSUPP;
1095
1096         switch (type) {
1097         case TC_SETUP_CLSFLOWER:
1098                 rtnl_lock();
1099                 err = nfp_fl_ct_offload_nft_flow(zt, flow);
1100                 rtnl_unlock();
1101                 break;
1102         default:
1103                 return -EOPNOTSUPP;
1104         }
1105         return err;
1106 }
1107
1108 static void
1109 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
1110 {
1111         struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
1112         struct nfp_fl_ct_map_entry *ct_map_ent;
1113
1114         list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
1115                                  list_node) {
1116                 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,
1117                                                     &nft_entry->cookie,
1118                                                     nfp_ct_map_params);
1119                 nfp_fl_ct_del_flow(ct_map_ent);
1120         }
1121 }
1122
1123 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
1124 {
1125         struct nfp_fl_ct_flow_entry *ct_entry;
1126         struct nfp_fl_ct_zone_entry *zt;
1127         struct rhashtable *m_table;
1128
1129         if (!ct_map_ent)
1130                 return -ENOENT;
1131
1132         zt = ct_map_ent->ct_entry->zt;
1133         ct_entry = ct_map_ent->ct_entry;
1134         m_table = &zt->priv->ct_map_table;
1135
1136         switch (ct_entry->type) {
1137         case CT_TYPE_PRE_CT:
1138                 zt->pre_ct_count--;
1139                 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1140                                        nfp_ct_map_params);
1141                 nfp_fl_ct_clean_flow_entry(ct_entry);
1142                 kfree(ct_map_ent);
1143
1144                 if (!zt->pre_ct_count) {
1145                         zt->nft = NULL;
1146                         nfp_fl_ct_clean_nft_entries(zt);
1147                 }
1148                 break;
1149         case CT_TYPE_POST_CT:
1150                 zt->post_ct_count--;
1151                 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1152                                        nfp_ct_map_params);
1153                 nfp_fl_ct_clean_flow_entry(ct_entry);
1154                 kfree(ct_map_ent);
1155                 break;
1156         case CT_TYPE_NFT:
1157                 zt->nft_flows_count--;
1158                 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node,
1159                                        nfp_ct_map_params);
1160                 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry);
1161                 kfree(ct_map_ent);
1162         default:
1163                 break;
1164         }
1165
1166         return 0;
1167 }