drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3
   4 #include <net/netfilter/nf_conntrack.h>
   5 #include <net/netfilter/nf_conntrack_core.h>
   6 #include <net/netfilter/nf_conntrack_zones.h>
   7 #include <net/netfilter/nf_conntrack_labels.h>
   8 #include <net/netfilter/nf_conntrack_helper.h>
   9 #include <net/netfilter/nf_conntrack_acct.h>
  10 #include <uapi/linux/tc_act/tc_pedit.h>
  11 #include <net/tc_act/tc_ct.h>
  12 #include <net/flow_offload.h>
  13 #include <net/netfilter/nf_flow_table.h>
  14 #include <linux/workqueue.h>
  15 #include <linux/refcount.h>
  16 #include <linux/xarray.h>
  17 #include <linux/if_macvlan.h>
  18 #include <linux/debugfs.h>
  19
  20 #include "lib/fs_chains.h"
  21 #include "en/tc_ct.h"
  22 #include "en/tc/ct_fs.h"
  23 #include "en/tc_priv.h"
  24 #include "en/mod_hdr.h"
  25 #include "en/mapping.h"
  26 #include "en/tc/post_act.h"
  27 #include "en.h"
  28 #include "en_tc.h"
  29 #include "en_rep.h"
  30 #include "fs_core.h"
  31
  32 #define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1)
  33 #define MLX5_CT_STATE_TRK_BIT BIT(2)
  34 #define MLX5_CT_STATE_NAT_BIT BIT(3)
  35 #define MLX5_CT_STATE_REPLY_BIT BIT(4)
  36 #define MLX5_CT_STATE_RELATED_BIT BIT(5)
  37 #define MLX5_CT_STATE_INVALID_BIT BIT(6)
  38
  39 #define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen)
  40 #define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0)
  41
  42 /* Statically allocate modify actions for
  43  * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10.
  44  * This will be increased dynamically if needed (for the ipv6 snat + dnat).
  45  */
  46 #define MLX5_CT_MIN_MOD_ACTS 10
  47
  48 #define ct_dbg(fmt, args...)\
  49         netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args)
  50
  51 struct mlx5_tc_ct_debugfs {
  52         struct {
  53                 atomic_t offloaded;
  54                 atomic_t rx_dropped;
  55         } stats;
  56
  57         struct dentry *root;
  58 };
  59
  60 struct mlx5_tc_ct_priv {
  61         struct mlx5_core_dev *dev;
  62         const struct net_device *netdev;
  63         struct mod_hdr_tbl *mod_hdr_tbl;
  64         struct xarray tuple_ids;
  65         struct rhashtable zone_ht;
  66         struct rhashtable ct_tuples_ht;
  67         struct rhashtable ct_tuples_nat_ht;
  68         struct mlx5_flow_table *ct;
  69         struct mlx5_flow_table *ct_nat;
  70         struct mlx5e_post_act *post_act;
  71         struct mutex control_lock; /* guards parallel adds/dels */
  72         struct mapping_ctx *zone_mapping;
  73         struct mapping_ctx *labels_mapping;
  74         enum mlx5_flow_namespace_type ns_type;
  75         struct mlx5_fs_chains *chains;
  76         struct mlx5_ct_fs *fs;
  77         struct mlx5_ct_fs_ops *fs_ops;
  78         spinlock_t ht_lock; /* protects ft entries */
  79         struct workqueue_struct *wq;
  80
  81         struct mlx5_tc_ct_debugfs debugfs;
  82 };
  83
  84 struct mlx5_ct_flow {
  85         struct mlx5_flow_attr *pre_ct_attr;
  86         struct mlx5_flow_handle *pre_ct_rule;
  87         struct mlx5_ct_ft *ft;
  88         u32 chain_mapping;
  89 };
  90
  91 struct mlx5_ct_zone_rule {
  92         struct mlx5_ct_fs_rule *rule;
  93         struct mlx5e_mod_hdr_handle *mh;
  94         struct mlx5_flow_attr *attr;
  95         bool nat;
  96 };
  97
  98 struct mlx5_tc_ct_pre {
  99         struct mlx5_flow_table *ft;
 100         struct mlx5_flow_group *flow_grp;
 101         struct mlx5_flow_group *miss_grp;
 102         struct mlx5_flow_handle *flow_rule;
 103         struct mlx5_flow_handle *miss_rule;
 104         struct mlx5_modify_hdr *modify_hdr;
 105 };
 106
 107 struct mlx5_ct_ft {
 108         struct rhash_head node;
 109         u16 zone;
 110         u32 zone_restore_id;
 111         refcount_t refcount;
 112         struct nf_flowtable *nf_ft;
 113         struct mlx5_tc_ct_priv *ct_priv;
 114         struct rhashtable ct_entries_ht;
 115         struct mlx5_tc_ct_pre pre_ct;
 116         struct mlx5_tc_ct_pre pre_ct_nat;
 117 };
 118
 119 struct mlx5_ct_tuple {
 120         u16 addr_type;
 121         __be16 n_proto;
 122         u8 ip_proto;
 123         struct {
 124                 union {
 125                         __be32 src_v4;
 126                         struct in6_addr src_v6;
 127                 };
 128                 union {
 129                         __be32 dst_v4;
 130                         struct in6_addr dst_v6;
 131                 };
 132         } ip;
 133         struct {
 134                 __be16 src;
 135                 __be16 dst;
 136         } port;
 137
 138         u16 zone;
 139 };
 140
 141 struct mlx5_ct_counter {
 142         struct mlx5_fc *counter;
 143         refcount_t refcount;
 144         bool is_shared;
 145 };
 146
 147 enum {
 148         MLX5_CT_ENTRY_FLAG_VALID,
 149 };
 150
 151 struct mlx5_ct_entry {
 152         struct rhash_head node;
 153         struct rhash_head tuple_node;
 154         struct rhash_head tuple_nat_node;
 155         struct mlx5_ct_counter *counter;
 156         unsigned long cookie;
 157         unsigned long restore_cookie;
 158         struct mlx5_ct_tuple tuple;
 159         struct mlx5_ct_tuple tuple_nat;
 160         struct mlx5_ct_zone_rule zone_rules[2];
 161
 162         struct mlx5_tc_ct_priv *ct_priv;
 163         struct work_struct work;
 164
 165         refcount_t refcnt;
 166         unsigned long flags;
 167 };
 168
 169 static void
 170 mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 171                                  struct mlx5_flow_attr *attr,
 172                                  struct mlx5e_mod_hdr_handle *mh);
 173
 174 static const struct rhashtable_params cts_ht_params = {
 175         .head_offset = offsetof(struct mlx5_ct_entry, node),
 176         .key_offset = offsetof(struct mlx5_ct_entry, cookie),
 177         .key_len = sizeof(((struct mlx5_ct_entry *)0)->cookie),
 178         .automatic_shrinking = true,
 179         .min_size = 16 * 1024,
 180 };
 181
 182 static const struct rhashtable_params zone_params = {
 183         .head_offset = offsetof(struct mlx5_ct_ft, node),
 184         .key_offset = offsetof(struct mlx5_ct_ft, zone),
 185         .key_len = sizeof(((struct mlx5_ct_ft *)0)->zone),
 186         .automatic_shrinking = true,
 187 };
 188
 189 static const struct rhashtable_params tuples_ht_params = {
 190         .head_offset = offsetof(struct mlx5_ct_entry, tuple_node),
 191         .key_offset = offsetof(struct mlx5_ct_entry, tuple),
 192         .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple),
 193         .automatic_shrinking = true,
 194         .min_size = 16 * 1024,
 195 };
 196
 197 static const struct rhashtable_params tuples_nat_ht_params = {
 198         .head_offset = offsetof(struct mlx5_ct_entry, tuple_nat_node),
 199         .key_offset = offsetof(struct mlx5_ct_entry, tuple_nat),
 200         .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple_nat),
 201         .automatic_shrinking = true,
 202         .min_size = 16 * 1024,
 203 };
 204
 205 static bool
 206 mlx5_tc_ct_entry_has_nat(struct mlx5_ct_entry *entry)
 207 {
 208         return !!(entry->tuple_nat_node.next);
 209 }
 210
 211 static int
 212 mlx5_get_label_mapping(struct mlx5_tc_ct_priv *ct_priv,
 213                        u32 *labels, u32 *id)
 214 {
 215         if (!memchr_inv(labels, 0, sizeof(u32) * 4)) {
 216                 *id = 0;
 217                 return 0;
 218         }
 219
 220         if (mapping_add(ct_priv->labels_mapping, labels, id))
 221                 return -EOPNOTSUPP;
 222
 223         return 0;
 224 }
 225
 226 static void
 227 mlx5_put_label_mapping(struct mlx5_tc_ct_priv *ct_priv, u32 id)
 228 {
 229         if (id)
 230                 mapping_remove(ct_priv->labels_mapping, id);
 231 }
 232
 233 static int
 234 mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple *tuple, struct flow_rule *rule)
 235 {
 236         struct flow_match_control control;
 237         struct flow_match_basic basic;
 238
 239         flow_rule_match_basic(rule, &basic);
 240         flow_rule_match_control(rule, &control);
 241
 242         tuple->n_proto = basic.key->n_proto;
 243         tuple->ip_proto = basic.key->ip_proto;
 244         tuple->addr_type = control.key->addr_type;
 245
 246         if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 247                 struct flow_match_ipv4_addrs match;
 248
 249                 flow_rule_match_ipv4_addrs(rule, &match);
 250                 tuple->ip.src_v4 = match.key->src;
 251                 tuple->ip.dst_v4 = match.key->dst;
 252         } else if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 253                 struct flow_match_ipv6_addrs match;
 254
 255                 flow_rule_match_ipv6_addrs(rule, &match);
 256                 tuple->ip.src_v6 = match.key->src;
 257                 tuple->ip.dst_v6 = match.key->dst;
 258         } else {
 259                 return -EOPNOTSUPP;
 260         }
 261
 262         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 263                 struct flow_match_ports match;
 264
 265                 flow_rule_match_ports(rule, &match);
 266                 switch (tuple->ip_proto) {
 267                 case IPPROTO_TCP:
 268                 case IPPROTO_UDP:
 269                         tuple->port.src = match.key->src;
 270                         tuple->port.dst = match.key->dst;
 271                         break;
 272                 default:
 273                         return -EOPNOTSUPP;
 274                 }
 275         } else {
 276                 if (tuple->ip_proto != IPPROTO_GRE)
 277                         return -EOPNOTSUPP;
 278         }
 279
 280         return 0;
 281 }
 282
 283 static int
 284 mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple *tuple,
 285                              struct flow_rule *rule)
 286 {
 287         struct flow_action *flow_action = &rule->action;
 288         struct flow_action_entry *act;
 289         u32 offset, val, ip6_offset;
 290         int i;
 291
 292         flow_action_for_each(i, act, flow_action) {
 293                 if (act->id != FLOW_ACTION_MANGLE)
 294                         continue;
 295
 296                 offset = act->mangle.offset;
 297                 val = act->mangle.val;
 298                 switch (act->mangle.htype) {
 299                 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 300                         if (offset == offsetof(struct iphdr, saddr))
 301                                 tuple->ip.src_v4 = cpu_to_be32(val);
 302                         else if (offset == offsetof(struct iphdr, daddr))
 303                                 tuple->ip.dst_v4 = cpu_to_be32(val);
 304                         else
 305                                 return -EOPNOTSUPP;
 306                         break;
 307
 308                 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 309                         ip6_offset = (offset - offsetof(struct ipv6hdr, saddr));
 310                         ip6_offset /= 4;
 311                         if (ip6_offset < 4)
 312                                 tuple->ip.src_v6.s6_addr32[ip6_offset] = cpu_to_be32(val);
 313                         else if (ip6_offset < 8)
 314                                 tuple->ip.dst_v6.s6_addr32[ip6_offset - 4] = cpu_to_be32(val);
 315                         else
 316                                 return -EOPNOTSUPP;
 317                         break;
 318
 319                 case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 320                         if (offset == offsetof(struct tcphdr, source))
 321                                 tuple->port.src = cpu_to_be16(val);
 322                         else if (offset == offsetof(struct tcphdr, dest))
 323                                 tuple->port.dst = cpu_to_be16(val);
 324                         else
 325                                 return -EOPNOTSUPP;
 326                         break;
 327
 328                 case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 329                         if (offset == offsetof(struct udphdr, source))
 330                                 tuple->port.src = cpu_to_be16(val);
 331                         else if (offset == offsetof(struct udphdr, dest))
 332                                 tuple->port.dst = cpu_to_be16(val);
 333                         else
 334                                 return -EOPNOTSUPP;
 335                         break;
 336
 337                 default:
 338                         return -EOPNOTSUPP;
 339                 }
 340         }
 341
 342         return 0;
 343 }
 344
 345 static int
 346 mlx5_tc_ct_get_flow_source_match(struct mlx5_tc_ct_priv *ct_priv,
 347                                  struct net_device *ndev)
 348 {
 349         struct mlx5e_priv *other_priv = netdev_priv(ndev);
 350         struct mlx5_core_dev *mdev = ct_priv->dev;
 351         bool vf_rep, uplink_rep;
 352
 353         vf_rep = mlx5e_eswitch_vf_rep(ndev) && mlx5_same_hw_devs(mdev, other_priv->mdev);
 354         uplink_rep = mlx5e_eswitch_uplink_rep(ndev) && mlx5_same_hw_devs(mdev, other_priv->mdev);
 355
 356         if (vf_rep)
 357                 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
 358         if (uplink_rep)
 359                 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
 360         if (is_vlan_dev(ndev))
 361                 return mlx5_tc_ct_get_flow_source_match(ct_priv, vlan_dev_real_dev(ndev));
 362         if (netif_is_macvlan(ndev))
 363                 return mlx5_tc_ct_get_flow_source_match(ct_priv, macvlan_dev_real_dev(ndev));
 364         if (mlx5e_get_tc_tun(ndev) || netif_is_lag_master(ndev))
 365                 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
 366
 367         return MLX5_FLOW_CONTEXT_FLOW_SOURCE_ANY_VPORT;
 368 }
 369
 370 static int
 371 mlx5_tc_ct_set_tuple_match(struct mlx5_tc_ct_priv *ct_priv,
 372                            struct mlx5_flow_spec *spec,
 373                            struct flow_rule *rule)
 374 {
 375         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 376                                        outer_headers);
 377         void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 378                                        outer_headers);
 379         u16 addr_type = 0;
 380         u8 ip_proto = 0;
 381
 382         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
 383                 struct flow_match_basic match;
 384
 385                 flow_rule_match_basic(rule, &match);
 386
 387                 mlx5e_tc_set_ethertype(ct_priv->dev, &match, true, headers_c, headers_v);
 388                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 389                          match.mask->ip_proto);
 390                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 391                          match.key->ip_proto);
 392
 393                 ip_proto = match.key->ip_proto;
 394         }
 395
 396         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
 397                 struct flow_match_control match;
 398
 399                 flow_rule_match_control(rule, &match);
 400                 addr_type = match.key->addr_type;
 401         }
 402
 403         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 404                 struct flow_match_ipv4_addrs match;
 405
 406                 flow_rule_match_ipv4_addrs(rule, &match);
 407                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 408                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
 409                        &match.mask->src, sizeof(match.mask->src));
 410                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 411                                     src_ipv4_src_ipv6.ipv4_layout.ipv4),
 412                        &match.key->src, sizeof(match.key->src));
 413                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 414                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 415                        &match.mask->dst, sizeof(match.mask->dst));
 416                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 417                                     dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 418                        &match.key->dst, sizeof(match.key->dst));
 419         }
 420
 421         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 422                 struct flow_match_ipv6_addrs match;
 423
 424                 flow_rule_match_ipv6_addrs(rule, &match);
 425                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 426                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 427                        &match.mask->src, sizeof(match.mask->src));
 428                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 429                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
 430                        &match.key->src, sizeof(match.key->src));
 431
 432                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 433                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 434                        &match.mask->dst, sizeof(match.mask->dst));
 435                 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 436                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 437                        &match.key->dst, sizeof(match.key->dst));
 438         }
 439
 440         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 441                 struct flow_match_ports match;
 442
 443                 flow_rule_match_ports(rule, &match);
 444                 switch (ip_proto) {
 445                 case IPPROTO_TCP:
 446                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 447                                  tcp_sport, ntohs(match.mask->src));
 448                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 449                                  tcp_sport, ntohs(match.key->src));
 450
 451                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 452                                  tcp_dport, ntohs(match.mask->dst));
 453                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 454                                  tcp_dport, ntohs(match.key->dst));
 455                         break;
 456
 457                 case IPPROTO_UDP:
 458                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 459                                  udp_sport, ntohs(match.mask->src));
 460                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 461                                  udp_sport, ntohs(match.key->src));
 462
 463                         MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 464                                  udp_dport, ntohs(match.mask->dst));
 465                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 466                                  udp_dport, ntohs(match.key->dst));
 467                         break;
 468                 default:
 469                         break;
 470                 }
 471         }
 472
 473         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
 474                 struct flow_match_tcp match;
 475
 476                 flow_rule_match_tcp(rule, &match);
 477                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
 478                          ntohs(match.mask->flags));
 479                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
 480                          ntohs(match.key->flags));
 481         }
 482
 483         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
 484                 struct flow_match_meta match;
 485
 486                 flow_rule_match_meta(rule, &match);
 487
 488                 if (match.key->ingress_ifindex & match.mask->ingress_ifindex) {
 489                         struct net_device *dev;
 490
 491                         dev = dev_get_by_index(&init_net, match.key->ingress_ifindex);
 492                         if (dev && MLX5_CAP_ESW_FLOWTABLE(ct_priv->dev, flow_source))
 493                                 spec->flow_context.flow_source =
 494                                         mlx5_tc_ct_get_flow_source_match(ct_priv, dev);
 495
 496                         dev_put(dev);
 497                 }
 498         }
 499
 500         return 0;
 501 }
 502
 503 static void
 504 mlx5_tc_ct_counter_put(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry)
 505 {
 506         if (entry->counter->is_shared &&
 507             !refcount_dec_and_test(&entry->counter->refcount))
 508                 return;
 509
 510         mlx5_fc_destroy(ct_priv->dev, entry->counter->counter);
 511         kfree(entry->counter);
 512 }
 513
 514 static void
 515 mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv,
 516                           struct mlx5_ct_entry *entry,
 517                           bool nat)
 518 {
 519         struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
 520         struct mlx5_flow_attr *attr = zone_rule->attr;
 521
 522         ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone);
 523
 524         ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule);
 525         mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
 526         mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
 527         kfree(attr);
 528 }
 529
 530 static void
 531 mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv,
 532                            struct mlx5_ct_entry *entry)
 533 {
 534         mlx5_tc_ct_entry_del_rule(ct_priv, entry, true);
 535         mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
 536
 537         atomic_dec(&ct_priv->debugfs.stats.offloaded);
 538 }
 539
 540 static struct flow_action_entry *
 541 mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule)
 542 {
 543         struct flow_action *flow_action = &flow_rule->action;
 544         struct flow_action_entry *act;
 545         int i;
 546
 547         flow_action_for_each(i, act, flow_action) {
 548                 if (act->id == FLOW_ACTION_CT_METADATA)
 549                         return act;
 550         }
 551
 552         return NULL;
 553 }
 554
 555 static int
 556 mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv,
 557                                struct mlx5e_tc_mod_hdr_acts *mod_acts,
 558                                u8 ct_state,
 559                                u32 mark,
 560                                u32 labels_id,
 561                                u8 zone_restore_id)
 562 {
 563         enum mlx5_flow_namespace_type ns = ct_priv->ns_type;
 564         struct mlx5_core_dev *dev = ct_priv->dev;
 565         int err;
 566
 567         err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 568                                         CTSTATE_TO_REG, ct_state);
 569         if (err)
 570                 return err;
 571
 572         err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 573                                         MARK_TO_REG, mark);
 574         if (err)
 575                 return err;
 576
 577         err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 578                                         LABELS_TO_REG, labels_id);
 579         if (err)
 580                 return err;
 581
 582         err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 583                                         ZONE_RESTORE_TO_REG, zone_restore_id);
 584         if (err)
 585                 return err;
 586
 587         /* Make another copy of zone id in reg_b for
 588          * NIC rx flows since we don't copy reg_c1 to
 589          * reg_b upon miss.
 590          */
 591         if (ns != MLX5_FLOW_NAMESPACE_FDB) {
 592                 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
 593                                                 NIC_ZONE_RESTORE_TO_REG, zone_restore_id);
 594                 if (err)
 595                         return err;
 596         }
 597         return 0;
 598 }
 599
 600 int mlx5_tc_ct_set_ct_clear_regs(struct mlx5_tc_ct_priv *priv,
 601                                  struct mlx5e_tc_mod_hdr_acts *mod_acts)
 602 {
 603                 return mlx5_tc_ct_entry_set_registers(priv, mod_acts, 0, 0, 0, 0);
 604 }
 605
 606 static int
 607 mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
 608                                    char *modact)
 609 {
 610         u32 offset = act->mangle.offset, field;
 611
 612         switch (act->mangle.htype) {
 613         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 614                 MLX5_SET(set_action_in, modact, length, 0);
 615                 if (offset == offsetof(struct iphdr, saddr))
 616                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV4;
 617                 else if (offset == offsetof(struct iphdr, daddr))
 618                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV4;
 619                 else
 620                         return -EOPNOTSUPP;
 621                 break;
 622
 623         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 624                 MLX5_SET(set_action_in, modact, length, 0);
 625                 if (offset == offsetof(struct ipv6hdr, saddr) + 12)
 626                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0;
 627                 else if (offset == offsetof(struct ipv6hdr, saddr) + 8)
 628                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
 629                 else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
 630                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64;
 631                 else if (offset == offsetof(struct ipv6hdr, saddr))
 632                         field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96;
 633                 else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
 634                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0;
 635                 else if (offset == offsetof(struct ipv6hdr, daddr) + 8)
 636                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
 637                 else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
 638                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64;
 639                 else if (offset == offsetof(struct ipv6hdr, daddr))
 640                         field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96;
 641                 else
 642                         return -EOPNOTSUPP;
 643                 break;
 644
 645         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 646                 MLX5_SET(set_action_in, modact, length, 16);
 647                 if (offset == offsetof(struct tcphdr, source))
 648                         field = MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT;
 649                 else if (offset == offsetof(struct tcphdr, dest))
 650                         field = MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT;
 651                 else
 652                         return -EOPNOTSUPP;
 653                 break;
 654
 655         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 656                 MLX5_SET(set_action_in, modact, length, 16);
 657                 if (offset == offsetof(struct udphdr, source))
 658                         field = MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT;
 659                 else if (offset == offsetof(struct udphdr, dest))
 660                         field = MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT;
 661                 else
 662                         return -EOPNOTSUPP;
 663                 break;
 664
 665         default:
 666                 return -EOPNOTSUPP;
 667         }
 668
 669         MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
 670         MLX5_SET(set_action_in, modact, offset, 0);
 671         MLX5_SET(set_action_in, modact, field, field);
 672         MLX5_SET(set_action_in, modact, data, act->mangle.val);
 673
 674         return 0;
 675 }
 676
 677 static int
 678 mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv,
 679                             struct flow_rule *flow_rule,
 680                             struct mlx5e_tc_mod_hdr_acts *mod_acts)
 681 {
 682         struct flow_action *flow_action = &flow_rule->action;
 683         struct mlx5_core_dev *mdev = ct_priv->dev;
 684         struct flow_action_entry *act;
 685         char *modact;
 686         int err, i;
 687
 688         flow_action_for_each(i, act, flow_action) {
 689                 switch (act->id) {
 690                 case FLOW_ACTION_MANGLE: {
 691                         modact = mlx5e_mod_hdr_alloc(mdev, ct_priv->ns_type, mod_acts);
 692                         if (IS_ERR(modact))
 693                                 return PTR_ERR(modact);
 694
 695                         err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact);
 696                         if (err)
 697                                 return err;
 698
 699                         mod_acts->num_actions++;
 700                 }
 701                 break;
 702
 703                 case FLOW_ACTION_CT_METADATA:
 704                         /* Handled earlier */
 705                         continue;
 706                 default:
 707                         return -EOPNOTSUPP;
 708                 }
 709         }
 710
 711         return 0;
 712 }
 713
 714 static int
 715 mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 716                                 struct mlx5_flow_attr *attr,
 717                                 struct flow_rule *flow_rule,
 718                                 struct mlx5e_mod_hdr_handle **mh,
 719                                 u8 zone_restore_id, bool nat_table, bool has_nat)
 720 {
 721         DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS);
 722         DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr);
 723         struct flow_action_entry *meta;
 724         u16 ct_state = 0;
 725         int err;
 726
 727         meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
 728         if (!meta)
 729                 return -EOPNOTSUPP;
 730
 731         err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels,
 732                                      &attr->ct_attr.ct_labels_id);
 733         if (err)
 734                 return -EOPNOTSUPP;
 735         if (nat_table) {
 736                 if (has_nat) {
 737                         err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule, &mod_acts);
 738                         if (err)
 739                                 goto err_mapping;
 740                 }
 741
 742                 ct_state |= MLX5_CT_STATE_NAT_BIT;
 743         }
 744
 745         ct_state |= MLX5_CT_STATE_ESTABLISHED_BIT | MLX5_CT_STATE_TRK_BIT;
 746         ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT;
 747         err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,
 748                                              ct_state,
 749                                              meta->ct_metadata.mark,
 750                                              attr->ct_attr.ct_labels_id,
 751                                              zone_restore_id);
 752         if (err)
 753                 goto err_mapping;
 754
 755         if (nat_table && has_nat) {
 756                 attr->modify_hdr = mlx5_modify_header_alloc(ct_priv->dev, ct_priv->ns_type,
 757                                                             mod_acts.num_actions,
 758                                                             mod_acts.actions);
 759                 if (IS_ERR(attr->modify_hdr)) {
 760                         err = PTR_ERR(attr->modify_hdr);
 761                         goto err_mapping;
 762                 }
 763
 764                 *mh = NULL;
 765         } else {
 766                 *mh = mlx5e_mod_hdr_attach(ct_priv->dev,
 767                                            ct_priv->mod_hdr_tbl,
 768                                            ct_priv->ns_type,
 769                                            &mod_acts);
 770                 if (IS_ERR(*mh)) {
 771                         err = PTR_ERR(*mh);
 772                         goto err_mapping;
 773                 }
 774                 attr->modify_hdr = mlx5e_mod_hdr_get(*mh);
 775         }
 776
 777         mlx5e_mod_hdr_dealloc(&mod_acts);
 778         return 0;
 779
 780 err_mapping:
 781         mlx5e_mod_hdr_dealloc(&mod_acts);
 782         mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
 783         return err;
 784 }
 785
 786 static void
 787 mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
 788                                  struct mlx5_flow_attr *attr,
 789                                  struct mlx5e_mod_hdr_handle *mh)
 790 {
 791         if (mh)
 792                 mlx5e_mod_hdr_detach(ct_priv->dev, ct_priv->mod_hdr_tbl, mh);
 793         else
 794                 mlx5_modify_header_dealloc(ct_priv->dev, attr->modify_hdr);
 795 }
 796
 797 static int
 798 mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
 799                           struct flow_rule *flow_rule,
 800                           struct mlx5_ct_entry *entry,
 801                           bool nat, u8 zone_restore_id)
 802 {
 803         struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
 804         struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
 805         struct mlx5_flow_spec *spec = NULL;
 806         struct mlx5_flow_attr *attr;
 807         int err;
 808
 809         zone_rule->nat = nat;
 810
 811         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 812         if (!spec)
 813                 return -ENOMEM;
 814
 815         attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
 816         if (!attr) {
 817                 err = -ENOMEM;
 818                 goto err_attr;
 819         }
 820
 821         err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule,
 822                                               &zone_rule->mh,
 823                                               zone_restore_id,
 824                                               nat,
 825                                               mlx5_tc_ct_entry_has_nat(entry));
 826         if (err) {
 827                 ct_dbg("Failed to create ct entry mod hdr");
 828                 goto err_mod_hdr;
 829         }
 830
 831         attr->action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
 832                        MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 833                        MLX5_FLOW_CONTEXT_ACTION_COUNT;
 834         attr->dest_chain = 0;
 835         attr->dest_ft = mlx5e_tc_post_act_get_ft(ct_priv->post_act);
 836         attr->ft = nat ? ct_priv->ct_nat : ct_priv->ct;
 837         if (entry->tuple.ip_proto == IPPROTO_TCP ||
 838             entry->tuple.ip_proto == IPPROTO_UDP)
 839                 attr->outer_match_level = MLX5_MATCH_L4;
 840         else
 841                 attr->outer_match_level = MLX5_MATCH_L3;
 842         attr->counter = entry->counter->counter;
 843         attr->flags |= MLX5_ATTR_FLAG_NO_IN_PORT;
 844         if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB)
 845                 attr->esw_attr->in_mdev = priv->mdev;
 846
 847         mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule);
 848         mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK);
 849
 850         zone_rule->rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule);
 851         if (IS_ERR(zone_rule->rule)) {
 852                 err = PTR_ERR(zone_rule->rule);
 853                 ct_dbg("Failed to add ct entry rule, nat: %d", nat);
 854                 goto err_rule;
 855         }
 856
 857         zone_rule->attr = attr;
 858
 859         kvfree(spec);
 860         ct_dbg("Offloaded ct entry rule in zone %d", entry->tuple.zone);
 861
 862         return 0;
 863
 864 err_rule:
 865         mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
 866         mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
 867 err_mod_hdr:
 868         kfree(attr);
 869 err_attr:
 870         kvfree(spec);
 871         return err;
 872 }
 873
 874 static bool
 875 mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry)
 876 {
 877         return test_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags);
 878 }
 879
 880 static struct mlx5_ct_entry *
 881 mlx5_tc_ct_entry_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_tuple *tuple)
 882 {
 883         struct mlx5_ct_entry *entry;
 884
 885         entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_ht, tuple,
 886                                        tuples_ht_params);
 887         if (entry && mlx5_tc_ct_entry_valid(entry) &&
 888             refcount_inc_not_zero(&entry->refcnt)) {
 889                 return entry;
 890         } else if (!entry) {
 891                 entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_nat_ht,
 892                                                tuple, tuples_nat_ht_params);
 893                 if (entry && mlx5_tc_ct_entry_valid(entry) &&
 894                     refcount_inc_not_zero(&entry->refcnt))
 895                         return entry;
 896         }
 897
 898         return entry ? ERR_PTR(-EINVAL) : NULL;
 899 }
 900
 901 static void mlx5_tc_ct_entry_remove_from_tuples(struct mlx5_ct_entry *entry)
 902 {
 903         struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv;
 904
 905         rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht,
 906                                &entry->tuple_nat_node,
 907                                tuples_nat_ht_params);
 908         rhashtable_remove_fast(&ct_priv->ct_tuples_ht, &entry->tuple_node,
 909                                tuples_ht_params);
 910 }
 911
 912 static void mlx5_tc_ct_entry_del(struct mlx5_ct_entry *entry)
 913 {
 914         struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv;
 915
 916         mlx5_tc_ct_entry_del_rules(ct_priv, entry);
 917
 918         spin_lock_bh(&ct_priv->ht_lock);
 919         mlx5_tc_ct_entry_remove_from_tuples(entry);
 920         spin_unlock_bh(&ct_priv->ht_lock);
 921
 922         mlx5_tc_ct_counter_put(ct_priv, entry);
 923         kfree(entry);
 924 }
 925
 926 static void
 927 mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry)
 928 {
 929         if (!refcount_dec_and_test(&entry->refcnt))
 930                 return;
 931
 932         mlx5_tc_ct_entry_del(entry);
 933 }
 934
 935 static void mlx5_tc_ct_entry_del_work(struct work_struct *work)
 936 {
 937         struct mlx5_ct_entry *entry = container_of(work, struct mlx5_ct_entry, work);
 938
 939         mlx5_tc_ct_entry_del(entry);
 940 }
 941
 942 static void
 943 __mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry)
 944 {
 945         if (!refcount_dec_and_test(&entry->refcnt))
 946                 return;
 947
 948         INIT_WORK(&entry->work, mlx5_tc_ct_entry_del_work);
 949         queue_work(entry->ct_priv->wq, &entry->work);
 950 }
 951
 952 static struct mlx5_ct_counter *
 953 mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv *ct_priv)
 954 {
 955         struct mlx5_ct_counter *counter;
 956         int ret;
 957
 958         counter = kzalloc(sizeof(*counter), GFP_KERNEL);
 959         if (!counter)
 960                 return ERR_PTR(-ENOMEM);
 961
 962         counter->is_shared = false;
 963         counter->counter = mlx5_fc_create_ex(ct_priv->dev, true);
 964         if (IS_ERR(counter->counter)) {
 965                 ct_dbg("Failed to create counter for ct entry");
 966                 ret = PTR_ERR(counter->counter);
 967                 kfree(counter);
 968                 return ERR_PTR(ret);
 969         }
 970
 971         return counter;
 972 }
 973
 974 static struct mlx5_ct_counter *
 975 mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv,
 976                               struct mlx5_ct_entry *entry)
 977 {
 978         struct mlx5_ct_tuple rev_tuple = entry->tuple;
 979         struct mlx5_ct_counter *shared_counter;
 980         struct mlx5_ct_entry *rev_entry;
 981
 982         /* get the reversed tuple */
 983         swap(rev_tuple.port.src, rev_tuple.port.dst);
 984
 985         if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 986                 __be32 tmp_addr = rev_tuple.ip.src_v4;
 987
 988                 rev_tuple.ip.src_v4 = rev_tuple.ip.dst_v4;
 989                 rev_tuple.ip.dst_v4 = tmp_addr;
 990         } else if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 991                 struct in6_addr tmp_addr = rev_tuple.ip.src_v6;
 992
 993                 rev_tuple.ip.src_v6 = rev_tuple.ip.dst_v6;
 994                 rev_tuple.ip.dst_v6 = tmp_addr;
 995         } else {
 996                 return ERR_PTR(-EOPNOTSUPP);
 997         }
 998
 999         /* Use the same counter as the reverse direction */
1000         spin_lock_bh(&ct_priv->ht_lock);
1001         rev_entry = mlx5_tc_ct_entry_get(ct_priv, &rev_tuple);
1002
1003         if (IS_ERR(rev_entry)) {
1004                 spin_unlock_bh(&ct_priv->ht_lock);
1005                 goto create_counter;
1006         }
1007
1008         if (rev_entry && refcount_inc_not_zero(&rev_entry->counter->refcount)) {
1009                 ct_dbg("Using shared counter entry=0x%p rev=0x%p", entry, rev_entry);
1010                 shared_counter = rev_entry->counter;
1011                 spin_unlock_bh(&ct_priv->ht_lock);
1012
1013                 mlx5_tc_ct_entry_put(rev_entry);
1014                 return shared_counter;
1015         }
1016
1017         spin_unlock_bh(&ct_priv->ht_lock);
1018
1019 create_counter:
1020
1021         shared_counter = mlx5_tc_ct_counter_create(ct_priv);
1022         if (IS_ERR(shared_counter))
1023                 return shared_counter;
1024
1025         shared_counter->is_shared = true;
1026         refcount_set(&shared_counter->refcount, 1);
1027         return shared_counter;
1028 }
1029
1030 static int
1031 mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv,
1032                            struct flow_rule *flow_rule,
1033                            struct mlx5_ct_entry *entry,
1034                            u8 zone_restore_id)
1035 {
1036         int err;
1037
1038         if (nf_ct_acct_enabled(dev_net(ct_priv->netdev)))
1039                 entry->counter = mlx5_tc_ct_counter_create(ct_priv);
1040         else
1041                 entry->counter = mlx5_tc_ct_shared_counter_get(ct_priv, entry);
1042
1043         if (IS_ERR(entry->counter)) {
1044                 err = PTR_ERR(entry->counter);
1045                 return err;
1046         }
1047
1048         err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false,
1049                                         zone_restore_id);
1050         if (err)
1051                 goto err_orig;
1052
1053         err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true,
1054                                         zone_restore_id);
1055         if (err)
1056                 goto err_nat;
1057
1058         atomic_inc(&ct_priv->debugfs.stats.offloaded);
1059         return 0;
1060
1061 err_nat:
1062         mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
1063 err_orig:
1064         mlx5_tc_ct_counter_put(ct_priv, entry);
1065         return err;
1066 }
1067
1068 static int
1069 mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
1070                                   struct flow_cls_offload *flow)
1071 {
1072         struct flow_rule *flow_rule = flow_cls_offload_flow_rule(flow);
1073         struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1074         struct flow_action_entry *meta_action;
1075         unsigned long cookie = flow->cookie;
1076         struct mlx5_ct_entry *entry;
1077         int err;
1078
1079         meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
1080         if (!meta_action)
1081                 return -EOPNOTSUPP;
1082
1083         spin_lock_bh(&ct_priv->ht_lock);
1084         entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1085         if (entry && refcount_inc_not_zero(&entry->refcnt)) {
1086                 spin_unlock_bh(&ct_priv->ht_lock);
1087                 mlx5_tc_ct_entry_put(entry);
1088                 return -EEXIST;
1089         }
1090         spin_unlock_bh(&ct_priv->ht_lock);
1091
1092         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1093         if (!entry)
1094                 return -ENOMEM;
1095
1096         entry->tuple.zone = ft->zone;
1097         entry->cookie = flow->cookie;
1098         entry->restore_cookie = meta_action->ct_metadata.cookie;
1099         refcount_set(&entry->refcnt, 2);
1100         entry->ct_priv = ct_priv;
1101
1102         err = mlx5_tc_ct_rule_to_tuple(&entry->tuple, flow_rule);
1103         if (err)
1104                 goto err_set;
1105
1106         memcpy(&entry->tuple_nat, &entry->tuple, sizeof(entry->tuple));
1107         err = mlx5_tc_ct_rule_to_tuple_nat(&entry->tuple_nat, flow_rule);
1108         if (err)
1109                 goto err_set;
1110
1111         spin_lock_bh(&ct_priv->ht_lock);
1112
1113         err = rhashtable_lookup_insert_fast(&ft->ct_entries_ht, &entry->node,
1114                                             cts_ht_params);
1115         if (err)
1116                 goto err_entries;
1117
1118         err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_ht,
1119                                             &entry->tuple_node,
1120                                             tuples_ht_params);
1121         if (err)
1122                 goto err_tuple;
1123
1124         if (memcmp(&entry->tuple, &entry->tuple_nat, sizeof(entry->tuple))) {
1125                 err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_nat_ht,
1126                                                     &entry->tuple_nat_node,
1127                                                     tuples_nat_ht_params);
1128                 if (err)
1129                         goto err_tuple_nat;
1130         }
1131         spin_unlock_bh(&ct_priv->ht_lock);
1132
1133         err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry,
1134                                          ft->zone_restore_id);
1135         if (err)
1136                 goto err_rules;
1137
1138         set_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags);
1139         mlx5_tc_ct_entry_put(entry); /* this function reference */
1140
1141         return 0;
1142
1143 err_rules:
1144         spin_lock_bh(&ct_priv->ht_lock);
1145         if (mlx5_tc_ct_entry_has_nat(entry))
1146                 rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht,
1147                                        &entry->tuple_nat_node, tuples_nat_ht_params);
1148 err_tuple_nat:
1149         rhashtable_remove_fast(&ct_priv->ct_tuples_ht,
1150                                &entry->tuple_node,
1151                                tuples_ht_params);
1152 err_tuple:
1153         rhashtable_remove_fast(&ft->ct_entries_ht,
1154                                &entry->node,
1155                                cts_ht_params);
1156 err_entries:
1157         spin_unlock_bh(&ct_priv->ht_lock);
1158 err_set:
1159         kfree(entry);
1160         if (err != -EEXIST)
1161                 netdev_warn(ct_priv->netdev, "Failed to offload ct entry, err: %d\n", err);
1162         return err;
1163 }
1164
1165 static int
1166 mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft,
1167                                   struct flow_cls_offload *flow)
1168 {
1169         struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1170         unsigned long cookie = flow->cookie;
1171         struct mlx5_ct_entry *entry;
1172
1173         spin_lock_bh(&ct_priv->ht_lock);
1174         entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1175         if (!entry) {
1176                 spin_unlock_bh(&ct_priv->ht_lock);
1177                 return -ENOENT;
1178         }
1179
1180         if (!mlx5_tc_ct_entry_valid(entry)) {
1181                 spin_unlock_bh(&ct_priv->ht_lock);
1182                 return -EINVAL;
1183         }
1184
1185         rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);
1186         spin_unlock_bh(&ct_priv->ht_lock);
1187
1188         mlx5_tc_ct_entry_put(entry);
1189
1190         return 0;
1191 }
1192
1193 static int
1194 mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft,
1195                                     struct flow_cls_offload *f)
1196 {
1197         struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1198         unsigned long cookie = f->cookie;
1199         struct mlx5_ct_entry *entry;
1200         u64 lastuse, packets, bytes;
1201
1202         spin_lock_bh(&ct_priv->ht_lock);
1203         entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1204         if (!entry) {
1205                 spin_unlock_bh(&ct_priv->ht_lock);
1206                 return -ENOENT;
1207         }
1208
1209         if (!mlx5_tc_ct_entry_valid(entry) || !refcount_inc_not_zero(&entry->refcnt)) {
1210                 spin_unlock_bh(&ct_priv->ht_lock);
1211                 return -EINVAL;
1212         }
1213
1214         spin_unlock_bh(&ct_priv->ht_lock);
1215
1216         mlx5_fc_query_cached(entry->counter->counter, &bytes, &packets, &lastuse);
1217         flow_stats_update(&f->stats, bytes, packets, 0, lastuse,
1218                           FLOW_ACTION_HW_STATS_DELAYED);
1219
1220         mlx5_tc_ct_entry_put(entry);
1221         return 0;
1222 }
1223
1224 static int
1225 mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data,
1226                               void *cb_priv)
1227 {
1228         struct flow_cls_offload *f = type_data;
1229         struct mlx5_ct_ft *ft = cb_priv;
1230
1231         if (type != TC_SETUP_CLSFLOWER)
1232                 return -EOPNOTSUPP;
1233
1234         switch (f->command) {
1235         case FLOW_CLS_REPLACE:
1236                 return mlx5_tc_ct_block_flow_offload_add(ft, f);
1237         case FLOW_CLS_DESTROY:
1238                 return mlx5_tc_ct_block_flow_offload_del(ft, f);
1239         case FLOW_CLS_STATS:
1240                 return mlx5_tc_ct_block_flow_offload_stats(ft, f);
1241         default:
1242                 break;
1243         }
1244
1245         return -EOPNOTSUPP;
1246 }
1247
1248 static bool
1249 mlx5_tc_ct_skb_to_tuple(struct sk_buff *skb, struct mlx5_ct_tuple *tuple,
1250                         u16 zone)
1251 {
1252         struct flow_keys flow_keys;
1253
1254         skb_reset_network_header(skb);
1255         skb_flow_dissect_flow_keys(skb, &flow_keys, FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP);
1256
1257         tuple->zone = zone;
1258
1259         if (flow_keys.basic.ip_proto != IPPROTO_TCP &&
1260             flow_keys.basic.ip_proto != IPPROTO_UDP &&
1261             flow_keys.basic.ip_proto != IPPROTO_GRE)
1262                 return false;
1263
1264         if (flow_keys.basic.ip_proto == IPPROTO_TCP ||
1265             flow_keys.basic.ip_proto == IPPROTO_UDP) {
1266                 tuple->port.src = flow_keys.ports.src;
1267                 tuple->port.dst = flow_keys.ports.dst;
1268         }
1269         tuple->n_proto = flow_keys.basic.n_proto;
1270         tuple->ip_proto = flow_keys.basic.ip_proto;
1271
1272         switch (flow_keys.basic.n_proto) {
1273         case htons(ETH_P_IP):
1274                 tuple->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1275                 tuple->ip.src_v4 = flow_keys.addrs.v4addrs.src;
1276                 tuple->ip.dst_v4 = flow_keys.addrs.v4addrs.dst;
1277                 break;
1278
1279         case htons(ETH_P_IPV6):
1280                 tuple->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1281                 tuple->ip.src_v6 = flow_keys.addrs.v6addrs.src;
1282                 tuple->ip.dst_v6 = flow_keys.addrs.v6addrs.dst;
1283                 break;
1284         default:
1285                 goto out;
1286         }
1287
1288         return true;
1289
1290 out:
1291         return false;
1292 }
1293
1294 int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec)
1295 {
1296         u32 ctstate = 0, ctstate_mask = 0;
1297
1298         mlx5e_tc_match_to_reg_get_match(spec, CTSTATE_TO_REG,
1299                                         &ctstate, &ctstate_mask);
1300
1301         if ((ctstate & ctstate_mask) == MLX5_CT_STATE_TRK_BIT)
1302                 return -EOPNOTSUPP;
1303
1304         ctstate_mask |= MLX5_CT_STATE_TRK_BIT;
1305         mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
1306                                     ctstate, ctstate_mask);
1307
1308         return 0;
1309 }
1310
1311 void mlx5_tc_ct_match_del(struct mlx5_tc_ct_priv *priv, struct mlx5_ct_attr *ct_attr)
1312 {
1313         if (!priv || !ct_attr->ct_labels_id)
1314                 return;
1315
1316         mlx5_put_label_mapping(priv, ct_attr->ct_labels_id);
1317 }
1318
1319 int
1320 mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
1321                      struct mlx5_flow_spec *spec,
1322                      struct flow_cls_offload *f,
1323                      struct mlx5_ct_attr *ct_attr,
1324                      struct netlink_ext_ack *extack)
1325 {
1326         bool trk, est, untrk, unest, new, rpl, unrpl, rel, unrel, inv, uninv;
1327         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
1328         struct flow_dissector_key_ct *mask, *key;
1329         u32 ctstate = 0, ctstate_mask = 0;
1330         u16 ct_state_on, ct_state_off;
1331         u16 ct_state, ct_state_mask;
1332         struct flow_match_ct match;
1333         u32 ct_labels[4];
1334
1335         if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT))
1336                 return 0;
1337
1338         if (!priv) {
1339                 NL_SET_ERR_MSG_MOD(extack,
1340                                    "offload of ct matching isn't available");
1341                 return -EOPNOTSUPP;
1342         }
1343
1344         flow_rule_match_ct(rule, &match);
1345
1346         key = match.key;
1347         mask = match.mask;
1348
1349         ct_state = key->ct_state;
1350         ct_state_mask = mask->ct_state;
1351
1352         if (ct_state_mask & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
1353                               TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED |
1354                               TCA_FLOWER_KEY_CT_FLAGS_NEW |
1355                               TCA_FLOWER_KEY_CT_FLAGS_REPLY |
1356                               TCA_FLOWER_KEY_CT_FLAGS_RELATED |
1357                               TCA_FLOWER_KEY_CT_FLAGS_INVALID)) {
1358                 NL_SET_ERR_MSG_MOD(extack,
1359                                    "only ct_state trk, est, new and rpl are supported for offload");
1360                 return -EOPNOTSUPP;
1361         }
1362
1363         ct_state_on = ct_state & ct_state_mask;
1364         ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask;
1365         trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
1366         new = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_NEW;
1367         est = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
1368         rpl = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
1369         rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
1370         inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
1371         untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
1372         unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
1373         unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
1374         unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
1375         uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
1376
1377         ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;
1378         ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
1379         ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0;
1380         ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;
1381         ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
1382         ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0;
1383         ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0;
1384         ctstate_mask |= uninv ? MLX5_CT_STATE_INVALID_BIT : 0;
1385
1386         if (rel) {
1387                 NL_SET_ERR_MSG_MOD(extack,
1388                                    "matching on ct_state +rel isn't supported");
1389                 return -EOPNOTSUPP;
1390         }
1391
1392         if (inv) {
1393                 NL_SET_ERR_MSG_MOD(extack,
1394                                    "matching on ct_state +inv isn't supported");
1395                 return -EOPNOTSUPP;
1396         }
1397
1398         if (new) {
1399                 NL_SET_ERR_MSG_MOD(extack,
1400                                    "matching on ct_state +new isn't supported");
1401                 return -EOPNOTSUPP;
1402         }
1403
1404         if (mask->ct_zone)
1405                 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
1406                                             key->ct_zone, MLX5_CT_ZONE_MASK);
1407         if (ctstate_mask)
1408                 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
1409                                             ctstate, ctstate_mask);
1410         if (mask->ct_mark)
1411                 mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG,
1412                                             key->ct_mark, mask->ct_mark);
1413         if (mask->ct_labels[0] || mask->ct_labels[1] || mask->ct_labels[2] ||
1414             mask->ct_labels[3]) {
1415                 ct_labels[0] = key->ct_labels[0] & mask->ct_labels[0];
1416                 ct_labels[1] = key->ct_labels[1] & mask->ct_labels[1];
1417                 ct_labels[2] = key->ct_labels[2] & mask->ct_labels[2];
1418                 ct_labels[3] = key->ct_labels[3] & mask->ct_labels[3];
1419                 if (mlx5_get_label_mapping(priv, ct_labels, &ct_attr->ct_labels_id))
1420                         return -EOPNOTSUPP;
1421                 mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG, ct_attr->ct_labels_id,
1422                                             MLX5_CT_LABELS_MASK);
1423         }
1424
1425         return 0;
1426 }
1427
1428 int
1429 mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
1430                         struct mlx5_flow_attr *attr,
1431                         struct mlx5e_tc_mod_hdr_acts *mod_acts,
1432                         const struct flow_action_entry *act,
1433                         struct netlink_ext_ack *extack)
1434 {
1435         if (!priv) {
1436                 NL_SET_ERR_MSG_MOD(extack,
1437                                    "offload of ct action isn't available");
1438                 return -EOPNOTSUPP;
1439         }
1440
1441         attr->ct_attr.zone = act->ct.zone;
1442         attr->ct_attr.ct_action = act->ct.action;
1443         attr->ct_attr.nf_ft = act->ct.flow_table;
1444
1445         return 0;
1446 }
1447
1448 static int tc_ct_pre_ct_add_rules(struct mlx5_ct_ft *ct_ft,
1449                                   struct mlx5_tc_ct_pre *pre_ct,
1450                                   bool nat)
1451 {
1452         struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1453         struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {};
1454         struct mlx5_core_dev *dev = ct_priv->dev;
1455         struct mlx5_flow_table *ft = pre_ct->ft;
1456         struct mlx5_flow_destination dest = {};
1457         struct mlx5_flow_act flow_act = {};
1458         struct mlx5_modify_hdr *mod_hdr;
1459         struct mlx5_flow_handle *rule;
1460         struct mlx5_flow_spec *spec;
1461         u32 ctstate;
1462         u16 zone;
1463         int err;
1464
1465         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1466         if (!spec)
1467                 return -ENOMEM;
1468
1469         zone = ct_ft->zone & MLX5_CT_ZONE_MASK;
1470         err = mlx5e_tc_match_to_reg_set(dev, &pre_mod_acts, ct_priv->ns_type,
1471                                         ZONE_TO_REG, zone);
1472         if (err) {
1473                 ct_dbg("Failed to set zone register mapping");
1474                 goto err_mapping;
1475         }
1476
1477         mod_hdr = mlx5_modify_header_alloc(dev, ct_priv->ns_type,
1478                                            pre_mod_acts.num_actions,
1479                                            pre_mod_acts.actions);
1480
1481         if (IS_ERR(mod_hdr)) {
1482                 err = PTR_ERR(mod_hdr);
1483                 ct_dbg("Failed to create pre ct mod hdr");
1484                 goto err_mapping;
1485         }
1486         pre_ct->modify_hdr = mod_hdr;
1487
1488         flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1489                           MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1490         flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
1491         flow_act.modify_hdr = mod_hdr;
1492         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1493
1494         /* add flow rule */
1495         mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
1496                                     zone, MLX5_CT_ZONE_MASK);
1497         ctstate = MLX5_CT_STATE_TRK_BIT;
1498         if (nat)
1499                 ctstate |= MLX5_CT_STATE_NAT_BIT;
1500         mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, ctstate, ctstate);
1501
1502         dest.ft = mlx5e_tc_post_act_get_ft(ct_priv->post_act);
1503         rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1504         if (IS_ERR(rule)) {
1505                 err = PTR_ERR(rule);
1506                 ct_dbg("Failed to add pre ct flow rule zone %d", zone);
1507                 goto err_flow_rule;
1508         }
1509         pre_ct->flow_rule = rule;
1510
1511         /* add miss rule */
1512         dest.ft = nat ? ct_priv->ct_nat : ct_priv->ct;
1513         rule = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1);
1514         if (IS_ERR(rule)) {
1515                 err = PTR_ERR(rule);
1516                 ct_dbg("Failed to add pre ct miss rule zone %d", zone);
1517                 goto err_miss_rule;
1518         }
1519         pre_ct->miss_rule = rule;
1520
1521         mlx5e_mod_hdr_dealloc(&pre_mod_acts);
1522         kvfree(spec);
1523         return 0;
1524
1525 err_miss_rule:
1526         mlx5_del_flow_rules(pre_ct->flow_rule);
1527 err_flow_rule:
1528         mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr);
1529 err_mapping:
1530         mlx5e_mod_hdr_dealloc(&pre_mod_acts);
1531         kvfree(spec);
1532         return err;
1533 }
1534
1535 static void
1536 tc_ct_pre_ct_del_rules(struct mlx5_ct_ft *ct_ft,
1537                        struct mlx5_tc_ct_pre *pre_ct)
1538 {
1539         struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1540         struct mlx5_core_dev *dev = ct_priv->dev;
1541
1542         mlx5_del_flow_rules(pre_ct->flow_rule);
1543         mlx5_del_flow_rules(pre_ct->miss_rule);
1544         mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr);
1545 }
1546
1547 static int
1548 mlx5_tc_ct_alloc_pre_ct(struct mlx5_ct_ft *ct_ft,
1549                         struct mlx5_tc_ct_pre *pre_ct,
1550                         bool nat)
1551 {
1552         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1553         struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1554         struct mlx5_core_dev *dev = ct_priv->dev;
1555         struct mlx5_flow_table_attr ft_attr = {};
1556         struct mlx5_flow_namespace *ns;
1557         struct mlx5_flow_table *ft;
1558         struct mlx5_flow_group *g;
1559         u32 metadata_reg_c_2_mask;
1560         u32 *flow_group_in;
1561         void *misc;
1562         int err;
1563
1564         ns = mlx5_get_flow_namespace(dev, ct_priv->ns_type);
1565         if (!ns) {
1566                 err = -EOPNOTSUPP;
1567                 ct_dbg("Failed to get flow namespace");
1568                 return err;
1569         }
1570
1571         flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1572         if (!flow_group_in)
1573                 return -ENOMEM;
1574
1575         ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
1576         ft_attr.prio =  ct_priv->ns_type ==  MLX5_FLOW_NAMESPACE_FDB ?
1577                         FDB_TC_OFFLOAD : MLX5E_TC_PRIO;
1578         ft_attr.max_fte = 2;
1579         ft_attr.level = 1;
1580         ft = mlx5_create_flow_table(ns, &ft_attr);
1581         if (IS_ERR(ft)) {
1582                 err = PTR_ERR(ft);
1583                 ct_dbg("Failed to create pre ct table");
1584                 goto out_free;
1585         }
1586         pre_ct->ft = ft;
1587
1588         /* create flow group */
1589         MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1590         MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
1591         MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1592                  MLX5_MATCH_MISC_PARAMETERS_2);
1593
1594         misc = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1595                             match_criteria.misc_parameters_2);
1596
1597         metadata_reg_c_2_mask = MLX5_CT_ZONE_MASK;
1598         metadata_reg_c_2_mask |= (MLX5_CT_STATE_TRK_BIT << 16);
1599         if (nat)
1600                 metadata_reg_c_2_mask |= (MLX5_CT_STATE_NAT_BIT << 16);
1601
1602         MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_2,
1603                  metadata_reg_c_2_mask);
1604
1605         g = mlx5_create_flow_group(ft, flow_group_in);
1606         if (IS_ERR(g)) {
1607                 err = PTR_ERR(g);
1608                 ct_dbg("Failed to create pre ct group");
1609                 goto err_flow_grp;
1610         }
1611         pre_ct->flow_grp = g;
1612
1613         /* create miss group */
1614         memset(flow_group_in, 0, inlen);
1615         MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
1616         MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
1617         g = mlx5_create_flow_group(ft, flow_group_in);
1618         if (IS_ERR(g)) {
1619                 err = PTR_ERR(g);
1620                 ct_dbg("Failed to create pre ct miss group");
1621                 goto err_miss_grp;
1622         }
1623         pre_ct->miss_grp = g;
1624
1625         err = tc_ct_pre_ct_add_rules(ct_ft, pre_ct, nat);
1626         if (err)
1627                 goto err_add_rules;
1628
1629         kvfree(flow_group_in);
1630         return 0;
1631
1632 err_add_rules:
1633         mlx5_destroy_flow_group(pre_ct->miss_grp);
1634 err_miss_grp:
1635         mlx5_destroy_flow_group(pre_ct->flow_grp);
1636 err_flow_grp:
1637         mlx5_destroy_flow_table(ft);
1638 out_free:
1639         kvfree(flow_group_in);
1640         return err;
1641 }
1642
1643 static void
1644 mlx5_tc_ct_free_pre_ct(struct mlx5_ct_ft *ct_ft,
1645                        struct mlx5_tc_ct_pre *pre_ct)
1646 {
1647         tc_ct_pre_ct_del_rules(ct_ft, pre_ct);
1648         mlx5_destroy_flow_group(pre_ct->miss_grp);
1649         mlx5_destroy_flow_group(pre_ct->flow_grp);
1650         mlx5_destroy_flow_table(pre_ct->ft);
1651 }
1652
1653 static int
1654 mlx5_tc_ct_alloc_pre_ct_tables(struct mlx5_ct_ft *ft)
1655 {
1656         int err;
1657
1658         err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct, false);
1659         if (err)
1660                 return err;
1661
1662         err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct_nat, true);
1663         if (err)
1664                 goto err_pre_ct_nat;
1665
1666         return 0;
1667
1668 err_pre_ct_nat:
1669         mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct);
1670         return err;
1671 }
1672
1673 static void
1674 mlx5_tc_ct_free_pre_ct_tables(struct mlx5_ct_ft *ft)
1675 {
1676         mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct_nat);
1677         mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct);
1678 }
1679
1680 /* To avoid false lock dependency warning set the ct_entries_ht lock
1681  * class different than the lock class of the ht being used when deleting
1682  * last flow from a group and then deleting a group, we get into del_sw_flow_group()
1683  * which call rhashtable_destroy on fg->ftes_hash which will take ht->mutex but
1684  * it's different than the ht->mutex here.
1685  */
1686 static struct lock_class_key ct_entries_ht_lock_key;
1687
1688 static struct mlx5_ct_ft *
1689 mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone,
1690                      struct nf_flowtable *nf_ft)
1691 {
1692         struct mlx5_ct_ft *ft;
1693         int err;
1694
1695         ft = rhashtable_lookup_fast(&ct_priv->zone_ht, &zone, zone_params);
1696         if (ft) {
1697                 refcount_inc(&ft->refcount);
1698                 return ft;
1699         }
1700
1701         ft = kzalloc(sizeof(*ft), GFP_KERNEL);
1702         if (!ft)
1703                 return ERR_PTR(-ENOMEM);
1704
1705         err = mapping_add(ct_priv->zone_mapping, &zone, &ft->zone_restore_id);
1706         if (err)
1707                 goto err_mapping;
1708
1709         ft->zone = zone;
1710         ft->nf_ft = nf_ft;
1711         ft->ct_priv = ct_priv;
1712         refcount_set(&ft->refcount, 1);
1713
1714         err = mlx5_tc_ct_alloc_pre_ct_tables(ft);
1715         if (err)
1716                 goto err_alloc_pre_ct;
1717
1718         err = rhashtable_init(&ft->ct_entries_ht, &cts_ht_params);
1719         if (err)
1720                 goto err_init;
1721
1722         lockdep_set_class(&ft->ct_entries_ht.mutex, &ct_entries_ht_lock_key);
1723
1724         err = rhashtable_insert_fast(&ct_priv->zone_ht, &ft->node,
1725                                      zone_params);
1726         if (err)
1727                 goto err_insert;
1728
1729         err = nf_flow_table_offload_add_cb(ft->nf_ft,
1730                                            mlx5_tc_ct_block_flow_offload, ft);
1731         if (err)
1732                 goto err_add_cb;
1733
1734         return ft;
1735
1736 err_add_cb:
1737         rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
1738 err_insert:
1739         rhashtable_destroy(&ft->ct_entries_ht);
1740 err_init:
1741         mlx5_tc_ct_free_pre_ct_tables(ft);
1742 err_alloc_pre_ct:
1743         mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
1744 err_mapping:
1745         kfree(ft);
1746         return ERR_PTR(err);
1747 }
1748
1749 static void
1750 mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg)
1751 {
1752         struct mlx5_ct_entry *entry = ptr;
1753
1754         mlx5_tc_ct_entry_put(entry);
1755 }
1756
1757 static void
1758 mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
1759 {
1760         if (!refcount_dec_and_test(&ft->refcount))
1761                 return;
1762
1763         flush_workqueue(ct_priv->wq);
1764         nf_flow_table_offload_del_cb(ft->nf_ft,
1765                                      mlx5_tc_ct_block_flow_offload, ft);
1766         rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
1767         rhashtable_free_and_destroy(&ft->ct_entries_ht,
1768                                     mlx5_tc_ct_flush_ft_entry,
1769                                     ct_priv);
1770         mlx5_tc_ct_free_pre_ct_tables(ft);
1771         mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
1772         kfree(ft);
1773 }
1774
1775 /* We translate the tc filter with CT action to the following HW model:
1776  *
1777  * +---------------------+
1778  * + ft prio (tc chain)  +
1779  * + original match      +
1780  * +---------------------+
1781  *      | set chain miss mapping
1782  *      | set fte_id
1783  *      | set tunnel_id
1784  *      | do decap
1785  *      v
1786  * +---------------------+
1787  * + pre_ct/pre_ct_nat   +  if matches     +-------------------------+
1788  * + zone+nat match      +---------------->+ post_act (see below)    +
1789  * +---------------------+  set zone       +-------------------------+
1790  *      | set zone
1791  *      v
1792  * +--------------------+
1793  * + CT (nat or no nat) +
1794  * + tuple + zone match +
1795  * +--------------------+
1796  *      | set mark
1797  *      | set labels_id
1798  *      | set established
1799  *      | set zone_restore
1800  *      | do nat (if needed)
1801  *      v
1802  * +--------------+
1803  * + post_act     + original filter actions
1804  * + fte_id match +------------------------>
1805  * +--------------+
1806  */
1807 static struct mlx5_flow_handle *
1808 __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv,
1809                           struct mlx5_flow_spec *orig_spec,
1810                           struct mlx5_flow_attr *attr)
1811 {
1812         bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
1813         struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
1814         struct mlx5e_tc_mod_hdr_acts *pre_mod_acts;
1815         u32 attr_sz = ns_to_attr_sz(ct_priv->ns_type);
1816         struct mlx5_flow_attr *pre_ct_attr;
1817         struct mlx5_modify_hdr *mod_hdr;
1818         struct mlx5_ct_flow *ct_flow;
1819         int chain_mapping = 0, err;
1820         struct mlx5_ct_ft *ft;
1821
1822         ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
1823         if (!ct_flow) {
1824                 return ERR_PTR(-ENOMEM);
1825         }
1826
1827         /* Register for CT established events */
1828         ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone,
1829                                   attr->ct_attr.nf_ft);
1830         if (IS_ERR(ft)) {
1831                 err = PTR_ERR(ft);
1832                 ct_dbg("Failed to register to ft callback");
1833                 goto err_ft;
1834         }
1835         ct_flow->ft = ft;
1836
1837         /* Base flow attributes of both rules on original rule attribute */
1838         ct_flow->pre_ct_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
1839         if (!ct_flow->pre_ct_attr) {
1840                 err = -ENOMEM;
1841                 goto err_alloc_pre;
1842         }
1843
1844         pre_ct_attr = ct_flow->pre_ct_attr;
1845         memcpy(pre_ct_attr, attr, attr_sz);
1846         pre_mod_acts = &pre_ct_attr->parse_attr->mod_hdr_acts;
1847
1848         /* Modify the original rule's action to fwd and modify, leave decap */
1849         pre_ct_attr->action = attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP;
1850         pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1851                                MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1852
1853         /* Write chain miss tag for miss in ct table as we
1854          * don't go though all prios of this chain as normal tc rules
1855          * miss.
1856          */
1857         err = mlx5_chains_get_chain_mapping(ct_priv->chains, attr->chain,
1858                                             &chain_mapping);
1859         if (err) {
1860                 ct_dbg("Failed to get chain register mapping for chain");
1861                 goto err_get_chain;
1862         }
1863         ct_flow->chain_mapping = chain_mapping;
1864
1865         err = mlx5e_tc_match_to_reg_set(priv->mdev, pre_mod_acts, ct_priv->ns_type,
1866                                         CHAIN_TO_REG, chain_mapping);
1867         if (err) {
1868                 ct_dbg("Failed to set chain register mapping");
1869                 goto err_mapping;
1870         }
1871
1872         /* If original flow is decap, we do it before going into ct table
1873          * so add a rewrite for the tunnel match_id.
1874          */
1875         if ((pre_ct_attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) &&
1876             attr->chain == 0) {
1877                 err = mlx5e_tc_match_to_reg_set(priv->mdev, pre_mod_acts,
1878                                                 ct_priv->ns_type,
1879                                                 TUNNEL_TO_REG,
1880                                                 attr->tunnel_id);
1881                 if (err) {
1882                         ct_dbg("Failed to set tunnel register mapping");
1883                         goto err_mapping;
1884                 }
1885         }
1886
1887         mod_hdr = mlx5_modify_header_alloc(priv->mdev, ct_priv->ns_type,
1888                                            pre_mod_acts->num_actions,
1889                                            pre_mod_acts->actions);
1890         if (IS_ERR(mod_hdr)) {
1891                 err = PTR_ERR(mod_hdr);
1892                 ct_dbg("Failed to create pre ct mod hdr");
1893                 goto err_mapping;
1894         }
1895         pre_ct_attr->modify_hdr = mod_hdr;
1896
1897         /* Change original rule point to ct table */
1898         pre_ct_attr->dest_chain = 0;
1899         pre_ct_attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft;
1900         ct_flow->pre_ct_rule = mlx5_tc_rule_insert(priv, orig_spec,
1901                                                    pre_ct_attr);
1902         if (IS_ERR(ct_flow->pre_ct_rule)) {
1903                 err = PTR_ERR(ct_flow->pre_ct_rule);
1904                 ct_dbg("Failed to add pre ct rule");
1905                 goto err_insert_orig;
1906         }
1907
1908         attr->ct_attr.ct_flow = ct_flow;
1909         mlx5e_mod_hdr_dealloc(pre_mod_acts);
1910
1911         return ct_flow->pre_ct_rule;
1912
1913 err_insert_orig:
1914         mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
1915 err_mapping:
1916         mlx5e_mod_hdr_dealloc(pre_mod_acts);
1917         mlx5_chains_put_chain_mapping(ct_priv->chains, ct_flow->chain_mapping);
1918 err_get_chain:
1919         kfree(ct_flow->pre_ct_attr);
1920 err_alloc_pre:
1921         mlx5_tc_ct_del_ft_cb(ct_priv, ft);
1922 err_ft:
1923         kfree(ct_flow);
1924         netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err);
1925         return ERR_PTR(err);
1926 }
1927
1928 struct mlx5_flow_handle *
1929 mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv,
1930                         struct mlx5_flow_spec *spec,
1931                         struct mlx5_flow_attr *attr,
1932                         struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
1933 {
1934         struct mlx5_flow_handle *rule;
1935
1936         if (!priv)
1937                 return ERR_PTR(-EOPNOTSUPP);
1938
1939         mutex_lock(&priv->control_lock);
1940         rule = __mlx5_tc_ct_flow_offload(priv, spec, attr);
1941         mutex_unlock(&priv->control_lock);
1942
1943         return rule;
1944 }
1945
1946 static void
1947 __mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv,
1948                          struct mlx5_ct_flow *ct_flow,
1949                          struct mlx5_flow_attr *attr)
1950 {
1951         struct mlx5_flow_attr *pre_ct_attr = ct_flow->pre_ct_attr;
1952         struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
1953
1954         mlx5_tc_rule_delete(priv, ct_flow->pre_ct_rule, pre_ct_attr);
1955         mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
1956
1957         mlx5_chains_put_chain_mapping(ct_priv->chains, ct_flow->chain_mapping);
1958         mlx5_tc_ct_del_ft_cb(ct_priv, ct_flow->ft);
1959
1960         kfree(ct_flow->pre_ct_attr);
1961         kfree(ct_flow);
1962 }
1963
1964 void
1965 mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv,
1966                        struct mlx5_flow_attr *attr)
1967 {
1968         struct mlx5_ct_flow *ct_flow = attr->ct_attr.ct_flow;
1969
1970         /* We are called on error to clean up stuff from parsing
1971          * but we don't have anything for now
1972          */
1973         if (!ct_flow)
1974                 return;
1975
1976         mutex_lock(&priv->control_lock);
1977         __mlx5_tc_ct_delete_flow(priv, ct_flow, attr);
1978         mutex_unlock(&priv->control_lock);
1979 }
1980
1981 static int
1982 mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv)
1983 {
1984         struct mlx5_flow_table *post_ct = mlx5e_tc_post_act_get_ft(ct_priv->post_act);
1985         struct mlx5_ct_fs_ops *fs_ops = mlx5_ct_fs_dmfs_ops_get();
1986         int err;
1987
1988         if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB &&
1989             ct_priv->dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_SMFS) {
1990                 ct_dbg("Using SMFS ct flow steering provider");
1991                 fs_ops = mlx5_ct_fs_smfs_ops_get();
1992         }
1993
1994         ct_priv->fs = kzalloc(sizeof(*ct_priv->fs) + fs_ops->priv_size, GFP_KERNEL);
1995         if (!ct_priv->fs)
1996                 return -ENOMEM;
1997
1998         ct_priv->fs->netdev = ct_priv->netdev;
1999         ct_priv->fs->dev = ct_priv->dev;
2000         ct_priv->fs_ops = fs_ops;
2001
2002         err = ct_priv->fs_ops->init(ct_priv->fs, ct_priv->ct, ct_priv->ct_nat, post_ct);
2003         if (err)
2004                 goto err_init;
2005
2006         return 0;
2007
2008 err_init:
2009         kfree(ct_priv->fs);
2010         return err;
2011 }
2012
2013 static int
2014 mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch *esw,
2015                                   const char **err_msg)
2016 {
2017         if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) {
2018                 /* vlan workaround should be avoided for multi chain rules.
2019                  * This is just a sanity check as pop vlan action should
2020                  * be supported by any FW that supports ignore_flow_level
2021                  */
2022
2023                 *err_msg = "firmware vlan actions support is missing";
2024                 return -EOPNOTSUPP;
2025         }
2026
2027         if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev,
2028                                     fdb_modify_header_fwd_to_table)) {
2029                 /* CT always writes to registers which are mod header actions.
2030                  * Therefore, mod header and goto is required
2031                  */
2032
2033                 *err_msg = "firmware fwd and modify support is missing";
2034                 return -EOPNOTSUPP;
2035         }
2036
2037         if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
2038                 *err_msg = "register loopback isn't supported";
2039                 return -EOPNOTSUPP;
2040         }
2041
2042         return 0;
2043 }
2044
2045 static int
2046 mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv,
2047                               enum mlx5_flow_namespace_type ns_type,
2048                               struct mlx5e_post_act *post_act)
2049 {
2050         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2051         const char *err_msg = NULL;
2052         int err = 0;
2053
2054 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
2055         /* cannot restore chain ID on HW miss */
2056
2057         err_msg = "tc skb extension missing";
2058         err = -EOPNOTSUPP;
2059         goto out_err;
2060 #endif
2061         if (IS_ERR_OR_NULL(post_act)) {
2062                 /* Ignore_flow_level support isn't supported by default for VFs and so post_act
2063                  * won't be supported. Skip showing error msg.
2064                  */
2065                 if (priv->mdev->coredev_type != MLX5_COREDEV_VF)
2066                         err_msg = "post action is missing";
2067                 err = -EOPNOTSUPP;
2068                 goto out_err;
2069         }
2070
2071         if (ns_type == MLX5_FLOW_NAMESPACE_FDB)
2072                 err = mlx5_tc_ct_init_check_esw_support(esw, &err_msg);
2073
2074 out_err:
2075         if (err && err_msg)
2076                 netdev_dbg(priv->netdev, "tc ct offload not supported, %s\n", err_msg);
2077         return err;
2078 }
2079
2080 static void
2081 mlx5_ct_tc_create_dbgfs(struct mlx5_tc_ct_priv *ct_priv)
2082 {
2083         bool is_fdb = ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB;
2084         struct mlx5_tc_ct_debugfs *ct_dbgfs = &ct_priv->debugfs;
2085         char dirname[16] = {};
2086
2087         if (sscanf(dirname, "ct_%s", is_fdb ? "fdb" : "nic") < 0)
2088                 return;
2089
2090         ct_dbgfs->root = debugfs_create_dir(dirname, mlx5_debugfs_get_dev_root(ct_priv->dev));
2091         debugfs_create_atomic_t("offloaded", 0400, ct_dbgfs->root,
2092                                 &ct_dbgfs->stats.offloaded);
2093         debugfs_create_atomic_t("rx_dropped", 0400, ct_dbgfs->root,
2094                                 &ct_dbgfs->stats.rx_dropped);
2095 }
2096
2097 static void
2098 mlx5_ct_tc_remove_dbgfs(struct mlx5_tc_ct_priv *ct_priv)
2099 {
2100         debugfs_remove_recursive(ct_priv->debugfs.root);
2101 }
2102
2103 #define INIT_ERR_PREFIX "tc ct offload init failed"
2104
2105 struct mlx5_tc_ct_priv *
2106 mlx5_tc_ct_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains,
2107                 struct mod_hdr_tbl *mod_hdr,
2108                 enum mlx5_flow_namespace_type ns_type,
2109                 struct mlx5e_post_act *post_act)
2110 {
2111         struct mlx5_tc_ct_priv *ct_priv;
2112         struct mlx5_core_dev *dev;
2113         u64 mapping_id;
2114         int err;
2115
2116         dev = priv->mdev;
2117         err = mlx5_tc_ct_init_check_support(priv, ns_type, post_act);
2118         if (err)
2119                 goto err_support;
2120
2121         ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL);
2122         if (!ct_priv)
2123                 goto err_alloc;
2124
2125         mapping_id = mlx5_query_nic_system_image_guid(dev);
2126
2127         ct_priv->zone_mapping = mapping_create_for_id(mapping_id, MAPPING_TYPE_ZONE,
2128                                                       sizeof(u16), 0, true);
2129         if (IS_ERR(ct_priv->zone_mapping)) {
2130                 err = PTR_ERR(ct_priv->zone_mapping);
2131                 goto err_mapping_zone;
2132         }
2133
2134         ct_priv->labels_mapping = mapping_create_for_id(mapping_id, MAPPING_TYPE_LABELS,
2135                                                         sizeof(u32) * 4, 0, true);
2136         if (IS_ERR(ct_priv->labels_mapping)) {
2137                 err = PTR_ERR(ct_priv->labels_mapping);
2138                 goto err_mapping_labels;
2139         }
2140
2141         spin_lock_init(&ct_priv->ht_lock);
2142         ct_priv->ns_type = ns_type;
2143         ct_priv->chains = chains;
2144         ct_priv->netdev = priv->netdev;
2145         ct_priv->dev = priv->mdev;
2146         ct_priv->mod_hdr_tbl = mod_hdr;
2147         ct_priv->ct = mlx5_chains_create_global_table(chains);
2148         if (IS_ERR(ct_priv->ct)) {
2149                 err = PTR_ERR(ct_priv->ct);
2150                 mlx5_core_warn(dev,
2151                                "%s, failed to create ct table err: %d\n",
2152                                INIT_ERR_PREFIX, err);
2153                 goto err_ct_tbl;
2154         }
2155
2156         ct_priv->ct_nat = mlx5_chains_create_global_table(chains);
2157         if (IS_ERR(ct_priv->ct_nat)) {
2158                 err = PTR_ERR(ct_priv->ct_nat);
2159                 mlx5_core_warn(dev,
2160                                "%s, failed to create ct nat table err: %d\n",
2161                                INIT_ERR_PREFIX, err);
2162                 goto err_ct_nat_tbl;
2163         }
2164
2165         ct_priv->post_act = post_act;
2166         mutex_init(&ct_priv->control_lock);
2167         if (rhashtable_init(&ct_priv->zone_ht, &zone_params))
2168                 goto err_ct_zone_ht;
2169         if (rhashtable_init(&ct_priv->ct_tuples_ht, &tuples_ht_params))
2170                 goto err_ct_tuples_ht;
2171         if (rhashtable_init(&ct_priv->ct_tuples_nat_ht, &tuples_nat_ht_params))
2172                 goto err_ct_tuples_nat_ht;
2173
2174         ct_priv->wq = alloc_ordered_workqueue("mlx5e_ct_priv_wq", 0);
2175         if (!ct_priv->wq) {
2176                 err = -ENOMEM;
2177                 goto err_wq;
2178         }
2179
2180         err = mlx5_tc_ct_fs_init(ct_priv);
2181         if (err)
2182                 goto err_init_fs;
2183
2184         mlx5_ct_tc_create_dbgfs(ct_priv);
2185         return ct_priv;
2186
2187 err_init_fs:
2188         destroy_workqueue(ct_priv->wq);
2189 err_wq:
2190         rhashtable_destroy(&ct_priv->ct_tuples_nat_ht);
2191 err_ct_tuples_nat_ht:
2192         rhashtable_destroy(&ct_priv->ct_tuples_ht);
2193 err_ct_tuples_ht:
2194         rhashtable_destroy(&ct_priv->zone_ht);
2195 err_ct_zone_ht:
2196         mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat);
2197 err_ct_nat_tbl:
2198         mlx5_chains_destroy_global_table(chains, ct_priv->ct);
2199 err_ct_tbl:
2200         mapping_destroy(ct_priv->labels_mapping);
2201 err_mapping_labels:
2202         mapping_destroy(ct_priv->zone_mapping);
2203 err_mapping_zone:
2204         kfree(ct_priv);
2205 err_alloc:
2206 err_support:
2207
2208         return NULL;
2209 }
2210
2211 void
2212 mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv)
2213 {
2214         struct mlx5_fs_chains *chains;
2215
2216         if (!ct_priv)
2217                 return;
2218
2219         destroy_workqueue(ct_priv->wq);
2220         mlx5_ct_tc_remove_dbgfs(ct_priv);
2221         chains = ct_priv->chains;
2222
2223         ct_priv->fs_ops->destroy(ct_priv->fs);
2224         kfree(ct_priv->fs);
2225
2226         mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat);
2227         mlx5_chains_destroy_global_table(chains, ct_priv->ct);
2228         mapping_destroy(ct_priv->zone_mapping);
2229         mapping_destroy(ct_priv->labels_mapping);
2230
2231         rhashtable_destroy(&ct_priv->ct_tuples_ht);
2232         rhashtable_destroy(&ct_priv->ct_tuples_nat_ht);
2233         rhashtable_destroy(&ct_priv->zone_ht);
2234         mutex_destroy(&ct_priv->control_lock);
2235         kfree(ct_priv);
2236 }
2237
2238 bool
2239 mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv *ct_priv,
2240                          struct sk_buff *skb, u8 zone_restore_id)
2241 {
2242         struct mlx5_ct_tuple tuple = {};
2243         struct mlx5_ct_entry *entry;
2244         u16 zone;
2245
2246         if (!ct_priv || !zone_restore_id)
2247                 return true;
2248
2249         if (mapping_find(ct_priv->zone_mapping, zone_restore_id, &zone))
2250                 goto out_inc_drop;
2251
2252         if (!mlx5_tc_ct_skb_to_tuple(skb, &tuple, zone))
2253                 goto out_inc_drop;
2254
2255         spin_lock(&ct_priv->ht_lock);
2256
2257         entry = mlx5_tc_ct_entry_get(ct_priv, &tuple);
2258         if (!entry) {
2259                 spin_unlock(&ct_priv->ht_lock);
2260                 goto out_inc_drop;
2261         }
2262
2263         if (IS_ERR(entry)) {
2264                 spin_unlock(&ct_priv->ht_lock);
2265                 goto out_inc_drop;
2266         }
2267         spin_unlock(&ct_priv->ht_lock);
2268
2269         tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie);
2270         __mlx5_tc_ct_entry_put(entry);
2271
2272         return true;
2273
2274 out_inc_drop:
2275         atomic_inc(&ct_priv->debugfs.stats.rx_dropped);
2276         return false;
2277 }