drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */
   3
   4 #include <linux/refcount.h>
   5
   6 #include "en_tc.h"
   7 #include "en/tc_priv.h"
   8 #include "en/tc_ct.h"
   9 #include "en/tc/ct_fs.h"
  10
  11 #include "lib/smfs.h"
  12
  13 #define INIT_ERR_PREFIX "ct_fs_smfs init failed"
  14 #define ct_dbg(fmt, args...)\
  15         netdev_dbg(fs->netdev, "ct_fs_smfs debug: " fmt "\n", ##args)
  16 #define MLX5_CT_TCP_FLAGS_MASK cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16)
  17
  18 struct mlx5_ct_fs_smfs_matcher {
  19         struct mlx5dr_matcher *dr_matcher;
  20         struct list_head list;
  21         int prio;
  22         refcount_t ref;
  23 };
  24
  25 struct mlx5_ct_fs_smfs_matchers {
  26         struct mlx5_ct_fs_smfs_matcher smfs_matchers[4];
  27         struct list_head used;
  28 };
  29
  30 struct mlx5_ct_fs_smfs {
  31         struct mlx5dr_table *ct_tbl, *ct_nat_tbl;
  32         struct mlx5_ct_fs_smfs_matchers matchers;
  33         struct mlx5_ct_fs_smfs_matchers matchers_nat;
  34         struct mlx5dr_action *fwd_action;
  35         struct mlx5_flow_table *ct_nat;
  36         struct mutex lock; /* Guards matchers */
  37 };
  38
  39 struct mlx5_ct_fs_smfs_rule {
  40         struct mlx5_ct_fs_rule fs_rule;
  41         struct mlx5dr_rule *rule;
  42         struct mlx5dr_action *count_action;
  43         struct mlx5_ct_fs_smfs_matcher *smfs_matcher;
  44 };
  45
  46 static inline void
  47 mlx5_ct_fs_smfs_fill_mask(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, bool ipv4, bool tcp)
  48 {
  49         void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers);
  50
  51         if (likely(MLX5_CAP_FLOWTABLE_NIC_RX(fs->dev, ft_field_support.outer_ip_version)))
  52                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_version);
  53         else
  54                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
  55
  56         MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
  57         if (likely(ipv4)) {
  58                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c,
  59                                  src_ipv4_src_ipv6.ipv4_layout.ipv4);
  60                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c,
  61                                  dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
  62         } else {
  63                 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
  64                                     dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
  65                        0xFF,
  66                        MLX5_FLD_SZ_BYTES(fte_match_set_lyr_2_4,
  67                                          dst_ipv4_dst_ipv6.ipv6_layout.ipv6));
  68                 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
  69                                     src_ipv4_src_ipv6.ipv6_layout.ipv6),
  70                        0xFF,
  71                        MLX5_FLD_SZ_BYTES(fte_match_set_lyr_2_4,
  72                                          src_ipv4_src_ipv6.ipv6_layout.ipv6));
  73         }
  74
  75         if (likely(tcp)) {
  76                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, tcp_sport);
  77                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, tcp_dport);
  78                 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
  79                          ntohs(MLX5_CT_TCP_FLAGS_MASK));
  80         } else {
  81                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, udp_sport);
  82                 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, udp_dport);
  83         }
  84
  85         mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, 0, MLX5_CT_ZONE_MASK);
  86 }
  87
  88 static struct mlx5dr_matcher *
  89 mlx5_ct_fs_smfs_matcher_create(struct mlx5_ct_fs *fs, struct mlx5dr_table *tbl, bool ipv4,
  90                                bool tcp, u32 priority)
  91 {
  92         struct mlx5dr_matcher *dr_matcher;
  93         struct mlx5_flow_spec *spec;
  94
  95         spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
  96         if (!spec)
  97                 return ERR_PTR(-ENOMEM);
  98
  99         mlx5_ct_fs_smfs_fill_mask(fs, spec, ipv4, tcp);
 100         spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2 | MLX5_MATCH_OUTER_HEADERS;
 101
 102         dr_matcher = mlx5_smfs_matcher_create(tbl, priority, spec);
 103         kfree(spec);
 104         if (!dr_matcher)
 105                 return ERR_PTR(-EINVAL);
 106
 107         return dr_matcher;
 108 }
 109
 110 static struct mlx5_ct_fs_smfs_matcher *
 111 mlx5_ct_fs_smfs_matcher_get(struct mlx5_ct_fs *fs, bool nat, bool ipv4, bool tcp)
 112 {
 113         struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
 114         struct mlx5_ct_fs_smfs_matcher *m, *smfs_matcher;
 115         struct mlx5_ct_fs_smfs_matchers *matchers;
 116         struct mlx5dr_matcher *dr_matcher;
 117         struct mlx5dr_table *tbl;
 118         struct list_head *prev;
 119         int prio;
 120
 121         matchers = nat ? &fs_smfs->matchers_nat : &fs_smfs->matchers;
 122         smfs_matcher = &matchers->smfs_matchers[ipv4 * 2 + tcp];
 123
 124         if (refcount_inc_not_zero(&smfs_matcher->ref))
 125                 return smfs_matcher;
 126
 127         mutex_lock(&fs_smfs->lock);
 128
 129         /* Retry with lock, as another thread might have already created the relevant matcher
 130          * till we acquired the lock
 131          */
 132         if (refcount_inc_not_zero(&smfs_matcher->ref))
 133                 goto out_unlock;
 134
 135         // Find next available priority in sorted used list
 136         prio = 0;
 137         prev = &matchers->used;
 138         list_for_each_entry(m, &matchers->used, list) {
 139                 prev = &m->list;
 140
 141                 if (m->prio == prio)
 142                         prio = m->prio + 1;
 143                 else
 144                         break;
 145         }
 146
 147         tbl = nat ? fs_smfs->ct_nat_tbl : fs_smfs->ct_tbl;
 148         dr_matcher = mlx5_ct_fs_smfs_matcher_create(fs, tbl, ipv4, tcp, prio);
 149         if (IS_ERR(dr_matcher)) {
 150                 netdev_warn(fs->netdev,
 151                             "ct_fs_smfs: failed to create matcher (nat %d, ipv4 %d, tcp %d), err: %ld\n",
 152                             nat, ipv4, tcp, PTR_ERR(dr_matcher));
 153
 154                 smfs_matcher = ERR_CAST(dr_matcher);
 155                 goto out_unlock;
 156         }
 157
 158         smfs_matcher->dr_matcher = dr_matcher;
 159         smfs_matcher->prio = prio;
 160         list_add(&smfs_matcher->list, prev);
 161         refcount_set(&smfs_matcher->ref, 1);
 162
 163 out_unlock:
 164         mutex_unlock(&fs_smfs->lock);
 165         return smfs_matcher;
 166 }
 167
 168 static void
 169 mlx5_ct_fs_smfs_matcher_put(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_smfs_matcher *smfs_matcher)
 170 {
 171         struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
 172
 173         if (!refcount_dec_and_mutex_lock(&smfs_matcher->ref, &fs_smfs->lock))
 174                 return;
 175
 176         mlx5_smfs_matcher_destroy(smfs_matcher->dr_matcher);
 177         list_del(&smfs_matcher->list);
 178         mutex_unlock(&fs_smfs->lock);
 179 }
 180
 181 static int
 182 mlx5_ct_fs_smfs_init(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct,
 183                      struct mlx5_flow_table *ct_nat, struct mlx5_flow_table *post_ct)
 184 {
 185         struct mlx5dr_table *ct_tbl, *ct_nat_tbl, *post_ct_tbl;
 186         struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
 187
 188         post_ct_tbl = mlx5_smfs_table_get_from_fs_ft(post_ct);
 189         ct_nat_tbl = mlx5_smfs_table_get_from_fs_ft(ct_nat);
 190         ct_tbl = mlx5_smfs_table_get_from_fs_ft(ct);
 191         fs_smfs->ct_nat = ct_nat;
 192
 193         if (!ct_tbl || !ct_nat_tbl || !post_ct_tbl) {
 194                 netdev_warn(fs->netdev, "ct_fs_smfs: failed to init, missing backing dr tables");
 195                 return -EOPNOTSUPP;
 196         }
 197
 198         ct_dbg("using smfs steering");
 199
 200         fs_smfs->fwd_action = mlx5_smfs_action_create_dest_table(post_ct_tbl);
 201         if (!fs_smfs->fwd_action) {
 202                 return -EINVAL;
 203         }
 204
 205         fs_smfs->ct_tbl = ct_tbl;
 206         fs_smfs->ct_nat_tbl = ct_nat_tbl;
 207         mutex_init(&fs_smfs->lock);
 208         INIT_LIST_HEAD(&fs_smfs->matchers.used);
 209         INIT_LIST_HEAD(&fs_smfs->matchers_nat.used);
 210
 211         return 0;
 212 }
 213
 214 static void
 215 mlx5_ct_fs_smfs_destroy(struct mlx5_ct_fs *fs)
 216 {
 217         struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
 218
 219         mlx5_smfs_action_destroy(fs_smfs->fwd_action);
 220 }
 221
 222 static inline bool
 223 mlx5_tc_ct_valid_used_dissector_keys(const u32 used_keys)
 224 {
 225 #define DISSECTOR_BIT(name) BIT(FLOW_DISSECTOR_KEY_ ## name)
 226         const u32 basic_keys = DISSECTOR_BIT(BASIC) | DISSECTOR_BIT(CONTROL) |
 227                                DISSECTOR_BIT(PORTS) | DISSECTOR_BIT(META);
 228         const u32 ipv4_tcp = basic_keys | DISSECTOR_BIT(IPV4_ADDRS) | DISSECTOR_BIT(TCP);
 229         const u32 ipv4_udp = basic_keys | DISSECTOR_BIT(IPV4_ADDRS);
 230         const u32 ipv6_tcp = basic_keys | DISSECTOR_BIT(IPV6_ADDRS) | DISSECTOR_BIT(TCP);
 231         const u32 ipv6_udp = basic_keys | DISSECTOR_BIT(IPV6_ADDRS);
 232
 233         return (used_keys == ipv4_tcp || used_keys == ipv4_udp || used_keys == ipv6_tcp ||
 234                 used_keys == ipv6_udp);
 235 }
 236
 237 static bool
 238 mlx5_ct_fs_smfs_ct_validate_flow_rule(struct mlx5_ct_fs *fs, struct flow_rule *flow_rule)
 239 {
 240         struct flow_match_ipv4_addrs ipv4_addrs;
 241         struct flow_match_ipv6_addrs ipv6_addrs;
 242         struct flow_match_control control;
 243         struct flow_match_basic basic;
 244         struct flow_match_ports ports;
 245         struct flow_match_tcp tcp;
 246
 247         if (!mlx5_tc_ct_valid_used_dissector_keys(flow_rule->match.dissector->used_keys)) {
 248                 ct_dbg("rule uses unexpected dissectors (0x%08x)",
 249                        flow_rule->match.dissector->used_keys);
 250                 return false;
 251         }
 252
 253         flow_rule_match_basic(flow_rule, &basic);
 254         flow_rule_match_control(flow_rule, &control);
 255         flow_rule_match_ipv4_addrs(flow_rule, &ipv4_addrs);
 256         flow_rule_match_ipv6_addrs(flow_rule, &ipv6_addrs);
 257         flow_rule_match_ports(flow_rule, &ports);
 258         flow_rule_match_tcp(flow_rule, &tcp);
 259
 260         if (basic.mask->n_proto != htons(0xFFFF) ||
 261             (basic.key->n_proto != htons(ETH_P_IP) && basic.key->n_proto != htons(ETH_P_IPV6)) ||
 262             basic.mask->ip_proto != 0xFF ||
 263             (basic.key->ip_proto != IPPROTO_UDP && basic.key->ip_proto != IPPROTO_TCP)) {
 264                 ct_dbg("rule uses unexpected basic match (n_proto 0x%04x/0x%04x, ip_proto 0x%02x/0x%02x)",
 265                        ntohs(basic.key->n_proto), ntohs(basic.mask->n_proto),
 266                        basic.key->ip_proto, basic.mask->ip_proto);
 267                 return false;
 268         }
 269
 270         if (ports.mask->src != htons(0xFFFF) || ports.mask->dst != htons(0xFFFF)) {
 271                 ct_dbg("rule uses ports match (src 0x%04x, dst 0x%04x)",
 272                        ports.mask->src, ports.mask->dst);
 273                 return false;
 274         }
 275
 276         if (basic.key->ip_proto == IPPROTO_TCP && tcp.mask->flags != MLX5_CT_TCP_FLAGS_MASK) {
 277                 ct_dbg("rule uses unexpected tcp match (flags 0x%02x)", tcp.mask->flags);
 278                 return false;
 279         }
 280
 281         return true;
 282 }
 283
 284 static struct mlx5_ct_fs_rule *
 285 mlx5_ct_fs_smfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec,
 286                             struct mlx5_flow_attr *attr, struct flow_rule *flow_rule)
 287 {
 288         struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs);
 289         struct mlx5_ct_fs_smfs_matcher *smfs_matcher;
 290         struct mlx5_ct_fs_smfs_rule *smfs_rule;
 291         struct mlx5dr_action *actions[5];
 292         struct mlx5dr_rule *rule;
 293         int num_actions = 0, err;
 294         bool nat, tcp, ipv4;
 295
 296         if (!mlx5_ct_fs_smfs_ct_validate_flow_rule(fs, flow_rule))
 297                 return ERR_PTR(-EOPNOTSUPP);
 298
 299         smfs_rule = kzalloc(sizeof(*smfs_rule), GFP_KERNEL);
 300         if (!smfs_rule)
 301                 return ERR_PTR(-ENOMEM);
 302
 303         smfs_rule->count_action = mlx5_smfs_action_create_flow_counter(mlx5_fc_id(attr->counter));
 304         if (!smfs_rule->count_action) {
 305                 err = -EINVAL;
 306                 goto err_count;
 307         }
 308
 309         actions[num_actions++] = smfs_rule->count_action;
 310         actions[num_actions++] = attr->modify_hdr->action.dr_action;
 311         actions[num_actions++] = fs_smfs->fwd_action;
 312
 313         nat = (attr->ft == fs_smfs->ct_nat);
 314         ipv4 = mlx5e_tc_get_ip_version(spec, true) == 4;
 315         tcp = MLX5_GET(fte_match_param, spec->match_value,
 316                        outer_headers.ip_protocol) == IPPROTO_TCP;
 317
 318         smfs_matcher = mlx5_ct_fs_smfs_matcher_get(fs, nat, ipv4, tcp);
 319         if (IS_ERR(smfs_matcher)) {
 320                 err = PTR_ERR(smfs_matcher);
 321                 goto err_matcher;
 322         }
 323
 324         rule = mlx5_smfs_rule_create(smfs_matcher->dr_matcher, spec, num_actions, actions,
 325                                      MLX5_FLOW_CONTEXT_FLOW_SOURCE_ANY_VPORT);
 326         if (!rule) {
 327                 err = -EINVAL;
 328                 goto err_create;
 329         }
 330
 331         smfs_rule->rule = rule;
 332         smfs_rule->smfs_matcher = smfs_matcher;
 333
 334         return &smfs_rule->fs_rule;
 335
 336 err_create:
 337         mlx5_ct_fs_smfs_matcher_put(fs, smfs_matcher);
 338 err_matcher:
 339         mlx5_smfs_action_destroy(smfs_rule->count_action);
 340 err_count:
 341         kfree(smfs_rule);
 342         return ERR_PTR(err);
 343 }
 344
 345 static void
 346 mlx5_ct_fs_smfs_ct_rule_del(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule)
 347 {
 348         struct mlx5_ct_fs_smfs_rule *smfs_rule = container_of(fs_rule,
 349                                                               struct mlx5_ct_fs_smfs_rule,
 350                                                               fs_rule);
 351
 352         mlx5_smfs_rule_destroy(smfs_rule->rule);
 353         mlx5_ct_fs_smfs_matcher_put(fs, smfs_rule->smfs_matcher);
 354         mlx5_smfs_action_destroy(smfs_rule->count_action);
 355         kfree(smfs_rule);
 356 }
 357
 358 static struct mlx5_ct_fs_ops fs_smfs_ops = {
 359         .ct_rule_add = mlx5_ct_fs_smfs_ct_rule_add,
 360         .ct_rule_del = mlx5_ct_fs_smfs_ct_rule_del,
 361
 362         .init = mlx5_ct_fs_smfs_init,
 363         .destroy = mlx5_ct_fs_smfs_destroy,
 364
 365         .priv_size = sizeof(struct mlx5_ct_fs_smfs),
 366 };
 367
 368 struct mlx5_ct_fs_ops *
 369 mlx5_ct_fs_smfs_ops_get(void)
 370 {
 371         return &fs_smfs_ops;
 372 }