drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 // Copyright (c) 2019 Mellanox Technologies.
   3
   4 #include <linux/mlx5/fs.h>
   5 #include "eswitch.h"
   6 #include "en_tc.h"
   7 #include "fs_core.h"
   8
   9 struct mlx5_termtbl_handle {
  10         struct hlist_node termtbl_hlist;
  11
  12         struct mlx5_flow_table *termtbl;
  13         struct mlx5_flow_act flow_act;
  14         struct mlx5_flow_destination dest;
  15
  16         struct mlx5_flow_handle *rule;
  17         int ref_count;
  18 };
  19
  20 static u32
  21 mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act,
  22                           struct mlx5_flow_destination *dest)
  23 {
  24         u32 hash;
  25
  26         hash = jhash_1word(flow_act->action, 0);
  27         hash = jhash((const void *)&flow_act->vlan,
  28                      sizeof(flow_act->vlan), hash);
  29         hash = jhash((const void *)&dest->vport.num,
  30                      sizeof(dest->vport.num), hash);
  31         hash = jhash((const void *)&dest->vport.vhca_id,
  32                      sizeof(dest->vport.num), hash);
  33         if (flow_act->pkt_reformat)
  34                 hash = jhash(flow_act->pkt_reformat,
  35                              sizeof(*flow_act->pkt_reformat),
  36                              hash);
  37         return hash;
  38 }
  39
  40 static int
  41 mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1,
  42                          struct mlx5_flow_destination *dest1,
  43                          struct mlx5_flow_act *flow_act2,
  44                          struct mlx5_flow_destination *dest2)
  45 {
  46         int ret;
  47
  48         ret = flow_act1->action != flow_act2->action ||
  49               dest1->vport.num != dest2->vport.num ||
  50               dest1->vport.vhca_id != dest2->vport.vhca_id ||
  51               memcmp(&flow_act1->vlan, &flow_act2->vlan,
  52                      sizeof(flow_act1->vlan));
  53         if (ret)
  54                 return ret;
  55
  56         if (flow_act1->pkt_reformat && flow_act2->pkt_reformat)
  57                 return memcmp(flow_act1->pkt_reformat, flow_act2->pkt_reformat,
  58                               sizeof(*flow_act1->pkt_reformat));
  59
  60         return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat);
  61 }
  62
  63 static int
  64 mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev,
  65                             struct mlx5_termtbl_handle *tt,
  66                             struct mlx5_flow_act *flow_act)
  67 {
  68         struct mlx5_flow_table_attr ft_attr = {};
  69         struct mlx5_flow_namespace *root_ns;
  70         int err, err2;
  71
  72         root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
  73         if (!root_ns) {
  74                 esw_warn(dev, "Failed to get FDB flow namespace\n");
  75                 return -EOPNOTSUPP;
  76         }
  77
  78         /* As this is the terminating action then the termination table is the
  79          * same prio as the slow path
  80          */
  81         ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED |
  82                         MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
  83         ft_attr.prio = FDB_TC_OFFLOAD;
  84         ft_attr.max_fte = 1;
  85         ft_attr.level = 1;
  86         ft_attr.autogroup.max_num_groups = 1;
  87         tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
  88         if (IS_ERR(tt->termtbl)) {
  89                 err = PTR_ERR(tt->termtbl);
  90                 esw_warn(dev, "Failed to create termination table, err %pe\n", tt->termtbl);
  91                 return err;
  92         }
  93
  94         tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act,
  95                                        &tt->dest, 1);
  96         if (IS_ERR(tt->rule)) {
  97                 err = PTR_ERR(tt->rule);
  98                 esw_warn(dev, "Failed to create termination table rule, err %pe\n", tt->rule);
  99                 goto add_flow_err;
 100         }
 101         return 0;
 102
 103 add_flow_err:
 104         err2 = mlx5_destroy_flow_table(tt->termtbl);
 105         if (err2)
 106                 esw_warn(dev, "Failed to destroy termination table, err %d\n", err2);
 107
 108         return err;
 109 }
 110
 111 static struct mlx5_termtbl_handle *
 112 mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw,
 113                                 struct mlx5_flow_act *flow_act,
 114                                 struct mlx5_flow_destination *dest,
 115                                 struct mlx5_esw_flow_attr *attr)
 116 {
 117         struct mlx5_termtbl_handle *tt;
 118         bool found = false;
 119         u32 hash_key;
 120         int err;
 121
 122         mutex_lock(&esw->offloads.termtbl_mutex);
 123         hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest);
 124         hash_for_each_possible(esw->offloads.termtbl_tbl, tt,
 125                                termtbl_hlist, hash_key) {
 126                 if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest,
 127                                               flow_act, dest)) {
 128                         found = true;
 129                         break;
 130                 }
 131         }
 132         if (found)
 133                 goto tt_add_ref;
 134
 135         tt = kzalloc(sizeof(*tt), GFP_KERNEL);
 136         if (!tt) {
 137                 err = -ENOMEM;
 138                 goto tt_create_err;
 139         }
 140
 141         tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
 142         tt->dest.vport.num = dest->vport.num;
 143         tt->dest.vport.vhca_id = dest->vport.vhca_id;
 144         tt->dest.vport.flags = dest->vport.flags;
 145         memcpy(&tt->flow_act, flow_act, sizeof(*flow_act));
 146
 147         err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act);
 148         if (err)
 149                 goto tt_create_err;
 150
 151         hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key);
 152 tt_add_ref:
 153         tt->ref_count++;
 154         mutex_unlock(&esw->offloads.termtbl_mutex);
 155         return tt;
 156 tt_create_err:
 157         kfree(tt);
 158         mutex_unlock(&esw->offloads.termtbl_mutex);
 159         return ERR_PTR(err);
 160 }
 161
 162 void
 163 mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw,
 164                          struct mlx5_termtbl_handle *tt)
 165 {
 166         mutex_lock(&esw->offloads.termtbl_mutex);
 167         if (--tt->ref_count == 0)
 168                 hash_del(&tt->termtbl_hlist);
 169         mutex_unlock(&esw->offloads.termtbl_mutex);
 170
 171         if (!tt->ref_count) {
 172                 mlx5_del_flow_rules(tt->rule);
 173                 mlx5_destroy_flow_table(tt->termtbl);
 174                 kfree(tt);
 175         }
 176 }
 177
 178 static void
 179 mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src,
 180                                   struct mlx5_flow_act *dst)
 181 {
 182         if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
 183                 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
 184                 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
 185                 memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0]));
 186                 memset(&src->vlan[0], 0, sizeof(src->vlan[0]));
 187
 188                 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
 189                         src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
 190                         dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
 191                         memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1]));
 192                         memset(&src->vlan[1], 0, sizeof(src->vlan[1]));
 193                 }
 194         }
 195 }
 196
 197 static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw,
 198                                                 const struct mlx5_flow_spec *spec)
 199 {
 200         u16 port_mask, port_value;
 201
 202         if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
 203                 return spec->flow_context.flow_source ==
 204                                         MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
 205
 206         port_mask = MLX5_GET(fte_match_param, spec->match_criteria,
 207                              misc_parameters.source_port);
 208         port_value = MLX5_GET(fte_match_param, spec->match_value,
 209                               misc_parameters.source_port);
 210         return (port_mask & port_value) == MLX5_VPORT_UPLINK;
 211 }
 212
 213 bool
 214 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
 215                               struct mlx5_flow_attr *attr,
 216                               struct mlx5_flow_act *flow_act,
 217                               struct mlx5_flow_spec *spec)
 218 {
 219         struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
 220         int i;
 221
 222         if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) ||
 223             !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) ||
 224             mlx5e_tc_attr_flags_skip(attr->flags) ||
 225             (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port))
 226                 return false;
 227
 228         /* push vlan on RX */
 229         if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
 230             !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) &
 231               MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX))
 232                 return true;
 233
 234         /* hairpin */
 235         for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
 236                 if (!esw_attr->dest_int_port && esw_attr->dests[i].vport_valid &&
 237                     esw_attr->dests[i].vport == MLX5_VPORT_UPLINK)
 238                         return true;
 239
 240         return false;
 241 }
 242
 243 struct mlx5_flow_handle *
 244 mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw,
 245                               struct mlx5_flow_table *fdb,
 246                               struct mlx5_flow_spec *spec,
 247                               struct mlx5_esw_flow_attr *attr,
 248                               struct mlx5_flow_act *flow_act,
 249                               struct mlx5_flow_destination *dest,
 250                               int num_dest)
 251 {
 252         struct mlx5_flow_act term_tbl_act = {};
 253         struct mlx5_flow_handle *rule = NULL;
 254         bool term_table_created = false;
 255         int num_vport_dests = 0;
 256         int i, curr_dest;
 257
 258         mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act);
 259         term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 260
 261         for (i = 0; i < num_dest; i++) {
 262                 struct mlx5_termtbl_handle *tt;
 263
 264                 /* only vport destinations can be terminated */
 265                 if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
 266                         continue;
 267
 268                 if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) {
 269                         term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
 270                         term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat;
 271                 } else {
 272                         term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
 273                         term_tbl_act.pkt_reformat = NULL;
 274                 }
 275
 276                 /* get the terminating table for the action list */
 277                 tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act,
 278                                                      &dest[i], attr);
 279                 if (IS_ERR(tt)) {
 280                         esw_warn(esw->dev, "Failed to get termination table, err %pe\n", tt);
 281                         goto revert_changes;
 282                 }
 283                 attr->dests[num_vport_dests].termtbl = tt;
 284                 num_vport_dests++;
 285
 286                 /* link the destination with the termination table */
 287                 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 288                 dest[i].ft = tt->termtbl;
 289                 term_table_created = true;
 290         }
 291
 292         /* at least one destination should reference a termination table */
 293         if (!term_table_created)
 294                 goto revert_changes;
 295
 296         /* create the FTE */
 297         flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
 298         flow_act->pkt_reformat = NULL;
 299         flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
 300         rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
 301         if (IS_ERR(rule))
 302                 goto revert_changes;
 303
 304         goto out;
 305
 306 revert_changes:
 307         /* revert the changes that were made to the original flow_act
 308          * and fall-back to the original rule actions
 309          */
 310         mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act);
 311
 312         for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) {
 313                 struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl;
 314
 315                 attr->dests[curr_dest].termtbl = NULL;
 316
 317                 /* search for the destination associated with the
 318                  * current term table
 319                  */
 320                 for (i = 0; i < num_dest; i++) {
 321                         if (dest[i].ft != tt->termtbl)
 322                                 continue;
 323
 324                         memset(&dest[i], 0, sizeof(dest[i]));
 325                         dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
 326                         dest[i].vport.num = tt->dest.vport.num;
 327                         dest[i].vport.vhca_id = tt->dest.vport.vhca_id;
 328                         mlx5_eswitch_termtbl_put(esw, tt);
 329                         break;
 330                 }
 331         }
 332         rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
 333 out:
 334         return rule;
 335 }