--- /dev/null
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2020 Mellanox Technologies
+
+#include <linux/jhash.h>
+#include "mod_hdr.h"
+
+#define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)
+
+struct mod_hdr_key {
+ int num_actions;
+ void *actions;
+};
+
+struct mlx5e_mod_hdr_handle {
+ /* a node of a hash table which keeps all the mod_hdr entries */
+ struct hlist_node mod_hdr_hlist;
+
+ struct mod_hdr_key key;
+
+ struct mlx5_modify_hdr *modify_hdr;
+
+ refcount_t refcnt;
+ struct completion res_ready;
+ int compl_result;
+};
+
+static u32 hash_mod_hdr_info(struct mod_hdr_key *key)
+{
+ return jhash(key->actions,
+ key->num_actions * MLX5_MH_ACT_SZ, 0);
+}
+
+static int cmp_mod_hdr_info(struct mod_hdr_key *a, struct mod_hdr_key *b)
+{
+ if (a->num_actions != b->num_actions)
+ return 1;
+
+ return memcmp(a->actions, b->actions,
+ a->num_actions * MLX5_MH_ACT_SZ);
+}
+
+void mlx5e_mod_hdr_tbl_init(struct mod_hdr_tbl *tbl)
+{
+ mutex_init(&tbl->lock);
+ hash_init(tbl->hlist);
+}
+
+void mlx5e_mod_hdr_tbl_destroy(struct mod_hdr_tbl *tbl)
+{
+ mutex_destroy(&tbl->lock);
+}
+
+static struct mlx5e_mod_hdr_handle *mod_hdr_get(struct mod_hdr_tbl *tbl,
+ struct mod_hdr_key *key,
+ u32 hash_key)
+{
+ struct mlx5e_mod_hdr_handle *mh, *found = NULL;
+
+ hash_for_each_possible(tbl->hlist, mh, mod_hdr_hlist, hash_key) {
+ if (!cmp_mod_hdr_info(&mh->key, key)) {
+ refcount_inc(&mh->refcnt);
+ found = mh;
+ break;
+ }
+ }
+
+ return found;
+}
+
+struct mlx5e_mod_hdr_handle *
+mlx5e_mod_hdr_attach(struct mlx5_core_dev *mdev,
+ struct mod_hdr_tbl *tbl,
+ enum mlx5_flow_namespace_type namespace,
+ struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
+{
+ int num_actions, actions_size, err;
+ struct mlx5e_mod_hdr_handle *mh;
+ struct mod_hdr_key key;
+ u32 hash_key;
+
+ num_actions = mod_hdr_acts->num_actions;
+ actions_size = MLX5_MH_ACT_SZ * num_actions;
+
+ key.actions = mod_hdr_acts->actions;
+ key.num_actions = num_actions;
+
+ hash_key = hash_mod_hdr_info(&key);
+
+ mutex_lock(&tbl->lock);
+ mh = mod_hdr_get(tbl, &key, hash_key);
+ if (mh) {
+ mutex_unlock(&tbl->lock);
+ wait_for_completion(&mh->res_ready);
+
+ if (mh->compl_result < 0) {
+ err = -EREMOTEIO;
+ goto attach_header_err;
+ }
+ goto attach_header;
+ }
+
+ mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL);
+ if (!mh) {
+ mutex_unlock(&tbl->lock);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ mh->key.actions = (void *)mh + sizeof(*mh);
+ memcpy(mh->key.actions, key.actions, actions_size);
+ mh->key.num_actions = num_actions;
+ refcount_set(&mh->refcnt, 1);
+ init_completion(&mh->res_ready);
+
+ hash_add(tbl->hlist, &mh->mod_hdr_hlist, hash_key);
+ mutex_unlock(&tbl->lock);
+
+ mh->modify_hdr = mlx5_modify_header_alloc(mdev, namespace,
+ mh->key.num_actions,
+ mh->key.actions);
+ if (IS_ERR(mh->modify_hdr)) {
+ err = PTR_ERR(mh->modify_hdr);
+ mh->compl_result = err;
+ goto alloc_header_err;
+ }
+ mh->compl_result = 1;
+ complete_all(&mh->res_ready);
+
+attach_header:
+ return mh;
+
+alloc_header_err:
+ complete_all(&mh->res_ready);
+attach_header_err:
+ mlx5e_mod_hdr_detach(mdev, tbl, mh);
+ return ERR_PTR(err);
+}
+
+void mlx5e_mod_hdr_detach(struct mlx5_core_dev *mdev,
+ struct mod_hdr_tbl *tbl,
+ struct mlx5e_mod_hdr_handle *mh)
+{
+ if (!refcount_dec_and_mutex_lock(&mh->refcnt, &tbl->lock))
+ return;
+ hash_del(&mh->mod_hdr_hlist);
+ mutex_unlock(&tbl->lock);
+
+ if (mh->compl_result > 0)
+ mlx5_modify_header_dealloc(mdev, mh->modify_hdr);
+
+ kfree(mh);
+}
+
+struct mlx5_modify_hdr *mlx5e_mod_hdr_get(struct mlx5e_mod_hdr_handle *mh)
+{
+ return mh->modify_hdr;
+}
+
#include "en/tc_tun.h"
#include "en/mapping.h"
#include "en/tc_ct.h"
+#include "en/mod_hdr.h"
#include "lib/devcom.h"
#include "lib/geneve.h"
#include "diag/en_tc_tracepoint.h"
*/
struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS];
struct mlx5e_tc_flow *peer_flow;
- struct mlx5e_mod_hdr_entry *mh; /* attached mod header instance */
- struct list_head mod_hdr; /* flows sharing the same mod hdr ID */
+ struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */
struct mlx5e_hairpin_entry *hpe; /* attached hairpin instance */
struct list_head hairpin; /* flows sharing the same hairpin */
struct list_head peer; /* flows with peer flow */
struct completion res_ready;
};
-struct mod_hdr_key {
- int num_actions;
- void *actions;
-};
-
-struct mlx5e_mod_hdr_entry {
- /* a node of a hash table which keeps all the mod_hdr entries */
- struct hlist_node mod_hdr_hlist;
-
- /* protects flows list */
- spinlock_t flows_lock;
- /* flows sharing the same mod_hdr entry */
- struct list_head flows;
-
- struct mod_hdr_key key;
-
- struct mlx5_modify_hdr *modify_hdr;
-
- refcount_t refcnt;
- struct completion res_ready;
- int compl_result;
-};
-
static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
struct mlx5e_tc_flow *flow);
return flow_flag_test(flow, OFFLOADED);
}
-static inline u32 hash_mod_hdr_info(struct mod_hdr_key *key)
-{
- return jhash(key->actions,
- key->num_actions * MLX5_MH_ACT_SZ, 0);
-}
-
-static inline int cmp_mod_hdr_info(struct mod_hdr_key *a,
- struct mod_hdr_key *b)
+static int get_flow_name_space(struct mlx5e_tc_flow *flow)
{
- if (a->num_actions != b->num_actions)
- return 1;
-
- return memcmp(a->actions, b->actions, a->num_actions * MLX5_MH_ACT_SZ);
+ return mlx5e_is_eswitch_flow(flow) ?
+ MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL;
}
static struct mod_hdr_tbl *
-get_mod_hdr_table(struct mlx5e_priv *priv, int namespace)
+get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow)
{
struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
- return namespace == MLX5_FLOW_NAMESPACE_FDB ? &esw->offloads.mod_hdr :
+ return get_flow_name_space(flow) == MLX5_FLOW_NAMESPACE_FDB ?
+ &esw->offloads.mod_hdr :
&priv->fs.tc.mod_hdr;
}
-static struct mlx5e_mod_hdr_entry *
-mlx5e_mod_hdr_get(struct mod_hdr_tbl *tbl, struct mod_hdr_key *key, u32 hash_key)
-{
- struct mlx5e_mod_hdr_entry *mh, *found = NULL;
-
- hash_for_each_possible(tbl->hlist, mh, mod_hdr_hlist, hash_key) {
- if (!cmp_mod_hdr_info(&mh->key, key)) {
- refcount_inc(&mh->refcnt);
- found = mh;
- break;
- }
- }
-
- return found;
-}
-
-static void mlx5e_mod_hdr_put(struct mlx5e_priv *priv,
- struct mlx5e_mod_hdr_entry *mh,
- int namespace)
-{
- struct mod_hdr_tbl *tbl = get_mod_hdr_table(priv, namespace);
-
- if (!refcount_dec_and_mutex_lock(&mh->refcnt, &tbl->lock))
- return;
- hash_del(&mh->mod_hdr_hlist);
- mutex_unlock(&tbl->lock);
-
- WARN_ON(!list_empty(&mh->flows));
- if (mh->compl_result > 0)
- mlx5_modify_header_dealloc(priv->mdev, mh->modify_hdr);
-
- kfree(mh);
-}
-
-static int get_flow_name_space(struct mlx5e_tc_flow *flow)
-{
- return mlx5e_is_eswitch_flow(flow) ?
- MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL;
-}
static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv,
struct mlx5e_tc_flow *flow,
struct mlx5e_tc_flow_parse_attr *parse_attr)
{
- int num_actions, actions_size, namespace, err;
- struct mlx5e_mod_hdr_entry *mh;
- struct mod_hdr_tbl *tbl;
- struct mod_hdr_key key;
- u32 hash_key;
-
- num_actions = parse_attr->mod_hdr_acts.num_actions;
- actions_size = MLX5_MH_ACT_SZ * num_actions;
-
- key.actions = parse_attr->mod_hdr_acts.actions;
- key.num_actions = num_actions;
-
- hash_key = hash_mod_hdr_info(&key);
-
- namespace = get_flow_name_space(flow);
- tbl = get_mod_hdr_table(priv, namespace);
-
- mutex_lock(&tbl->lock);
- mh = mlx5e_mod_hdr_get(tbl, &key, hash_key);
- if (mh) {
- mutex_unlock(&tbl->lock);
- wait_for_completion(&mh->res_ready);
-
- if (mh->compl_result < 0) {
- err = -EREMOTEIO;
- goto attach_header_err;
- }
- goto attach_flow;
- }
-
- mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL);
- if (!mh) {
- mutex_unlock(&tbl->lock);
- return -ENOMEM;
- }
-
- mh->key.actions = (void *)mh + sizeof(*mh);
- memcpy(mh->key.actions, key.actions, actions_size);
- mh->key.num_actions = num_actions;
- spin_lock_init(&mh->flows_lock);
- INIT_LIST_HEAD(&mh->flows);
- refcount_set(&mh->refcnt, 1);
- init_completion(&mh->res_ready);
-
- hash_add(tbl->hlist, &mh->mod_hdr_hlist, hash_key);
- mutex_unlock(&tbl->lock);
+ struct mlx5_modify_hdr *modify_hdr;
+ struct mlx5e_mod_hdr_handle *mh;
- mh->modify_hdr = mlx5_modify_header_alloc(priv->mdev, namespace,
- mh->key.num_actions,
- mh->key.actions);
- if (IS_ERR(mh->modify_hdr)) {
- err = PTR_ERR(mh->modify_hdr);
- mh->compl_result = err;
- goto alloc_header_err;
- }
- mh->compl_result = 1;
- complete_all(&mh->res_ready);
+ mh = mlx5e_mod_hdr_attach(priv->mdev, get_mod_hdr_table(priv, flow),
+ get_flow_name_space(flow),
+ &parse_attr->mod_hdr_acts);
+ if (IS_ERR(mh))
+ return PTR_ERR(mh);
-attach_flow:
- flow->mh = mh;
- spin_lock(&mh->flows_lock);
- list_add(&flow->mod_hdr, &mh->flows);
- spin_unlock(&mh->flows_lock);
+ modify_hdr = mlx5e_mod_hdr_get(mh);
if (mlx5e_is_eswitch_flow(flow))
- flow->esw_attr->modify_hdr = mh->modify_hdr;
+ flow->esw_attr->modify_hdr = modify_hdr;
else
- flow->nic_attr->modify_hdr = mh->modify_hdr;
+ flow->nic_attr->modify_hdr = modify_hdr;
+ flow->mh = mh;
return 0;
-
-alloc_header_err:
- complete_all(&mh->res_ready);
-attach_header_err:
- mlx5e_mod_hdr_put(priv, mh, namespace);
- return err;
}
static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv,
if (!flow->mh)
return;
- spin_lock(&flow->mh->flows_lock);
- list_del(&flow->mod_hdr);
- spin_unlock(&flow->mh->flows_lock);
-
- mlx5e_mod_hdr_put(priv, flow->mh, get_flow_name_space(flow));
+ mlx5e_mod_hdr_detach(priv->mdev, get_mod_hdr_table(priv, flow),
+ flow->mh);
flow->mh = NULL;
}
flow->priv = priv;
for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++)
INIT_LIST_HEAD(&flow->encaps[out_index].list);
- INIT_LIST_HEAD(&flow->mod_hdr);
INIT_LIST_HEAD(&flow->hairpin);
INIT_LIST_HEAD(&flow->l3_to_l2_reformat);
refcount_set(&flow->refcnt, 1);
struct mlx5e_tc_table *tc = &priv->fs.tc;
int err;
+ mlx5e_mod_hdr_tbl_init(&tc->mod_hdr);
mutex_init(&tc->t_lock);
- mutex_init(&tc->mod_hdr.lock);
- hash_init(tc->mod_hdr.hlist);
mutex_init(&tc->hairpin_tbl_lock);
hash_init(tc->hairpin_tbl);
&tc->netdevice_nb,
&tc->netdevice_nn);
- mutex_destroy(&tc->mod_hdr.lock);
+ mlx5e_mod_hdr_tbl_destroy(&tc->mod_hdr);
mutex_destroy(&tc->hairpin_tbl_lock);
rhashtable_destroy(&tc->ht);