#include "en/tc_tun.h"
#include "en/mapping.h"
#include "en/tc_ct.h"
+#include "en/mod_hdr.h"
#include "lib/devcom.h"
#include "lib/geneve.h"
#include "diag/en_tc_tracepoint.h"
*/
struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS];
struct mlx5e_tc_flow *peer_flow;
- struct mlx5e_mod_hdr_entry *mh; /* attached mod header instance */
- struct list_head mod_hdr; /* flows sharing the same mod hdr ID */
+ struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */
struct mlx5e_hairpin_entry *hpe; /* attached hairpin instance */
struct list_head hairpin; /* flows sharing the same hairpin */
struct list_head peer; /* flows with peer flow */
},
[TUNNEL_TO_REG] = {
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,
- .moffset = 3,
- .mlen = 1,
+ .moffset = 1,
+ .mlen = 3,
.soffset = MLX5_BYTE_OFF(fte_match_param,
misc_parameters_2.metadata_reg_c_1),
},
[ZONE_TO_REG] = zone_to_reg_ct,
+ [ZONE_RESTORE_TO_REG] = zone_restore_to_reg_ct,
[CTSTATE_TO_REG] = ctstate_to_reg_ct,
[MARK_TO_REG] = mark_to_reg_ct,
[LABELS_TO_REG] = labels_to_reg_ct,
[FTEID_TO_REG] = fteid_to_reg_ct,
- [TUPLEID_TO_REG] = tupleid_to_reg_ct,
};
static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow);
spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
}
+void
+mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec,
+ enum mlx5e_tc_attr_to_reg type,
+ u32 *data,
+ u32 *mask)
+{
+ int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset;
+ int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen;
+ void *headers_c = spec->match_criteria;
+ void *headers_v = spec->match_value;
+ void *fmask, *fval;
+
+ fmask = headers_c + soffset;
+ fval = headers_v + soffset;
+
+ memcpy(mask, fmask, match_len);
+ memcpy(data, fval, match_len);
+
+ *mask = be32_to_cpu((__force __be32)(*mask << (32 - (match_len * 8))));
+ *data = be32_to_cpu((__force __be32)(*data << (32 - (match_len * 8))));
+}
+
int
mlx5e_tc_match_to_reg_set(struct mlx5_core_dev *mdev,
struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts,
struct completion res_ready;
};
-struct mod_hdr_key {
- int num_actions;
- void *actions;
-};
-
-struct mlx5e_mod_hdr_entry {
- /* a node of a hash table which keeps all the mod_hdr entries */
- struct hlist_node mod_hdr_hlist;
-
- /* protects flows list */
- spinlock_t flows_lock;
- /* flows sharing the same mod_hdr entry */
- struct list_head flows;
-
- struct mod_hdr_key key;
-
- struct mlx5_modify_hdr *modify_hdr;
-
- refcount_t refcnt;
- struct completion res_ready;
- int compl_result;
-};
-
static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
struct mlx5e_tc_flow *flow);
return flow_flag_test(flow, OFFLOADED);
}
-static inline u32 hash_mod_hdr_info(struct mod_hdr_key *key)
-{
- return jhash(key->actions,
- key->num_actions * MLX5_MH_ACT_SZ, 0);
-}
-
-static inline int cmp_mod_hdr_info(struct mod_hdr_key *a,
- struct mod_hdr_key *b)
+static int get_flow_name_space(struct mlx5e_tc_flow *flow)
{
- if (a->num_actions != b->num_actions)
- return 1;
-
- return memcmp(a->actions, b->actions, a->num_actions * MLX5_MH_ACT_SZ);
+ return mlx5e_is_eswitch_flow(flow) ?
+ MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL;
}
static struct mod_hdr_tbl *
-get_mod_hdr_table(struct mlx5e_priv *priv, int namespace)
+get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow)
{
struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
- return namespace == MLX5_FLOW_NAMESPACE_FDB ? &esw->offloads.mod_hdr :
+ return get_flow_name_space(flow) == MLX5_FLOW_NAMESPACE_FDB ?
+ &esw->offloads.mod_hdr :
&priv->fs.tc.mod_hdr;
}
-static struct mlx5e_mod_hdr_entry *
-mlx5e_mod_hdr_get(struct mod_hdr_tbl *tbl, struct mod_hdr_key *key, u32 hash_key)
-{
- struct mlx5e_mod_hdr_entry *mh, *found = NULL;
-
- hash_for_each_possible(tbl->hlist, mh, mod_hdr_hlist, hash_key) {
- if (!cmp_mod_hdr_info(&mh->key, key)) {
- refcount_inc(&mh->refcnt);
- found = mh;
- break;
- }
- }
-
- return found;
-}
-
-static void mlx5e_mod_hdr_put(struct mlx5e_priv *priv,
- struct mlx5e_mod_hdr_entry *mh,
- int namespace)
-{
- struct mod_hdr_tbl *tbl = get_mod_hdr_table(priv, namespace);
-
- if (!refcount_dec_and_mutex_lock(&mh->refcnt, &tbl->lock))
- return;
- hash_del(&mh->mod_hdr_hlist);
- mutex_unlock(&tbl->lock);
-
- WARN_ON(!list_empty(&mh->flows));
- if (mh->compl_result > 0)
- mlx5_modify_header_dealloc(priv->mdev, mh->modify_hdr);
-
- kfree(mh);
-}
-
-static int get_flow_name_space(struct mlx5e_tc_flow *flow)
-{
- return mlx5e_is_eswitch_flow(flow) ?
- MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL;
-}
static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv,
struct mlx5e_tc_flow *flow,
struct mlx5e_tc_flow_parse_attr *parse_attr)
{
- int num_actions, actions_size, namespace, err;
- struct mlx5e_mod_hdr_entry *mh;
- struct mod_hdr_tbl *tbl;
- struct mod_hdr_key key;
- u32 hash_key;
-
- num_actions = parse_attr->mod_hdr_acts.num_actions;
- actions_size = MLX5_MH_ACT_SZ * num_actions;
-
- key.actions = parse_attr->mod_hdr_acts.actions;
- key.num_actions = num_actions;
-
- hash_key = hash_mod_hdr_info(&key);
-
- namespace = get_flow_name_space(flow);
- tbl = get_mod_hdr_table(priv, namespace);
-
- mutex_lock(&tbl->lock);
- mh = mlx5e_mod_hdr_get(tbl, &key, hash_key);
- if (mh) {
- mutex_unlock(&tbl->lock);
- wait_for_completion(&mh->res_ready);
-
- if (mh->compl_result < 0) {
- err = -EREMOTEIO;
- goto attach_header_err;
- }
- goto attach_flow;
- }
-
- mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL);
- if (!mh) {
- mutex_unlock(&tbl->lock);
- return -ENOMEM;
- }
-
- mh->key.actions = (void *)mh + sizeof(*mh);
- memcpy(mh->key.actions, key.actions, actions_size);
- mh->key.num_actions = num_actions;
- spin_lock_init(&mh->flows_lock);
- INIT_LIST_HEAD(&mh->flows);
- refcount_set(&mh->refcnt, 1);
- init_completion(&mh->res_ready);
-
- hash_add(tbl->hlist, &mh->mod_hdr_hlist, hash_key);
- mutex_unlock(&tbl->lock);
+ struct mlx5_modify_hdr *modify_hdr;
+ struct mlx5e_mod_hdr_handle *mh;
- mh->modify_hdr = mlx5_modify_header_alloc(priv->mdev, namespace,
- mh->key.num_actions,
- mh->key.actions);
- if (IS_ERR(mh->modify_hdr)) {
- err = PTR_ERR(mh->modify_hdr);
- mh->compl_result = err;
- goto alloc_header_err;
- }
- mh->compl_result = 1;
- complete_all(&mh->res_ready);
+ mh = mlx5e_mod_hdr_attach(priv->mdev, get_mod_hdr_table(priv, flow),
+ get_flow_name_space(flow),
+ &parse_attr->mod_hdr_acts);
+ if (IS_ERR(mh))
+ return PTR_ERR(mh);
-attach_flow:
- flow->mh = mh;
- spin_lock(&mh->flows_lock);
- list_add(&flow->mod_hdr, &mh->flows);
- spin_unlock(&mh->flows_lock);
+ modify_hdr = mlx5e_mod_hdr_get(mh);
if (mlx5e_is_eswitch_flow(flow))
- flow->esw_attr->modify_hdr = mh->modify_hdr;
+ flow->esw_attr->modify_hdr = modify_hdr;
else
- flow->nic_attr->modify_hdr = mh->modify_hdr;
+ flow->nic_attr->modify_hdr = modify_hdr;
+ flow->mh = mh;
return 0;
-
-alloc_header_err:
- complete_all(&mh->res_ready);
-attach_header_err:
- mlx5e_mod_hdr_put(priv, mh, namespace);
- return err;
}
static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv,
if (!flow->mh)
return;
- spin_lock(&flow->mh->flows_lock);
- list_del(&flow->mod_hdr);
- spin_unlock(&flow->mh->flows_lock);
-
- mlx5e_mod_hdr_put(priv, flow->mh, get_flow_name_space(flow));
+ mlx5e_mod_hdr_detach(priv->mdev, get_mod_hdr_table(priv, flow),
+ flow->mh);
flow->mh = NULL;
}
static int is_action_keys_supported(const struct flow_action_entry *act,
bool ct_flow, bool *modify_ip_header,
+ bool *modify_tuple,
struct netlink_ext_ack *extack)
{
u32 mask, offset;
*modify_ip_header = true;
}
- if (ct_flow && offset >= offsetof(struct iphdr, saddr)) {
+ if (offset >= offsetof(struct iphdr, saddr))
+ *modify_tuple = true;
+
+ if (ct_flow && *modify_tuple) {
NL_SET_ERR_MSG_MOD(extack,
"can't offload re-write of ipv4 address with action ct");
return -EOPNOTSUPP;
*modify_ip_header = true;
}
- if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr)) {
+ if (ct_flow && offset >= offsetof(struct ipv6hdr, saddr))
+ *modify_tuple = true;
+
+ if (ct_flow && *modify_tuple) {
NL_SET_ERR_MSG_MOD(extack,
"can't offload re-write of ipv6 address with action ct");
return -EOPNOTSUPP;
}
- } else if (ct_flow && (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP ||
- htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP)) {
- NL_SET_ERR_MSG_MOD(extack,
- "can't offload re-write of transport header ports with action ct");
- return -EOPNOTSUPP;
+ } else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_TCP ||
+ htype == FLOW_ACT_MANGLE_HDR_TYPE_UDP) {
+ *modify_tuple = true;
+ if (ct_flow) {
+ NL_SET_ERR_MSG_MOD(extack,
+ "can't offload re-write of transport header ports with action ct");
+ return -EOPNOTSUPP;
+ }
}
return 0;
}
-static bool modify_header_match_supported(struct mlx5_flow_spec *spec,
+static bool modify_header_match_supported(struct mlx5e_priv *priv,
+ struct mlx5_flow_spec *spec,
struct flow_action *flow_action,
u32 actions, bool ct_flow,
+ bool ct_clear,
struct netlink_ext_ack *extack)
{
const struct flow_action_entry *act;
- bool modify_ip_header;
+ bool modify_ip_header, modify_tuple;
void *headers_c;
void *headers_v;
u16 ethertype;
goto out_ok;
modify_ip_header = false;
+ modify_tuple = false;
flow_action_for_each(i, act, flow_action) {
if (act->id != FLOW_ACTION_MANGLE &&
act->id != FLOW_ACTION_ADD)
continue;
err = is_action_keys_supported(act, ct_flow,
- &modify_ip_header, extack);
+ &modify_ip_header,
+ &modify_tuple, extack);
if (err)
return err;
}
+ /* Add ct_state=-trk match so it will be offloaded for non ct flows
+ * (or after clear action), as otherwise, since the tuple is changed,
+ * we can't restore ct state
+ */
+ if (!ct_clear && modify_tuple &&
+ mlx5_tc_ct_add_no_trk_match(priv, spec)) {
+ NL_SET_ERR_MSG_MOD(extack,
+ "can't offload tuple modify header with ct matches");
+ netdev_info(priv->netdev,
+ "can't offload tuple modify header with ct matches");
+ return false;
+ }
+
ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol);
if (modify_ip_header && ip_proto != IPPROTO_TCP &&
ip_proto != IPPROTO_UDP && ip_proto != IPPROTO_ICMP) {
NL_SET_ERR_MSG_MOD(extack,
"can't offload re-write of non TCP/UDP");
- pr_info("can't offload re-write of ip proto %d\n", ip_proto);
+ netdev_info(priv->netdev, "can't offload re-write of ip proto %d\n",
+ ip_proto);
return false;
}
struct mlx5e_tc_flow *flow,
struct netlink_ext_ack *extack)
{
- bool ct_flow;
+ bool ct_flow = false, ct_clear = false;
u32 actions;
- ct_flow = flow_flag_test(flow, CT);
if (mlx5e_is_eswitch_flow(flow)) {
actions = flow->esw_attr->action;
-
+ ct_clear = flow->esw_attr->ct_attr.ct_action &
+ TCA_CT_ACT_CLEAR;
+ ct_flow = flow_flag_test(flow, CT) && !ct_clear;
if (flow->esw_attr->split_count && ct_flow) {
/* All registers used by ct are cleared when using
* split rules.
}
if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
- return modify_header_match_supported(&parse_attr->spec,
+ return modify_header_match_supported(priv, &parse_attr->spec,
flow_action, actions,
- ct_flow, extack);
+ ct_flow, ct_clear,
+ extack);
return true;
}
flow->priv = priv;
for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++)
INIT_LIST_HEAD(&flow->encaps[out_index].list);
- INIT_LIST_HEAD(&flow->mod_hdr);
INIT_LIST_HEAD(&flow->hairpin);
INIT_LIST_HEAD(&flow->l3_to_l2_reformat);
refcount_set(&flow->refcnt, 1);
if (err)
goto err_free;
- err = parse_tc_fdb_actions(priv, &rule->action, flow, extack, filter_dev);
+ /* actions validation depends on parsing the ct matches first */
+ err = mlx5_tc_ct_parse_match(priv, &parse_attr->spec, f,
+ &flow->esw_attr->ct_attr, extack);
if (err)
goto err_free;
- err = mlx5_tc_ct_parse_match(priv, &parse_attr->spec, f, extack);
+ err = parse_tc_fdb_actions(priv, &rule->action, flow, extack, filter_dev);
if (err)
goto err_free;
no_peer_counter:
mlx5_devcom_release_peer_data(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
out:
- flow_stats_update(&f->stats, bytes, packets, lastuse,
+ flow_stats_update(&f->stats, bytes, packets, 0, lastuse,
FLOW_ACTION_HW_STATS_DELAYED);
trace_mlx5e_stats_flower(f);
errout:
dpkts = cur_stats.rx_packets - rpriv->prev_vf_vport_stats.rx_packets;
dbytes = cur_stats.rx_bytes - rpriv->prev_vf_vport_stats.rx_bytes;
rpriv->prev_vf_vport_stats = cur_stats;
- flow_stats_update(&ma->stats, dbytes, dpkts, jiffies,
+ flow_stats_update(&ma->stats, dbytes, dpkts, 0, jiffies,
FLOW_ACTION_HW_STATS_DELAYED);
}
struct mlx5e_tc_table *tc = &priv->fs.tc;
int err;
+ mlx5e_mod_hdr_tbl_init(&tc->mod_hdr);
mutex_init(&tc->t_lock);
- mutex_init(&tc->mod_hdr.lock);
- hash_init(tc->mod_hdr.hlist);
mutex_init(&tc->hairpin_tbl_lock);
hash_init(tc->hairpin_tbl);
&tc->netdevice_nb,
&tc->netdevice_nn);
- mutex_destroy(&tc->mod_hdr.lock);
+ mlx5e_mod_hdr_tbl_destroy(&tc->mod_hdr);
mutex_destroy(&tc->hairpin_tbl_lock);
rhashtable_destroy(&tc->ht);
projects / linux-2.6-microblaze.git / blobdiff