net/mlx5e: Store all XFRM SAs in Xarray

Instead of performing custom hash calculations, rely on FW that returns
unique identifier to every created SA. That identifier is Xarray ready,
which provides better semantic with efficient access.

In addition, store both TX and RX SAs to allow correlation between event
generated by HW when limits are armed and XFRM states.

Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index 8f08dbf..fe10f1a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -50,57 +50,6 @@ static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x)
        return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle;
 }
 
-struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,
-                                             unsigned int handle)
-{
-       struct mlx5e_ipsec_sa_entry *sa_entry;
-       struct xfrm_state *ret = NULL;
-
-       rcu_read_lock();
-       hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
-               if (sa_entry->handle == handle) {
-                       ret = sa_entry->x;
-                       xfrm_state_hold(ret);
-                       break;
-               }
-       rcu_read_unlock();
-
-       return ret;
-}
-
-static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)
-{
-       unsigned int handle = sa_entry->ipsec_obj_id;
-       struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
-       struct mlx5e_ipsec_sa_entry *_sa_entry;
-       unsigned long flags;
-
-       rcu_read_lock();
-       hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle)
-               if (_sa_entry->handle == handle) {
-                       rcu_read_unlock();
-                       return  -EEXIST;
-               }
-       rcu_read_unlock();
-
-       spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
-       sa_entry->handle = handle;
-       hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle);
-       spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
-
-       return 0;
-}
-
-static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry)
-{
-       struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
-       unsigned long flags;
-
-       spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
-       hash_del_rcu(&sa_entry->hlist);
-       spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
-}
-
 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
 {
        struct xfrm_replay_state_esn *replay_esn;
@@ -291,6 +240,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
 {
        struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
        struct net_device *netdev = x->xso.real_dev;
+       struct mlx5e_ipsec *ipsec;
        struct mlx5e_priv *priv;
        int err;
 
@@ -298,6 +248,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
        if (!priv->ipsec)
                return -EOPNOTSUPP;
 
+       ipsec = priv->ipsec;
        err = mlx5e_xfrm_validate_state(x);
        if (err)
                return err;
@@ -309,7 +260,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
        }
 
        sa_entry->x = x;
-       sa_entry->ipsec = priv->ipsec;
+       sa_entry->ipsec = ipsec;
 
        /* check esn */
        mlx5e_ipsec_update_esn_state(sa_entry);
@@ -324,18 +275,22 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
        if (err)
                goto err_hw_ctx;
 
-       if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) {
-               err = mlx5e_ipsec_sadb_rx_add(sa_entry);
-               if (err)
-                       goto err_add_rule;
-       } else {
+       /* We use *_bh() variant because xfrm_timer_handler(), which runs
+        * in softirq context, can reach our state delete logic and we need
+        * xa_erase_bh() there.
+        */
+       err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry,
+                          GFP_KERNEL);
+       if (err)
+               goto err_add_rule;
+
+       if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT)
                sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
                                mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
-       }
 
        INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);
        x->xso.offload_handle = (unsigned long)sa_entry;
-       goto out;
+       return 0;
 
 err_add_rule:
        mlx5e_accel_ipsec_fs_del_rule(sa_entry);
@@ -350,9 +305,11 @@ out:
 static void mlx5e_xfrm_del_state(struct xfrm_state *x)
 {
        struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
+       struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+       struct mlx5e_ipsec_sa_entry *old;
 
-       if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
-               mlx5e_ipsec_sadb_rx_del(sa_entry);
+       old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id);
+       WARN_ON(old != sa_entry);
 }
 
 static void mlx5e_xfrm_free_state(struct xfrm_state *x)
@@ -379,8 +336,7 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv)
        if (!ipsec)
                return;
 
-       hash_init(ipsec->sadb_rx);
-       spin_lock_init(&ipsec->sadb_rx_lock);
+       xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC);
        ipsec->mdev = priv->mdev;
        ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
                                            priv->netdev->name);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
index 492be25..724f2df 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
@@ -120,8 +120,7 @@ struct mlx5e_ipsec_aso {
 
 struct mlx5e_ipsec {
        struct mlx5_core_dev *mdev;
-       DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS);
-       spinlock_t sadb_rx_lock; /* Protects sadb_rx */
+       struct xarray sadb;
        struct mlx5e_ipsec_sw_stats sw_stats;
        struct mlx5e_ipsec_hw_stats hw_stats;
        struct workqueue_struct *wq;
@@ -150,9 +149,7 @@ struct mlx5e_ipsec_modify_state_work {
 };
 
 struct mlx5e_ipsec_sa_entry {
-       struct hlist_node hlist; /* Item in SADB_RX hashtable */
        struct mlx5e_ipsec_esn_state esn_state;
-       unsigned int handle; /* Handle in SADB_RX */
        struct xfrm_state *x;
        struct mlx5e_ipsec *ipsec;
        struct mlx5_accel_esp_xfrm_attrs attrs;
@@ -193,9 +190,6 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv);
 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv);
 void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv);
 
-struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev,
-                                             unsigned int handle);
-
 void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec);
 int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec);
 int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
index 9f07e58..eab5bc7 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
@@ -314,8 +314,8 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
        u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
        struct mlx5e_priv *priv = netdev_priv(netdev);
        struct mlx5e_ipsec *ipsec = priv->ipsec;
+       struct mlx5e_ipsec_sa_entry *sa_entry;
        struct xfrm_offload *xo;
-       struct xfrm_state *xs;
        struct sec_path *sp;
        u32  sa_handle;
 
@@ -326,13 +326,17 @@ void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
                return;
        }
 
-       xs = mlx5e_ipsec_sadb_rx_lookup(ipsec, sa_handle);
-       if (unlikely(!xs)) {
+       rcu_read_lock();
+       sa_entry = xa_load(&ipsec->sadb, sa_handle);
+       if (unlikely(!sa_entry)) {
+               rcu_read_unlock();
                atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
                return;
        }
+       xfrm_state_hold(sa_entry->x);
+       rcu_read_unlock();
 
-       sp->xvec[sp->len++] = xs;
+       sp->xvec[sp->len++] = sa_entry->x;
        sp->olen++;
 
        xo = xfrm_offload(skb);