1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
7 /* It matches XDP_UMEM_MIN_CHUNK_SIZE, but as this constant is private and may
8 * change unexpectedly, and mlx5e has a minimum valid stride size for striding
9 * RQ, keep this check in the driver.
11 #define MLX5E_MIN_XSK_CHUNK_SIZE 2048
13 bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
14 struct mlx5e_xsk_param *xsk,
15 struct mlx5_core_dev *mdev)
17 /* AF_XDP doesn't support frames larger than PAGE_SIZE. */
18 if (xsk->chunk_size > PAGE_SIZE ||
19 xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE)
22 /* Current MTU and XSK headroom don't allow packets to fit the frames. */
23 if (mlx5e_rx_get_min_frag_sz(params, xsk) > xsk->chunk_size)
26 /* frag_sz is different for regular and XSK RQs, so ensure that linear
27 * SKB mode is possible.
29 switch (params->rq_wq_type) {
30 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
31 return mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);
32 default: /* MLX5_WQ_TYPE_CYCLIC */
33 return mlx5e_rx_is_linear_skb(params, xsk);
37 static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv,
38 struct mlx5e_params *params,
39 struct mlx5e_xsk_param *xsk,
40 struct mlx5e_channel_param *cparam)
42 mlx5e_build_rq_param(priv, params, xsk, &cparam->rq);
43 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
44 mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq);
45 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
48 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
49 struct mlx5e_xsk_param *xsk, struct xdp_umem *umem,
50 struct mlx5e_channel *c)
52 struct mlx5e_channel_param *cparam;
55 if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
58 cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL);
62 mlx5e_build_xsk_cparam(priv, params, xsk, cparam);
64 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->xskrq.cq);
68 err = mlx5e_open_rq(c, params, &cparam->rq, xsk, umem, &c->xskrq);
72 err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xsksq.cq);
76 /* Create a separate SQ, so that when the UMEM is disabled, we could
77 * close this SQ safely and stop receiving CQEs. In other case, e.g., if
78 * the XDPSQ was used instead, we might run into trouble when the UMEM
79 * is disabled and then reenabled, but the SQ continues receiving CQEs
82 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, umem, &c->xsksq, true);
88 set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
93 mlx5e_close_cq(&c->xsksq.cq);
96 mlx5e_close_rq(&c->xskrq);
99 mlx5e_close_cq(&c->xskrq.cq);
107 void mlx5e_close_xsk(struct mlx5e_channel *c)
109 clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
110 napi_synchronize(&c->napi);
111 synchronize_rcu(); /* Sync with the XSK wakeup. */
113 mlx5e_close_rq(&c->xskrq);
114 mlx5e_close_cq(&c->xskrq.cq);
115 mlx5e_close_xdpsq(&c->xsksq);
116 mlx5e_close_cq(&c->xsksq.cq);
118 memset(&c->xskrq, 0, sizeof(c->xskrq));
119 memset(&c->xsksq, 0, sizeof(c->xsksq));
122 void mlx5e_activate_xsk(struct mlx5e_channel *c)
124 set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
125 /* TX queue is created active. */
127 spin_lock(&c->async_icosq_lock);
128 mlx5e_trigger_irq(&c->async_icosq);
129 spin_unlock(&c->async_icosq_lock);
132 void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
134 mlx5e_deactivate_rq(&c->xskrq);
135 /* TX queue is disabled on close. */
138 static int mlx5e_redirect_xsk_rqt(struct mlx5e_priv *priv, u16 ix, u32 rqn)
140 struct mlx5e_redirect_rqt_param direct_rrp = {
147 u32 rqtn = priv->xsk_tir[ix].rqt.rqtn;
149 return mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
152 int mlx5e_xsk_redirect_rqt_to_channel(struct mlx5e_priv *priv, struct mlx5e_channel *c)
154 return mlx5e_redirect_xsk_rqt(priv, c->ix, c->xskrq.rqn);
157 int mlx5e_xsk_redirect_rqt_to_drop(struct mlx5e_priv *priv, u16 ix)
159 return mlx5e_redirect_xsk_rqt(priv, ix, priv->drop_rq.rqn);
162 int mlx5e_xsk_redirect_rqts_to_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
166 if (!priv->xsk.refcnt)
169 for (i = 0; i < chs->num; i++) {
170 struct mlx5e_channel *c = chs->c[i];
172 if (!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
175 err = mlx5e_xsk_redirect_rqt_to_channel(priv, c);
183 for (i--; i >= 0; i--) {
184 if (!test_bit(MLX5E_CHANNEL_STATE_XSK, chs->c[i]->state))
187 mlx5e_xsk_redirect_rqt_to_drop(priv, i);
193 void mlx5e_xsk_redirect_rqts_to_drop(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
197 if (!priv->xsk.refcnt)
200 for (i = 0; i < chs->num; i++) {
201 if (!test_bit(MLX5E_CHANNEL_STATE_XSK, chs->c[i]->state))
204 mlx5e_xsk_redirect_rqt_to_drop(priv, i);