1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */
4 #include <linux/vdpa.h>
5 #include <uapi/linux/virtio_ids.h>
6 #include <linux/virtio_config.h>
7 #include <linux/mlx5/qp.h>
8 #include <linux/mlx5/device.h>
9 #include <linux/mlx5/vport.h>
10 #include <linux/mlx5/fs.h>
11 #include <linux/mlx5/device.h>
12 #include "mlx5_vnet.h"
13 #include "mlx5_vdpa_ifc.h"
14 #include "mlx5_vdpa.h"
16 #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev)
18 #define VALID_FEATURES_MASK \
19 (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \
20 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \
21 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \
22 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \
23 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \
24 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \
25 BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \
26 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \
27 BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \
28 BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \
29 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \
30 BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \
31 BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV))
33 #define VALID_STATUS_MASK \
34 (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \
35 VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED)
37 struct mlx5_vdpa_net_resources {
45 struct mlx5_vdpa_cq_buf {
46 struct mlx5_frag_buf_ctrl fbc;
47 struct mlx5_frag_buf frag_buf;
53 struct mlx5_core_cq mcq;
54 struct mlx5_vdpa_cq_buf buf;
59 struct mlx5_vdpa_umem {
60 struct mlx5_frag_buf_ctrl fbc;
61 struct mlx5_frag_buf frag_buf;
67 struct mlx5_core_qp mqp;
68 struct mlx5_frag_buf frag_buf;
74 struct mlx5_vq_restore_info {
81 struct vdpa_callback cb;
85 struct mlx5_vdpa_virtqueue {
91 struct vdpa_callback event_cb;
93 /* Resources for implementing the notification channel from the device
94 * to the driver. fwqp is the firmware end of an RC connection; the
95 * other end is vqqp used by the driver. cq is is where completions are
98 struct mlx5_vdpa_cq cq;
99 struct mlx5_vdpa_qp fwqp;
100 struct mlx5_vdpa_qp vqqp;
102 /* umem resources are required for the virtqueue operation. They're use
103 * is internal and they must be provided by the driver.
105 struct mlx5_vdpa_umem umem1;
106 struct mlx5_vdpa_umem umem2;
107 struct mlx5_vdpa_umem umem3;
112 struct mlx5_vdpa_net *ndev;
116 /* keep last in the struct */
117 struct mlx5_vq_restore_info ri;
120 /* We will remove this limitation once mlx5_vdpa_alloc_resources()
121 * provides for driver space allocation
123 #define MLX5_MAX_SUPPORTED_VQS 16
125 struct mlx5_vdpa_net {
126 struct mlx5_vdpa_dev mvdev;
127 struct mlx5_vdpa_net_resources res;
128 struct virtio_net_config config;
129 struct mlx5_vdpa_virtqueue vqs[MLX5_MAX_SUPPORTED_VQS];
131 /* Serialize vq resources creation and destruction. This is required
132 * since memory map might change and we need to destroy and create
133 * resources while driver in operational.
135 struct mutex reslock;
136 struct mlx5_flow_table *rxft;
137 struct mlx5_fc *rx_counter;
138 struct mlx5_flow_handle *rx_rule;
143 static void free_resources(struct mlx5_vdpa_net *ndev);
144 static void init_mvqs(struct mlx5_vdpa_net *ndev);
145 static int setup_driver(struct mlx5_vdpa_net *ndev);
146 static void teardown_driver(struct mlx5_vdpa_net *ndev);
148 static bool mlx5_vdpa_debug;
150 #define MLX5_LOG_VIO_FLAG(_feature) \
152 if (features & BIT_ULL(_feature)) \
153 mlx5_vdpa_info(mvdev, "%s\n", #_feature); \
156 #define MLX5_LOG_VIO_STAT(_status) \
158 if (status & (_status)) \
159 mlx5_vdpa_info(mvdev, "%s\n", #_status); \
162 static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set)
164 if (status & ~VALID_STATUS_MASK)
165 mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n",
166 status & ~VALID_STATUS_MASK);
168 if (!mlx5_vdpa_debug)
171 mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get");
172 if (set && !status) {
173 mlx5_vdpa_info(mvdev, "driver resets the device\n");
177 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE);
178 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER);
179 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK);
180 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK);
181 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET);
182 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED);
185 static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set)
187 if (features & ~VALID_FEATURES_MASK)
188 mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n",
189 features & ~VALID_FEATURES_MASK);
191 if (!mlx5_vdpa_debug)
194 mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads");
196 mlx5_vdpa_info(mvdev, "all feature bits are cleared\n");
198 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM);
199 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM);
200 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
201 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU);
202 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC);
203 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4);
204 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6);
205 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN);
206 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO);
207 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4);
208 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6);
209 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN);
210 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO);
211 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF);
212 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS);
213 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ);
214 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX);
215 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN);
216 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA);
217 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE);
218 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ);
219 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR);
220 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT);
221 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS);
222 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT);
223 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY);
224 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX);
225 MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY);
226 MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT);
227 MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1);
228 MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM);
229 MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED);
230 MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM);
231 MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV);
234 static int create_tis(struct mlx5_vdpa_net *ndev)
236 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
237 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
241 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
242 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn);
243 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn);
245 mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err);
250 static void destroy_tis(struct mlx5_vdpa_net *ndev)
252 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn);
255 #define MLX5_VDPA_CQE_SIZE 64
256 #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE)
258 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent)
260 struct mlx5_frag_buf *frag_buf = &buf->frag_buf;
261 u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE;
262 u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE;
265 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf,
266 ndev->mvdev.mdev->priv.numa_node);
270 mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc);
272 buf->cqe_size = MLX5_VDPA_CQE_SIZE;
278 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size)
280 struct mlx5_frag_buf *frag_buf = &umem->frag_buf;
282 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf,
283 ndev->mvdev.mdev->priv.numa_node);
286 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf)
288 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf);
291 static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n)
293 return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n);
296 static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf)
298 struct mlx5_cqe64 *cqe64;
302 for (i = 0; i < buf->nent; i++) {
303 cqe = get_cqe(vcq, i);
305 cqe64->op_own = MLX5_CQE_INVALID << 4;
309 static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n)
311 struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1));
313 if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) &&
314 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe)))
320 static void rx_post(struct mlx5_vdpa_qp *vqp, int n)
323 vqp->db.db[0] = cpu_to_be32(vqp->head);
326 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in,
327 struct mlx5_vdpa_virtqueue *mvq, u32 num_ent)
329 struct mlx5_vdpa_qp *vqp;
333 vqp = fw ? &mvq->fwqp : &mvq->vqqp;
334 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid);
335 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
337 /* Firmware QP is allocated by the driver for the firmware's
338 * use so we can skip part of the params as they will be chosen by firmware
340 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
341 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
342 MLX5_SET(qpc, qpc, no_sq, 1);
346 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
347 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
348 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn);
349 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES);
350 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index);
351 MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
352 MLX5_SET(qpc, qpc, no_sq, 1);
353 MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn);
354 MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent));
355 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
356 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas);
357 mlx5_fill_page_frag_array(&vqp->frag_buf, pas);
360 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent)
362 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev,
363 num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf,
364 ndev->mvdev.mdev->priv.numa_node);
367 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp)
369 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf);
372 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq,
373 struct mlx5_vdpa_qp *vqp)
375 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
376 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
377 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
384 err = rq_buf_alloc(ndev, vqp, mvq->num_ent);
388 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db);
391 inlen += vqp->frag_buf.npages * sizeof(__be64);
394 in = kzalloc(inlen, GFP_KERNEL);
400 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent);
401 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
402 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
403 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
404 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn);
405 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES);
407 MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma);
408 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
409 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
414 vqp->mqp.uid = ndev->mvdev.res.uid;
415 vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn);
418 rx_post(vqp, mvq->num_ent);
424 mlx5_db_free(ndev->mvdev.mdev, &vqp->db);
427 rq_buf_free(ndev, vqp);
432 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp)
434 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
436 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
437 MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn);
438 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid);
439 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in))
440 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn);
442 mlx5_db_free(ndev->mvdev.mdev, &vqp->db);
443 rq_buf_free(ndev, vqp);
447 static void *next_cqe_sw(struct mlx5_vdpa_cq *cq)
449 return get_sw_cqe(cq, cq->mcq.cons_index);
452 static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq)
454 struct mlx5_cqe64 *cqe64;
456 cqe64 = next_cqe_sw(vcq);
460 vcq->mcq.cons_index++;
464 static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num)
466 mlx5_cq_set_ci(&mvq->cq.mcq);
467 rx_post(&mvq->vqqp, num);
468 if (mvq->event_cb.callback)
469 mvq->event_cb.callback(mvq->event_cb.private);
472 static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
474 struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq);
475 struct mlx5_vdpa_net *ndev = mvq->ndev;
476 void __iomem *uar_page = ndev->mvdev.res.uar->map;
479 while (!mlx5_vdpa_poll_one(&mvq->cq)) {
481 if (num > mvq->num_ent / 2) {
482 /* If completions keep coming while we poll, we want to
483 * let the hardware know that we consumed them by
484 * updating the doorbell record. We also let vdpa core
485 * know about this so it passes it on the virtio driver
488 mlx5_vdpa_handle_completions(mvq, num);
494 mlx5_vdpa_handle_completions(mvq, num);
496 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index);
499 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent)
501 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
502 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
503 void __iomem *uar_page = ndev->mvdev.res.uar->map;
504 u32 out[MLX5_ST_SZ_DW(create_cq_out)];
505 struct mlx5_vdpa_cq *vcq = &mvq->cq;
514 err = mlx5_db_alloc(mdev, &vcq->db);
518 vcq->mcq.set_ci_db = vcq->db.db;
519 vcq->mcq.arm_db = vcq->db.db + 1;
520 vcq->mcq.cqe_sz = 64;
522 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent);
526 cq_frag_buf_init(vcq, &vcq->buf);
528 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
529 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages;
530 in = kzalloc(inlen, GFP_KERNEL);
536 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid);
537 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
538 mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas);
540 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
541 MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
543 /* Use vector 0 by default. Consider adding code to choose least used
546 err = mlx5_vector2eqn(mdev, 0, &eqn, &irqn);
550 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
551 MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent));
552 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index);
553 MLX5_SET(cqc, cqc, c_eqn, eqn);
554 MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma);
556 err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out));
560 vcq->mcq.comp = mlx5_vdpa_cq_comp;
562 vcq->mcq.set_ci_db = vcq->db.db;
563 vcq->mcq.arm_db = vcq->db.db + 1;
564 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index);
571 cq_frag_buf_free(ndev, &vcq->buf);
573 mlx5_db_free(ndev->mvdev.mdev, &vcq->db);
577 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx)
579 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
580 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
581 struct mlx5_vdpa_cq *vcq = &mvq->cq;
583 if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) {
584 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn);
587 cq_frag_buf_free(ndev, &vcq->buf);
588 mlx5_db_free(ndev->mvdev.mdev, &vcq->db);
591 static int umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num,
592 struct mlx5_vdpa_umem **umemp)
594 struct mlx5_core_dev *mdev = ndev->mvdev.mdev;
600 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a);
601 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b);
602 *umemp = &mvq->umem1;
605 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a);
606 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b);
607 *umemp = &mvq->umem2;
610 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a);
611 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b);
612 *umemp = &mvq->umem3;
615 return p_a * mvq->num_ent + p_b;
618 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem)
620 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf);
623 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num)
626 u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {};
632 struct mlx5_vdpa_umem *umem;
634 size = umem_size(ndev, mvq, num, &umem);
639 err = umem_frag_buf_alloc(ndev, umem, size);
643 inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages;
645 in = kzalloc(inlen, GFP_KERNEL);
651 MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM);
652 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid);
653 um = MLX5_ADDR_OF(create_umem_in, in, umem);
654 MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
655 MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages);
657 pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]);
658 mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW);
660 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
662 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err);
667 umem->id = MLX5_GET(create_umem_out, out, umem_id);
674 umem_frag_buf_free(ndev, umem);
678 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num)
680 u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {};
681 u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {};
682 struct mlx5_vdpa_umem *umem;
696 MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM);
697 MLX5_SET(destroy_umem_in, in, umem_id, umem->id);
698 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)))
701 umem_frag_buf_free(ndev, umem);
704 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
709 for (num = 1; num <= 3; num++) {
710 err = create_umem(ndev, mvq, num);
717 for (num--; num > 0; num--)
718 umem_destroy(ndev, mvq, num);
723 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
727 for (num = 3; num > 0; num--)
728 umem_destroy(ndev, mvq, num);
731 static int get_queue_type(struct mlx5_vdpa_net *ndev)
735 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type);
737 /* prefer split queue */
738 if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)
739 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED;
741 WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT));
743 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT;
746 static bool vq_is_tx(u16 idx)
751 static u16 get_features_12_3(u64 features)
753 return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) |
754 (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) |
755 (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) |
756 (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6);
759 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
761 int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
762 u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
769 err = umems_create(ndev, mvq);
773 in = kzalloc(inlen, GFP_KERNEL);
779 cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr);
781 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
782 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
783 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
785 obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context);
786 MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
787 MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
788 get_features_12_3(ndev->mvdev.actual_features));
789 vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context);
790 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev));
792 if (vq_is_tx(mvq->index))
793 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn);
795 MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE);
796 MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index);
797 MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn);
798 MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent);
799 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
800 !!(ndev->mvdev.actual_features & VIRTIO_F_VERSION_1));
801 MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
802 MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
803 MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
804 MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey.key);
805 MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
806 MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
807 MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id);
808 MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem1.size);
809 MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id);
810 MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem1.size);
811 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn);
812 if (MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, eth_frame_offload_type))
813 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 1);
815 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
820 mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
827 umems_destroy(ndev, mvq);
831 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
833 u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {};
834 u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {};
836 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode,
837 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
838 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id);
839 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid);
840 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type,
841 MLX5_OBJ_TYPE_VIRTIO_NET_Q);
842 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) {
843 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id);
846 umems_destroy(ndev, mvq);
849 static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw)
851 return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn;
854 static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw)
856 return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn;
859 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out,
860 int *outlen, u32 qpn, u32 rqpn)
866 case MLX5_CMD_OP_2RST_QP:
867 *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in);
868 *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out);
869 *in = kzalloc(*inlen, GFP_KERNEL);
870 *out = kzalloc(*outlen, GFP_KERNEL);
874 MLX5_SET(qp_2rst_in, *in, opcode, cmd);
875 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid);
876 MLX5_SET(qp_2rst_in, *in, qpn, qpn);
878 case MLX5_CMD_OP_RST2INIT_QP:
879 *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in);
880 *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out);
881 *in = kzalloc(*inlen, GFP_KERNEL);
882 *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL);
886 MLX5_SET(rst2init_qp_in, *in, opcode, cmd);
887 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid);
888 MLX5_SET(rst2init_qp_in, *in, qpn, qpn);
889 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc);
890 MLX5_SET(qpc, qpc, remote_qpn, rqpn);
891 MLX5_SET(qpc, qpc, rwe, 1);
892 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
893 MLX5_SET(ads, pp, vhca_port_num, 1);
895 case MLX5_CMD_OP_INIT2RTR_QP:
896 *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in);
897 *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out);
898 *in = kzalloc(*inlen, GFP_KERNEL);
899 *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL);
903 MLX5_SET(init2rtr_qp_in, *in, opcode, cmd);
904 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid);
905 MLX5_SET(init2rtr_qp_in, *in, qpn, qpn);
906 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc);
907 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES);
908 MLX5_SET(qpc, qpc, log_msg_max, 30);
909 MLX5_SET(qpc, qpc, remote_qpn, rqpn);
910 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
911 MLX5_SET(ads, pp, fl, 1);
913 case MLX5_CMD_OP_RTR2RTS_QP:
914 *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in);
915 *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out);
916 *in = kzalloc(*inlen, GFP_KERNEL);
917 *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL);
921 MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd);
922 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid);
923 MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn);
924 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc);
925 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
926 MLX5_SET(ads, pp, ack_timeout, 14);
927 MLX5_SET(qpc, qpc, retry_count, 7);
928 MLX5_SET(qpc, qpc, rnr_retry, 7);
944 static void free_inout(void *in, void *out)
950 /* Two QPs are used by each virtqueue. One is used by the driver and one by
951 * firmware. The fw argument indicates whether the subjected QP is the one used
954 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd)
962 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw));
966 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen);
971 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
975 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP);
979 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP);
983 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP);
987 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP);
991 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP);
995 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP);
999 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP);
1002 struct mlx5_virtq_attr {
1004 u16 available_index;
1007 static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq,
1008 struct mlx5_virtq_attr *attr)
1010 int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out);
1011 u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {};
1017 out = kzalloc(outlen, GFP_KERNEL);
1021 cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr);
1023 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
1024 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
1025 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id);
1026 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
1027 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen);
1031 obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context);
1032 memset(attr, 0, sizeof(*attr));
1033 attr->state = MLX5_GET(virtio_net_q_object, obj_context, state);
1034 attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index);
1043 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state)
1045 int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in);
1046 u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {};
1052 in = kzalloc(inlen, GFP_KERNEL);
1056 cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr);
1058 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
1059 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
1060 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id);
1061 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
1063 obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context);
1064 MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select,
1065 MLX5_VIRTQ_MODIFY_MASK_STATE);
1066 MLX5_SET(virtio_net_q_object, obj_context, state, state);
1067 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
1070 mvq->fw_state = state;
1075 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1077 u16 idx = mvq->index;
1083 if (mvq->initialized) {
1084 mlx5_vdpa_warn(&ndev->mvdev, "attempt re init\n");
1088 err = cq_create(ndev, idx, mvq->num_ent);
1092 err = qp_create(ndev, mvq, &mvq->fwqp);
1096 err = qp_create(ndev, mvq, &mvq->vqqp);
1100 err = connect_qps(ndev, mvq);
1104 err = create_virtqueue(ndev, mvq);
1109 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
1111 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n",
1117 mvq->initialized = true;
1121 qp_destroy(ndev, &mvq->vqqp);
1123 qp_destroy(ndev, &mvq->fwqp);
1125 cq_destroy(ndev, idx);
1129 static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1131 struct mlx5_virtq_attr attr;
1133 if (!mvq->initialized)
1136 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)
1139 if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND))
1140 mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n");
1142 if (query_virtqueue(ndev, mvq, &attr)) {
1143 mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n");
1146 mvq->avail_idx = attr.available_index;
1149 static void suspend_vqs(struct mlx5_vdpa_net *ndev)
1153 for (i = 0; i < MLX5_MAX_SUPPORTED_VQS; i++)
1154 suspend_vq(ndev, &ndev->vqs[i]);
1157 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1159 if (!mvq->initialized)
1162 suspend_vq(ndev, mvq);
1163 destroy_virtqueue(ndev, mvq);
1164 qp_destroy(ndev, &mvq->vqqp);
1165 qp_destroy(ndev, &mvq->fwqp);
1166 cq_destroy(ndev, mvq->index);
1167 mvq->initialized = false;
1170 static int create_rqt(struct mlx5_vdpa_net *ndev)
1180 log_max_rqt = min_t(int, 1, MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
1181 if (log_max_rqt < 1)
1184 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + (1 << log_max_rqt) * MLX5_ST_SZ_BYTES(rq_num);
1185 in = kzalloc(inlen, GFP_KERNEL);
1189 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid);
1190 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1192 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q);
1193 MLX5_SET(rqtc, rqtc, rqt_max_size, 1 << log_max_rqt);
1194 MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
1195 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]);
1196 for (i = 0, j = 0; j < ndev->mvdev.max_vqs; j++) {
1197 if (!ndev->vqs[j].initialized)
1200 if (!vq_is_tx(ndev->vqs[j].index)) {
1201 list[i] = cpu_to_be32(ndev->vqs[j].virtq_id);
1206 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn);
1214 static void destroy_rqt(struct mlx5_vdpa_net *ndev)
1216 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn);
1219 static int create_tir(struct mlx5_vdpa_net *ndev)
1221 #define HASH_IP_L4PORTS \
1222 (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \
1223 MLX5_HASH_FIELD_SEL_L4_DPORT)
1224 static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7,
1225 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94,
1226 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1,
1227 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59,
1228 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a };
1235 in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL);
1239 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid);
1240 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1241 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
1243 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1244 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1245 rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1246 memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key));
1248 outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1249 MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4);
1250 MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP);
1251 MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS);
1253 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn);
1254 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn);
1256 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn);
1261 static void destroy_tir(struct mlx5_vdpa_net *ndev)
1263 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn);
1266 static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev)
1268 struct mlx5_flow_destination dest[2] = {};
1269 struct mlx5_flow_table_attr ft_attr = {};
1270 struct mlx5_flow_act flow_act = {};
1271 struct mlx5_flow_namespace *ns;
1274 /* for now, one entry, match all, forward to tir */
1275 ft_attr.max_fte = 1;
1276 ft_attr.autogroup.max_num_groups = 1;
1278 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS);
1280 mlx5_vdpa_warn(&ndev->mvdev, "get flow namespace\n");
1284 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
1285 if (IS_ERR(ndev->rxft))
1286 return PTR_ERR(ndev->rxft);
1288 ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false);
1289 if (IS_ERR(ndev->rx_counter)) {
1290 err = PTR_ERR(ndev->rx_counter);
1294 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT;
1295 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1296 dest[0].tir_num = ndev->res.tirn;
1297 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1298 dest[1].counter_id = mlx5_fc_id(ndev->rx_counter);
1299 ndev->rx_rule = mlx5_add_flow_rules(ndev->rxft, NULL, &flow_act, dest, 2);
1300 if (IS_ERR(ndev->rx_rule)) {
1301 err = PTR_ERR(ndev->rx_rule);
1302 ndev->rx_rule = NULL;
1309 mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter);
1311 mlx5_destroy_flow_table(ndev->rxft);
1315 static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev)
1320 mlx5_del_flow_rules(ndev->rx_rule);
1321 mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter);
1322 mlx5_destroy_flow_table(ndev->rxft);
1324 ndev->rx_rule = NULL;
1327 static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx)
1329 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1330 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1331 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1333 if (unlikely(!mvq->ready))
1336 iowrite16(idx, ndev->mvdev.res.kick_addr);
1339 static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area,
1340 u64 driver_area, u64 device_area)
1342 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1343 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1344 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1346 mvq->desc_addr = desc_area;
1347 mvq->device_addr = device_area;
1348 mvq->driver_addr = driver_area;
1352 static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
1354 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1355 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1356 struct mlx5_vdpa_virtqueue *mvq;
1358 mvq = &ndev->vqs[idx];
1362 static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
1364 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1365 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1366 struct mlx5_vdpa_virtqueue *vq = &ndev->vqs[idx];
1371 static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready)
1373 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1374 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1375 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1378 suspend_vq(ndev, mvq);
1383 static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx)
1385 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1386 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1387 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1392 static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx,
1393 const struct vdpa_vq_state *state)
1395 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1396 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1397 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1399 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) {
1400 mlx5_vdpa_warn(mvdev, "can't modify available index\n");
1404 mvq->avail_idx = state->avail_index;
1408 static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
1410 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1411 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1412 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx];
1413 struct mlx5_virtq_attr attr;
1416 /* If the virtq object was destroyed, use the value saved at
1417 * the last minute of suspend_vq. This caters for userspace
1418 * that cares about emulating the index after vq is stopped.
1420 if (!mvq->initialized) {
1421 state->avail_index = mvq->avail_idx;
1425 err = query_virtqueue(ndev, mvq, &attr);
1427 mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n");
1430 state->avail_index = attr.available_index;
1434 static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev)
1439 enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9,
1440 MLX5_VIRTIO_NET_F_CSUM = 1 << 10,
1441 MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11,
1442 MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12,
1445 static u64 mlx_to_vritio_features(u16 dev_features)
1449 if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM)
1450 result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM);
1451 if (dev_features & MLX5_VIRTIO_NET_F_CSUM)
1452 result |= BIT_ULL(VIRTIO_NET_F_CSUM);
1453 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6)
1454 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6);
1455 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4)
1456 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4);
1461 static u64 mlx5_vdpa_get_features(struct vdpa_device *vdev)
1463 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1464 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1467 dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, device_features_bits_mask);
1468 ndev->mvdev.mlx_features = mlx_to_vritio_features(dev_features);
1469 if (MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, virtio_version_1_0))
1470 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_VERSION_1);
1471 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM);
1472 print_features(mvdev, ndev->mvdev.mlx_features, false);
1473 return ndev->mvdev.mlx_features;
1476 static int verify_min_features(struct mlx5_vdpa_dev *mvdev, u64 features)
1478 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)))
1484 static int setup_virtqueues(struct mlx5_vdpa_net *ndev)
1489 for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); i++) {
1490 err = setup_vq(ndev, &ndev->vqs[i]);
1498 for (--i; i >= 0; i--)
1499 teardown_vq(ndev, &ndev->vqs[i]);
1504 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev)
1506 struct mlx5_vdpa_virtqueue *mvq;
1509 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) {
1510 mvq = &ndev->vqs[i];
1511 if (!mvq->initialized)
1514 teardown_vq(ndev, mvq);
1518 /* TODO: cross-endian support */
1519 static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev)
1521 return virtio_legacy_is_little_endian() ||
1522 (mvdev->actual_features & (1ULL << VIRTIO_F_VERSION_1));
1525 static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features)
1527 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1528 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1531 print_features(mvdev, features, true);
1533 err = verify_min_features(mvdev, features);
1537 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
1538 ndev->config.mtu = __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev),
1543 static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)
1545 /* not implemented */
1546 mlx5_vdpa_warn(to_mvdev(vdev), "set config callback not supported\n");
1549 #define MLX5_VDPA_MAX_VQ_ENTRIES 256
1550 static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev)
1552 return MLX5_VDPA_MAX_VQ_ENTRIES;
1555 static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev)
1557 return VIRTIO_ID_NET;
1560 static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev)
1562 return PCI_VENDOR_ID_MELLANOX;
1565 static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev)
1567 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1568 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1570 print_status(mvdev, ndev->mvdev.status, false);
1571 return ndev->mvdev.status;
1574 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1576 struct mlx5_vq_restore_info *ri = &mvq->ri;
1577 struct mlx5_virtq_attr attr;
1580 if (!mvq->initialized)
1583 err = query_virtqueue(ndev, mvq, &attr);
1587 ri->avail_index = attr.available_index;
1588 ri->ready = mvq->ready;
1589 ri->num_ent = mvq->num_ent;
1590 ri->desc_addr = mvq->desc_addr;
1591 ri->device_addr = mvq->device_addr;
1592 ri->driver_addr = mvq->driver_addr;
1593 ri->cb = mvq->event_cb;
1598 static int save_channels_info(struct mlx5_vdpa_net *ndev)
1602 for (i = 0; i < ndev->mvdev.max_vqs; i++) {
1603 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri));
1604 save_channel_info(ndev, &ndev->vqs[i]);
1609 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev)
1613 for (i = 0; i < ndev->mvdev.max_vqs; i++)
1614 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
1617 static void restore_channels_info(struct mlx5_vdpa_net *ndev)
1619 struct mlx5_vdpa_virtqueue *mvq;
1620 struct mlx5_vq_restore_info *ri;
1623 mlx5_clear_vqs(ndev);
1625 for (i = 0; i < ndev->mvdev.max_vqs; i++) {
1626 mvq = &ndev->vqs[i];
1631 mvq->avail_idx = ri->avail_index;
1632 mvq->ready = ri->ready;
1633 mvq->num_ent = ri->num_ent;
1634 mvq->desc_addr = ri->desc_addr;
1635 mvq->device_addr = ri->device_addr;
1636 mvq->driver_addr = ri->driver_addr;
1637 mvq->event_cb = ri->cb;
1641 static int mlx5_vdpa_change_map(struct mlx5_vdpa_net *ndev, struct vhost_iotlb *iotlb)
1646 err = save_channels_info(ndev);
1650 teardown_driver(ndev);
1651 mlx5_vdpa_destroy_mr(&ndev->mvdev);
1652 err = mlx5_vdpa_create_mr(&ndev->mvdev, iotlb);
1656 restore_channels_info(ndev);
1657 err = setup_driver(ndev);
1664 mlx5_vdpa_destroy_mr(&ndev->mvdev);
1669 static int setup_driver(struct mlx5_vdpa_net *ndev)
1673 mutex_lock(&ndev->reslock);
1675 mlx5_vdpa_warn(&ndev->mvdev, "setup driver called for already setup driver\n");
1679 err = setup_virtqueues(ndev);
1681 mlx5_vdpa_warn(&ndev->mvdev, "setup_virtqueues\n");
1685 err = create_rqt(ndev);
1687 mlx5_vdpa_warn(&ndev->mvdev, "create_rqt\n");
1691 err = create_tir(ndev);
1693 mlx5_vdpa_warn(&ndev->mvdev, "create_tir\n");
1697 err = add_fwd_to_tir(ndev);
1699 mlx5_vdpa_warn(&ndev->mvdev, "add_fwd_to_tir\n");
1703 mutex_unlock(&ndev->reslock);
1712 teardown_virtqueues(ndev);
1714 mutex_unlock(&ndev->reslock);
1718 static void teardown_driver(struct mlx5_vdpa_net *ndev)
1720 mutex_lock(&ndev->reslock);
1724 remove_fwd_to_tir(ndev);
1727 teardown_virtqueues(ndev);
1728 ndev->setup = false;
1730 mutex_unlock(&ndev->reslock);
1733 static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
1735 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1736 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1739 print_status(mvdev, status, true);
1741 mlx5_vdpa_info(mvdev, "performing device reset\n");
1742 teardown_driver(ndev);
1743 mlx5_vdpa_destroy_mr(&ndev->mvdev);
1744 ndev->mvdev.status = 0;
1745 ndev->mvdev.mlx_features = 0;
1746 ++mvdev->generation;
1750 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) {
1751 if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
1752 err = setup_driver(ndev);
1754 mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
1758 mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
1763 ndev->mvdev.status = status;
1767 mlx5_vdpa_destroy_mr(&ndev->mvdev);
1768 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
1771 static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf,
1774 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1775 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1777 if (offset + len < sizeof(struct virtio_net_config))
1778 memcpy(buf, (u8 *)&ndev->config + offset, len);
1781 static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf,
1787 static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev)
1789 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1791 return mvdev->generation;
1794 static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb)
1796 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1797 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1801 err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map);
1803 mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err);
1808 return mlx5_vdpa_change_map(ndev, iotlb);
1813 static void mlx5_vdpa_free(struct vdpa_device *vdev)
1815 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1816 struct mlx5_vdpa_net *ndev;
1818 ndev = to_mlx5_vdpa_ndev(mvdev);
1820 free_resources(ndev);
1821 mlx5_vdpa_free_resources(&ndev->mvdev);
1822 mutex_destroy(&ndev->reslock);
1825 static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx)
1827 struct vdpa_notification_area ret = {};
1832 static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx)
1837 static const struct vdpa_config_ops mlx5_vdpa_ops = {
1838 .set_vq_address = mlx5_vdpa_set_vq_address,
1839 .set_vq_num = mlx5_vdpa_set_vq_num,
1840 .kick_vq = mlx5_vdpa_kick_vq,
1841 .set_vq_cb = mlx5_vdpa_set_vq_cb,
1842 .set_vq_ready = mlx5_vdpa_set_vq_ready,
1843 .get_vq_ready = mlx5_vdpa_get_vq_ready,
1844 .set_vq_state = mlx5_vdpa_set_vq_state,
1845 .get_vq_state = mlx5_vdpa_get_vq_state,
1846 .get_vq_notification = mlx5_get_vq_notification,
1847 .get_vq_irq = mlx5_get_vq_irq,
1848 .get_vq_align = mlx5_vdpa_get_vq_align,
1849 .get_features = mlx5_vdpa_get_features,
1850 .set_features = mlx5_vdpa_set_features,
1851 .set_config_cb = mlx5_vdpa_set_config_cb,
1852 .get_vq_num_max = mlx5_vdpa_get_vq_num_max,
1853 .get_device_id = mlx5_vdpa_get_device_id,
1854 .get_vendor_id = mlx5_vdpa_get_vendor_id,
1855 .get_status = mlx5_vdpa_get_status,
1856 .set_status = mlx5_vdpa_set_status,
1857 .get_config = mlx5_vdpa_get_config,
1858 .set_config = mlx5_vdpa_set_config,
1859 .get_generation = mlx5_vdpa_get_generation,
1860 .set_map = mlx5_vdpa_set_map,
1861 .free = mlx5_vdpa_free,
1864 static int alloc_resources(struct mlx5_vdpa_net *ndev)
1866 struct mlx5_vdpa_net_resources *res = &ndev->res;
1870 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n");
1874 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn);
1878 err = create_tis(ndev);
1887 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn);
1891 static void free_resources(struct mlx5_vdpa_net *ndev)
1893 struct mlx5_vdpa_net_resources *res = &ndev->res;
1899 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn);
1903 static void init_mvqs(struct mlx5_vdpa_net *ndev)
1905 struct mlx5_vdpa_virtqueue *mvq;
1908 for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); ++i) {
1909 mvq = &ndev->vqs[i];
1910 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
1913 mvq->fwqp.fw = true;
1915 for (; i < ndev->mvdev.max_vqs; i++) {
1916 mvq = &ndev->vqs[i];
1917 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
1923 void *mlx5_vdpa_add_dev(struct mlx5_core_dev *mdev)
1925 struct virtio_net_config *config;
1926 struct mlx5_vdpa_dev *mvdev;
1927 struct mlx5_vdpa_net *ndev;
1931 /* we save one virtqueue for control virtqueue should we require it */
1932 max_vqs = MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues);
1933 max_vqs = min_t(u32, max_vqs, MLX5_MAX_SUPPORTED_VQS);
1935 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops,
1936 2 * mlx5_vdpa_max_qps(max_vqs));
1940 ndev->mvdev.max_vqs = max_vqs;
1941 mvdev = &ndev->mvdev;
1944 mutex_init(&ndev->reslock);
1945 config = &ndev->config;
1946 err = mlx5_query_nic_vport_mtu(mdev, &ndev->mtu);
1950 err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac);
1954 mvdev->vdev.dma_dev = mdev->device;
1955 err = mlx5_vdpa_alloc_resources(&ndev->mvdev);
1959 err = alloc_resources(ndev);
1963 err = vdpa_register_device(&mvdev->vdev);
1970 free_resources(ndev);
1972 mlx5_vdpa_free_resources(&ndev->mvdev);
1974 mutex_destroy(&ndev->reslock);
1975 put_device(&mvdev->vdev.dev);
1976 return ERR_PTR(err);
1979 void mlx5_vdpa_remove_dev(struct mlx5_vdpa_dev *mvdev)
1981 vdpa_unregister_device(&mvdev->vdev);