1 // SPDX-License-Identifier: GPL-2.0-only
3 * Intel IFC VF NIC driver for virtio dataplane offloading
5 * Copyright (C) 2020 Intel Corporation.
7 * Author: Zhu Lingshan <lingshan.zhu@intel.com>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/sysfs.h>
15 #include "ifcvf_base.h"
17 #define DRIVER_AUTHOR "Intel Corporation"
18 #define IFCVF_DRIVER_NAME "ifcvf"
20 static irqreturn_t ifcvf_config_changed(int irq, void *arg)
22 struct ifcvf_hw *vf = arg;
24 if (vf->config_cb.callback)
25 return vf->config_cb.callback(vf->config_cb.private);
30 static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
32 struct vring_info *vring = arg;
34 if (vring->cb.callback)
35 return vring->cb.callback(vring->cb.private);
40 static void ifcvf_free_irq_vectors(void *data)
42 pci_free_irq_vectors(data);
45 static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
47 struct pci_dev *pdev = adapter->pdev;
48 struct ifcvf_hw *vf = &adapter->vf;
52 for (i = 0; i < queues; i++) {
53 devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
54 vf->vring[i].irq = -EINVAL;
57 devm_free_irq(&pdev->dev, vf->config_irq, vf);
58 ifcvf_free_irq_vectors(pdev);
61 static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
63 struct pci_dev *pdev = adapter->pdev;
64 struct ifcvf_hw *vf = &adapter->vf;
65 int vector, i, ret, irq;
68 /* all queues and config interrupt */
69 max_intr = vf->nr_vring + 1;
71 ret = pci_alloc_irq_vectors(pdev, max_intr,
72 max_intr, PCI_IRQ_MSIX);
74 IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
78 snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
81 vf->config_irq = pci_irq_vector(pdev, vector);
82 ret = devm_request_irq(&pdev->dev, vf->config_irq,
83 ifcvf_config_changed, 0,
84 vf->config_msix_name, vf);
86 IFCVF_ERR(pdev, "Failed to request config irq\n");
90 for (i = 0; i < vf->nr_vring; i++) {
91 snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
93 vector = i + IFCVF_MSI_QUEUE_OFF;
94 irq = pci_irq_vector(pdev, vector);
95 ret = devm_request_irq(&pdev->dev, irq,
96 ifcvf_intr_handler, 0,
97 vf->vring[i].msix_name,
101 "Failed to request irq for vq %d\n", i);
102 ifcvf_free_irq(adapter, i);
107 vf->vring[i].irq = irq;
113 static int ifcvf_start_datapath(void *private)
115 struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
119 ret = ifcvf_start_hw(vf);
121 status = ifcvf_get_status(vf);
122 status |= VIRTIO_CONFIG_S_FAILED;
123 ifcvf_set_status(vf, status);
129 static int ifcvf_stop_datapath(void *private)
131 struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
134 for (i = 0; i < vf->nr_vring; i++)
135 vf->vring[i].cb.callback = NULL;
142 static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
144 struct ifcvf_hw *vf = ifcvf_private_to_vf(adapter);
147 for (i = 0; i < vf->nr_vring; i++) {
148 vf->vring[i].last_avail_idx = 0;
149 vf->vring[i].desc = 0;
150 vf->vring[i].avail = 0;
151 vf->vring[i].used = 0;
152 vf->vring[i].ready = 0;
153 vf->vring[i].cb.callback = NULL;
154 vf->vring[i].cb.private = NULL;
160 static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev)
162 return container_of(vdpa_dev, struct ifcvf_adapter, vdpa);
165 static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev)
167 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
172 static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev)
174 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
175 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
176 struct pci_dev *pdev = adapter->pdev;
177 u32 type = vf->dev_type;
180 if (type == VIRTIO_ID_NET || type == VIRTIO_ID_BLOCK)
181 features = ifcvf_get_features(vf);
184 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type);
190 static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features)
192 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
195 ret = ifcvf_verify_min_features(vf, features);
199 vf->req_features = features;
204 static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev)
206 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
208 return ifcvf_get_status(vf);
211 static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
213 struct ifcvf_adapter *adapter;
218 vf = vdpa_to_vf(vdpa_dev);
219 adapter = vdpa_to_adapter(vdpa_dev);
220 status_old = ifcvf_get_status(vf);
222 if (status_old == status)
225 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
226 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
227 ret = ifcvf_request_irq(adapter);
229 status = ifcvf_get_status(vf);
230 status |= VIRTIO_CONFIG_S_FAILED;
231 ifcvf_set_status(vf, status);
235 if (ifcvf_start_datapath(adapter) < 0)
236 IFCVF_ERR(adapter->pdev,
237 "Failed to set ifcvf vdpa status %u\n",
241 ifcvf_set_status(vf, status);
244 static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev)
246 struct ifcvf_adapter *adapter;
250 vf = vdpa_to_vf(vdpa_dev);
251 adapter = vdpa_to_adapter(vdpa_dev);
252 status_old = ifcvf_get_status(vf);
257 if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) {
258 ifcvf_stop_datapath(adapter);
259 ifcvf_free_irq(adapter, vf->nr_vring);
262 ifcvf_reset_vring(adapter);
267 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
269 return IFCVF_QUEUE_MAX;
272 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
273 struct vdpa_vq_state *state)
275 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
277 state->split.avail_index = ifcvf_get_vq_state(vf, qid);
281 static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
282 const struct vdpa_vq_state *state)
284 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
286 return ifcvf_set_vq_state(vf, qid, state->split.avail_index);
289 static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
290 struct vdpa_callback *cb)
292 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
294 vf->vring[qid].cb = *cb;
297 static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev,
300 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
302 vf->vring[qid].ready = ready;
305 static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
307 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
309 return vf->vring[qid].ready;
312 static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid,
315 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
317 vf->vring[qid].size = num;
320 static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
321 u64 desc_area, u64 driver_area,
324 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
326 vf->vring[qid].desc = desc_area;
327 vf->vring[qid].avail = driver_area;
328 vf->vring[qid].used = device_area;
333 static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
335 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
337 ifcvf_notify_queue(vf, qid);
340 static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev)
342 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
344 return ioread8(&vf->common_cfg->config_generation);
347 static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
349 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
354 static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
356 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
357 struct pci_dev *pdev = adapter->pdev;
359 return pdev->subsystem_vendor;
362 static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
364 return IFCVF_QUEUE_ALIGNMENT;
367 static size_t ifcvf_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
369 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
370 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
371 struct pci_dev *pdev = adapter->pdev;
374 switch (vf->dev_type) {
376 size = sizeof(struct virtio_net_config);
378 case VIRTIO_ID_BLOCK:
379 size = sizeof(struct virtio_blk_config);
383 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type);
389 static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,
391 void *buf, unsigned int len)
393 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
395 WARN_ON(offset + len > sizeof(struct virtio_net_config));
396 ifcvf_read_net_config(vf, offset, buf, len);
399 static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
400 unsigned int offset, const void *buf,
403 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
405 WARN_ON(offset + len > sizeof(struct virtio_net_config));
406 ifcvf_write_net_config(vf, offset, buf, len);
409 static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
410 struct vdpa_callback *cb)
412 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
414 vf->config_cb.callback = cb->callback;
415 vf->config_cb.private = cb->private;
418 static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
421 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
423 return vf->vring[qid].irq;
426 static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
429 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
430 struct vdpa_notification_area area;
432 area.addr = vf->vring[idx].notify_pa;
433 if (!vf->notify_off_multiplier)
434 area.size = PAGE_SIZE;
436 area.size = vf->notify_off_multiplier;
442 * IFCVF currently does't have on-chip IOMMU, so not
443 * implemented set_map()/dma_map()/dma_unmap()
445 static const struct vdpa_config_ops ifc_vdpa_ops = {
446 .get_features = ifcvf_vdpa_get_features,
447 .set_features = ifcvf_vdpa_set_features,
448 .get_status = ifcvf_vdpa_get_status,
449 .set_status = ifcvf_vdpa_set_status,
450 .reset = ifcvf_vdpa_reset,
451 .get_vq_num_max = ifcvf_vdpa_get_vq_num_max,
452 .get_vq_state = ifcvf_vdpa_get_vq_state,
453 .set_vq_state = ifcvf_vdpa_set_vq_state,
454 .set_vq_cb = ifcvf_vdpa_set_vq_cb,
455 .set_vq_ready = ifcvf_vdpa_set_vq_ready,
456 .get_vq_ready = ifcvf_vdpa_get_vq_ready,
457 .set_vq_num = ifcvf_vdpa_set_vq_num,
458 .set_vq_address = ifcvf_vdpa_set_vq_address,
459 .get_vq_irq = ifcvf_vdpa_get_vq_irq,
460 .kick_vq = ifcvf_vdpa_kick_vq,
461 .get_generation = ifcvf_vdpa_get_generation,
462 .get_device_id = ifcvf_vdpa_get_device_id,
463 .get_vendor_id = ifcvf_vdpa_get_vendor_id,
464 .get_vq_align = ifcvf_vdpa_get_vq_align,
465 .get_config_size = ifcvf_vdpa_get_config_size,
466 .get_config = ifcvf_vdpa_get_config,
467 .set_config = ifcvf_vdpa_set_config,
468 .set_config_cb = ifcvf_vdpa_set_config_cb,
469 .get_vq_notification = ifcvf_get_vq_notification,
472 static struct virtio_device_id id_table_net[] = {
473 {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
477 static struct virtio_device_id id_table_blk[] = {
478 {VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},
482 static u32 get_dev_type(struct pci_dev *pdev)
486 /* This drirver drives both modern virtio devices and transitional
487 * devices in modern mode.
488 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM,
489 * so legacy devices and transitional devices in legacy
490 * mode will not work for vDPA, this driver will not
491 * drive devices with legacy interface.
494 if (pdev->device < 0x1040)
495 dev_type = pdev->subsystem_device;
497 dev_type = pdev->device - 0x1040;
502 static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
503 const struct vdpa_dev_set_config *config)
505 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
506 struct ifcvf_adapter *adapter;
507 struct pci_dev *pdev;
512 ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
513 if (ifcvf_mgmt_dev->adapter)
516 pdev = ifcvf_mgmt_dev->pdev;
518 adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
519 dev, &ifc_vdpa_ops, name, false);
520 if (IS_ERR(adapter)) {
521 IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
522 return PTR_ERR(adapter);
525 ifcvf_mgmt_dev->adapter = adapter;
526 pci_set_drvdata(pdev, ifcvf_mgmt_dev);
529 vf->dev_type = get_dev_type(pdev);
530 vf->base = pcim_iomap_table(pdev);
532 adapter->pdev = pdev;
533 adapter->vdpa.dma_dev = &pdev->dev;
535 ret = ifcvf_init_hw(vf, pdev);
537 IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
541 for (i = 0; i < vf->nr_vring; i++)
542 vf->vring[i].irq = -EINVAL;
544 vf->hw_features = ifcvf_get_hw_features(vf);
546 adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
547 ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);
549 IFCVF_ERR(pdev, "Failed to register to vDPA bus");
556 put_device(&adapter->vdpa.dev);
560 static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
562 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
564 ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
565 _vdpa_unregister_device(dev);
566 ifcvf_mgmt_dev->adapter = NULL;
569 static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {
570 .dev_add = ifcvf_vdpa_dev_add,
571 .dev_del = ifcvf_vdpa_dev_del
574 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
576 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
577 struct device *dev = &pdev->dev;
581 ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
582 if (!ifcvf_mgmt_dev) {
583 IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
587 dev_type = get_dev_type(pdev);
590 ifcvf_mgmt_dev->mdev.id_table = id_table_net;
592 case VIRTIO_ID_BLOCK:
593 ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
596 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
601 ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
602 ifcvf_mgmt_dev->mdev.device = dev;
603 ifcvf_mgmt_dev->pdev = pdev;
605 ret = pcim_enable_device(pdev);
607 IFCVF_ERR(pdev, "Failed to enable device\n");
611 ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
614 IFCVF_ERR(pdev, "Failed to request MMIO region\n");
618 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
620 IFCVF_ERR(pdev, "No usable DMA configuration\n");
624 ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
627 "Failed for adding devres for freeing irq vectors\n");
631 pci_set_master(pdev);
633 ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
636 "Failed to initialize the management interfaces\n");
643 kfree(ifcvf_mgmt_dev);
647 static void ifcvf_remove(struct pci_dev *pdev)
649 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
651 ifcvf_mgmt_dev = pci_get_drvdata(pdev);
652 vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
653 kfree(ifcvf_mgmt_dev);
656 static struct pci_device_id ifcvf_pci_ids[] = {
657 /* N3000 network device */
658 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
661 N3000_SUBSYS_DEVICE_ID) },
662 /* C5000X-PL network device */
663 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
667 /* C5000X-PL block device */
668 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
669 VIRTIO_TRANS_ID_BLOCK,
675 MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
677 static struct pci_driver ifcvf_driver = {
678 .name = IFCVF_DRIVER_NAME,
679 .id_table = ifcvf_pci_ids,
680 .probe = ifcvf_probe,
681 .remove = ifcvf_remove,
684 module_pci_driver(ifcvf_driver);
686 MODULE_LICENSE("GPL v2");