1 // SPDX-License-Identifier: GPL-2.0-only
3 * Intel IFC VF NIC driver for virtio dataplane offloading
5 * Copyright (C) 2020 Intel Corporation.
7 * Author: Zhu Lingshan <lingshan.zhu@intel.com>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/sysfs.h>
15 #include "ifcvf_base.h"
17 #define DRIVER_AUTHOR "Intel Corporation"
18 #define IFCVF_DRIVER_NAME "ifcvf"
20 static irqreturn_t ifcvf_config_changed(int irq, void *arg)
22 struct ifcvf_hw *vf = arg;
24 if (vf->config_cb.callback)
25 return vf->config_cb.callback(vf->config_cb.private);
30 static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
32 struct vring_info *vring = arg;
34 if (vring->cb.callback)
35 return vring->cb.callback(vring->cb.private);
40 static void ifcvf_free_irq_vectors(void *data)
42 pci_free_irq_vectors(data);
45 static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
47 struct pci_dev *pdev = adapter->pdev;
48 struct ifcvf_hw *vf = &adapter->vf;
52 for (i = 0; i < queues; i++) {
53 devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
54 vf->vring[i].irq = -EINVAL;
57 devm_free_irq(&pdev->dev, vf->config_irq, vf);
58 ifcvf_free_irq_vectors(pdev);
61 static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
63 struct pci_dev *pdev = adapter->pdev;
64 struct ifcvf_hw *vf = &adapter->vf;
65 int vector, i, ret, irq;
67 ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
68 IFCVF_MAX_INTR, PCI_IRQ_MSIX);
70 IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
74 snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
77 vf->config_irq = pci_irq_vector(pdev, vector);
78 ret = devm_request_irq(&pdev->dev, vf->config_irq,
79 ifcvf_config_changed, 0,
80 vf->config_msix_name, vf);
82 IFCVF_ERR(pdev, "Failed to request config irq\n");
86 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
87 snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
89 vector = i + IFCVF_MSI_QUEUE_OFF;
90 irq = pci_irq_vector(pdev, vector);
91 ret = devm_request_irq(&pdev->dev, irq,
92 ifcvf_intr_handler, 0,
93 vf->vring[i].msix_name,
97 "Failed to request irq for vq %d\n", i);
98 ifcvf_free_irq(adapter, i);
103 vf->vring[i].irq = irq;
109 static int ifcvf_start_datapath(void *private)
111 struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
115 vf->nr_vring = IFCVF_MAX_QUEUE_PAIRS * 2;
116 ret = ifcvf_start_hw(vf);
118 status = ifcvf_get_status(vf);
119 status |= VIRTIO_CONFIG_S_FAILED;
120 ifcvf_set_status(vf, status);
126 static int ifcvf_stop_datapath(void *private)
128 struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
131 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++)
132 vf->vring[i].cb.callback = NULL;
139 static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
141 struct ifcvf_hw *vf = ifcvf_private_to_vf(adapter);
144 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
145 vf->vring[i].last_avail_idx = 0;
146 vf->vring[i].desc = 0;
147 vf->vring[i].avail = 0;
148 vf->vring[i].used = 0;
149 vf->vring[i].ready = 0;
150 vf->vring[i].cb.callback = NULL;
151 vf->vring[i].cb.private = NULL;
157 static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev)
159 return container_of(vdpa_dev, struct ifcvf_adapter, vdpa);
162 static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev)
164 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
169 static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev)
171 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
174 features = ifcvf_get_features(vf) & IFCVF_SUPPORTED_FEATURES;
179 static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features)
181 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
184 ret = ifcvf_verify_min_features(vf, features);
188 vf->req_features = features;
193 static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev)
195 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
197 return ifcvf_get_status(vf);
200 static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
202 struct ifcvf_adapter *adapter;
207 vf = vdpa_to_vf(vdpa_dev);
208 adapter = dev_get_drvdata(vdpa_dev->dev.parent);
209 status_old = ifcvf_get_status(vf);
211 if (status_old == status)
214 if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) &&
215 !(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
216 ifcvf_stop_datapath(adapter);
217 ifcvf_free_irq(adapter, IFCVF_MAX_QUEUE_PAIRS * 2);
221 ifcvf_reset_vring(adapter);
225 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
226 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
227 ret = ifcvf_request_irq(adapter);
229 status = ifcvf_get_status(vf);
230 status |= VIRTIO_CONFIG_S_FAILED;
231 ifcvf_set_status(vf, status);
235 if (ifcvf_start_datapath(adapter) < 0)
236 IFCVF_ERR(adapter->pdev,
237 "Failed to set ifcvf vdpa status %u\n",
241 ifcvf_set_status(vf, status);
244 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
246 return IFCVF_QUEUE_MAX;
249 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
250 struct vdpa_vq_state *state)
252 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
254 state->avail_index = ifcvf_get_vq_state(vf, qid);
258 static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
259 const struct vdpa_vq_state *state)
261 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
263 return ifcvf_set_vq_state(vf, qid, state->avail_index);
266 static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
267 struct vdpa_callback *cb)
269 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
271 vf->vring[qid].cb = *cb;
274 static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev,
277 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
279 vf->vring[qid].ready = ready;
282 static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
284 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
286 return vf->vring[qid].ready;
289 static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid,
292 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
294 vf->vring[qid].size = num;
297 static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
298 u64 desc_area, u64 driver_area,
301 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
303 vf->vring[qid].desc = desc_area;
304 vf->vring[qid].avail = driver_area;
305 vf->vring[qid].used = device_area;
310 static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
312 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
314 ifcvf_notify_queue(vf, qid);
317 static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev)
319 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
321 return ioread8(&vf->common_cfg->config_generation);
324 static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
326 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
331 static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
333 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
334 struct pci_dev *pdev = adapter->pdev;
336 return pdev->subsystem_vendor;
339 static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
341 return IFCVF_QUEUE_ALIGNMENT;
344 static size_t ifcvf_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
346 return sizeof(struct virtio_net_config);
349 static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,
351 void *buf, unsigned int len)
353 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
355 WARN_ON(offset + len > sizeof(struct virtio_net_config));
356 ifcvf_read_net_config(vf, offset, buf, len);
359 static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
360 unsigned int offset, const void *buf,
363 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
365 WARN_ON(offset + len > sizeof(struct virtio_net_config));
366 ifcvf_write_net_config(vf, offset, buf, len);
369 static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
370 struct vdpa_callback *cb)
372 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
374 vf->config_cb.callback = cb->callback;
375 vf->config_cb.private = cb->private;
378 static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
381 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
383 return vf->vring[qid].irq;
387 * IFCVF currently does't have on-chip IOMMU, so not
388 * implemented set_map()/dma_map()/dma_unmap()
390 static const struct vdpa_config_ops ifc_vdpa_ops = {
391 .get_features = ifcvf_vdpa_get_features,
392 .set_features = ifcvf_vdpa_set_features,
393 .get_status = ifcvf_vdpa_get_status,
394 .set_status = ifcvf_vdpa_set_status,
395 .get_vq_num_max = ifcvf_vdpa_get_vq_num_max,
396 .get_vq_state = ifcvf_vdpa_get_vq_state,
397 .set_vq_state = ifcvf_vdpa_set_vq_state,
398 .set_vq_cb = ifcvf_vdpa_set_vq_cb,
399 .set_vq_ready = ifcvf_vdpa_set_vq_ready,
400 .get_vq_ready = ifcvf_vdpa_get_vq_ready,
401 .set_vq_num = ifcvf_vdpa_set_vq_num,
402 .set_vq_address = ifcvf_vdpa_set_vq_address,
403 .get_vq_irq = ifcvf_vdpa_get_vq_irq,
404 .kick_vq = ifcvf_vdpa_kick_vq,
405 .get_generation = ifcvf_vdpa_get_generation,
406 .get_device_id = ifcvf_vdpa_get_device_id,
407 .get_vendor_id = ifcvf_vdpa_get_vendor_id,
408 .get_vq_align = ifcvf_vdpa_get_vq_align,
409 .get_config_size = ifcvf_vdpa_get_config_size,
410 .get_config = ifcvf_vdpa_get_config,
411 .set_config = ifcvf_vdpa_set_config,
412 .set_config_cb = ifcvf_vdpa_set_config_cb,
415 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
417 struct device *dev = &pdev->dev;
418 struct ifcvf_adapter *adapter;
422 ret = pcim_enable_device(pdev);
424 IFCVF_ERR(pdev, "Failed to enable device\n");
428 ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
431 IFCVF_ERR(pdev, "Failed to request MMIO region\n");
435 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
437 IFCVF_ERR(pdev, "No usable DMA configuration\n");
441 ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
444 "Failed for adding devres for freeing irq vectors\n");
448 adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
449 dev, &ifc_vdpa_ops, NULL);
450 if (adapter == NULL) {
451 IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
455 pci_set_master(pdev);
456 pci_set_drvdata(pdev, adapter);
460 /* This drirver drives both modern virtio devices and transitional
461 * devices in modern mode.
462 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM,
463 * so legacy devices and transitional devices in legacy
464 * mode will not work for vDPA, this driver will not
465 * drive devices with legacy interface.
467 if (pdev->device < 0x1040)
468 vf->dev_type = pdev->subsystem_device;
470 vf->dev_type = pdev->device - 0x1040;
472 vf->base = pcim_iomap_table(pdev);
474 adapter->pdev = pdev;
475 adapter->vdpa.dma_dev = &pdev->dev;
477 ret = ifcvf_init_hw(vf, pdev);
479 IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
483 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++)
484 vf->vring[i].irq = -EINVAL;
486 vf->hw_features = ifcvf_get_hw_features(vf);
488 ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
490 IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus");
497 put_device(&adapter->vdpa.dev);
501 static void ifcvf_remove(struct pci_dev *pdev)
503 struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
505 vdpa_unregister_device(&adapter->vdpa);
508 static struct pci_device_id ifcvf_pci_ids[] = {
509 { PCI_DEVICE_SUB(N3000_VENDOR_ID,
511 N3000_SUBSYS_VENDOR_ID,
512 N3000_SUBSYS_DEVICE_ID) },
513 { PCI_DEVICE_SUB(C5000X_PL_VENDOR_ID,
515 C5000X_PL_SUBSYS_VENDOR_ID,
516 C5000X_PL_SUBSYS_DEVICE_ID) },
520 MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
522 static struct pci_driver ifcvf_driver = {
523 .name = IFCVF_DRIVER_NAME,
524 .id_table = ifcvf_pci_ids,
525 .probe = ifcvf_probe,
526 .remove = ifcvf_remove,
529 module_pci_driver(ifcvf_driver);
531 MODULE_LICENSE("GPL v2");