1 // SPDX-License-Identifier: GPL-2.0-only
3 * Intel IFC VF NIC driver for virtio dataplane offloading
5 * Copyright (C) 2020 Intel Corporation.
7 * Author: Zhu Lingshan <lingshan.zhu@intel.com>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/sysfs.h>
15 #include "ifcvf_base.h"
17 #define DRIVER_AUTHOR "Intel Corporation"
18 #define IFCVF_DRIVER_NAME "ifcvf"
20 static irqreturn_t ifcvf_config_changed(int irq, void *arg)
22 struct ifcvf_hw *vf = arg;
24 if (vf->config_cb.callback)
25 return vf->config_cb.callback(vf->config_cb.private);
30 static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
32 struct vring_info *vring = arg;
34 if (vring->cb.callback)
35 return vring->cb.callback(vring->cb.private);
40 static void ifcvf_free_irq_vectors(void *data)
42 pci_free_irq_vectors(data);
45 static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
47 struct pci_dev *pdev = adapter->pdev;
48 struct ifcvf_hw *vf = &adapter->vf;
52 for (i = 0; i < queues; i++) {
53 devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
54 vf->vring[i].irq = -EINVAL;
57 devm_free_irq(&pdev->dev, vf->config_irq, vf);
58 ifcvf_free_irq_vectors(pdev);
61 static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
63 struct pci_dev *pdev = adapter->pdev;
64 struct ifcvf_hw *vf = &adapter->vf;
65 int vector, i, ret, irq;
67 ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
68 IFCVF_MAX_INTR, PCI_IRQ_MSIX);
70 IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
74 snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
77 vf->config_irq = pci_irq_vector(pdev, vector);
78 ret = devm_request_irq(&pdev->dev, vf->config_irq,
79 ifcvf_config_changed, 0,
80 vf->config_msix_name, vf);
82 IFCVF_ERR(pdev, "Failed to request config irq\n");
86 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
87 snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
89 vector = i + IFCVF_MSI_QUEUE_OFF;
90 irq = pci_irq_vector(pdev, vector);
91 ret = devm_request_irq(&pdev->dev, irq,
92 ifcvf_intr_handler, 0,
93 vf->vring[i].msix_name,
97 "Failed to request irq for vq %d\n", i);
98 ifcvf_free_irq(adapter, i);
103 vf->vring[i].irq = irq;
109 static int ifcvf_start_datapath(void *private)
111 struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
115 vf->nr_vring = IFCVF_MAX_QUEUE_PAIRS * 2;
116 ret = ifcvf_start_hw(vf);
118 status = ifcvf_get_status(vf);
119 status |= VIRTIO_CONFIG_S_FAILED;
120 ifcvf_set_status(vf, status);
126 static int ifcvf_stop_datapath(void *private)
128 struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
131 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++)
132 vf->vring[i].cb.callback = NULL;
139 static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
141 struct ifcvf_hw *vf = ifcvf_private_to_vf(adapter);
144 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
145 vf->vring[i].last_avail_idx = 0;
146 vf->vring[i].desc = 0;
147 vf->vring[i].avail = 0;
148 vf->vring[i].used = 0;
149 vf->vring[i].ready = 0;
150 vf->vring[i].cb.callback = NULL;
151 vf->vring[i].cb.private = NULL;
157 static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev)
159 return container_of(vdpa_dev, struct ifcvf_adapter, vdpa);
162 static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev)
164 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
169 static u64 ifcvf_vdpa_get_features(struct vdpa_device *vdpa_dev)
171 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
174 features = ifcvf_get_features(vf) & IFCVF_SUPPORTED_FEATURES;
179 static int ifcvf_vdpa_set_features(struct vdpa_device *vdpa_dev, u64 features)
181 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
184 ret = ifcvf_verify_min_features(vf, features);
188 vf->req_features = features;
193 static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev)
195 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
197 return ifcvf_get_status(vf);
200 static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
202 struct ifcvf_adapter *adapter;
207 vf = vdpa_to_vf(vdpa_dev);
208 adapter = dev_get_drvdata(vdpa_dev->dev.parent);
209 status_old = ifcvf_get_status(vf);
211 if (status_old == status)
214 if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) &&
215 !(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
216 ifcvf_stop_datapath(adapter);
217 ifcvf_free_irq(adapter, IFCVF_MAX_QUEUE_PAIRS * 2);
221 ifcvf_reset_vring(adapter);
225 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
226 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
227 ret = ifcvf_request_irq(adapter);
229 status = ifcvf_get_status(vf);
230 status |= VIRTIO_CONFIG_S_FAILED;
231 ifcvf_set_status(vf, status);
235 if (ifcvf_start_datapath(adapter) < 0)
236 IFCVF_ERR(adapter->pdev,
237 "Failed to set ifcvf vdpa status %u\n",
241 ifcvf_set_status(vf, status);
244 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
246 return IFCVF_QUEUE_MAX;
249 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
250 struct vdpa_vq_state *state)
252 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
254 state->avail_index = ifcvf_get_vq_state(vf, qid);
258 static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
259 const struct vdpa_vq_state *state)
261 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
263 return ifcvf_set_vq_state(vf, qid, state->avail_index);
266 static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
267 struct vdpa_callback *cb)
269 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
271 vf->vring[qid].cb = *cb;
274 static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev,
277 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
279 vf->vring[qid].ready = ready;
282 static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
284 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
286 return vf->vring[qid].ready;
289 static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid,
292 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
294 vf->vring[qid].size = num;
297 static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
298 u64 desc_area, u64 driver_area,
301 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
303 vf->vring[qid].desc = desc_area;
304 vf->vring[qid].avail = driver_area;
305 vf->vring[qid].used = device_area;
310 static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
312 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
314 ifcvf_notify_queue(vf, qid);
317 static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev)
319 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
321 return ioread8(&vf->common_cfg->config_generation);
324 static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
326 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
327 struct pci_dev *pdev = adapter->pdev;
330 if (pdev->device < 0x1000 || pdev->device > 0x107f)
333 if (pdev->device < 0x1040)
334 ret = pdev->subsystem_device;
336 ret = pdev->device - 0x1040;
341 static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
343 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
344 struct pci_dev *pdev = adapter->pdev;
346 return pdev->subsystem_vendor;
349 static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
351 return IFCVF_QUEUE_ALIGNMENT;
354 static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,
356 void *buf, unsigned int len)
358 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
360 WARN_ON(offset + len > sizeof(struct virtio_net_config));
361 ifcvf_read_net_config(vf, offset, buf, len);
364 static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
365 unsigned int offset, const void *buf,
368 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
370 WARN_ON(offset + len > sizeof(struct virtio_net_config));
371 ifcvf_write_net_config(vf, offset, buf, len);
374 static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
375 struct vdpa_callback *cb)
377 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
379 vf->config_cb.callback = cb->callback;
380 vf->config_cb.private = cb->private;
383 static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
386 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
388 return vf->vring[qid].irq;
392 * IFCVF currently does't have on-chip IOMMU, so not
393 * implemented set_map()/dma_map()/dma_unmap()
395 static const struct vdpa_config_ops ifc_vdpa_ops = {
396 .get_features = ifcvf_vdpa_get_features,
397 .set_features = ifcvf_vdpa_set_features,
398 .get_status = ifcvf_vdpa_get_status,
399 .set_status = ifcvf_vdpa_set_status,
400 .get_vq_num_max = ifcvf_vdpa_get_vq_num_max,
401 .get_vq_state = ifcvf_vdpa_get_vq_state,
402 .set_vq_state = ifcvf_vdpa_set_vq_state,
403 .set_vq_cb = ifcvf_vdpa_set_vq_cb,
404 .set_vq_ready = ifcvf_vdpa_set_vq_ready,
405 .get_vq_ready = ifcvf_vdpa_get_vq_ready,
406 .set_vq_num = ifcvf_vdpa_set_vq_num,
407 .set_vq_address = ifcvf_vdpa_set_vq_address,
408 .get_vq_irq = ifcvf_vdpa_get_vq_irq,
409 .kick_vq = ifcvf_vdpa_kick_vq,
410 .get_generation = ifcvf_vdpa_get_generation,
411 .get_device_id = ifcvf_vdpa_get_device_id,
412 .get_vendor_id = ifcvf_vdpa_get_vendor_id,
413 .get_vq_align = ifcvf_vdpa_get_vq_align,
414 .get_config = ifcvf_vdpa_get_config,
415 .set_config = ifcvf_vdpa_set_config,
416 .set_config_cb = ifcvf_vdpa_set_config_cb,
419 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
421 struct device *dev = &pdev->dev;
422 struct ifcvf_adapter *adapter;
426 ret = pcim_enable_device(pdev);
428 IFCVF_ERR(pdev, "Failed to enable device\n");
432 ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),
435 IFCVF_ERR(pdev, "Failed to request MMIO region\n");
439 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
441 IFCVF_ERR(pdev, "No usable DMA configuration\n");
445 ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
448 "Failed for adding devres for freeing irq vectors\n");
452 adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
453 dev, &ifc_vdpa_ops, NULL);
454 if (adapter == NULL) {
455 IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
459 pci_set_master(pdev);
460 pci_set_drvdata(pdev, adapter);
463 vf->base = pcim_iomap_table(pdev);
465 adapter->pdev = pdev;
466 adapter->vdpa.dma_dev = &pdev->dev;
468 ret = ifcvf_init_hw(vf, pdev);
470 IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
474 for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++)
475 vf->vring[i].irq = -EINVAL;
477 vf->hw_features = ifcvf_get_hw_features(vf);
479 ret = vdpa_register_device(&adapter->vdpa, IFCVF_MAX_QUEUE_PAIRS * 2);
481 IFCVF_ERR(pdev, "Failed to register ifcvf to vdpa bus");
488 put_device(&adapter->vdpa.dev);
492 static void ifcvf_remove(struct pci_dev *pdev)
494 struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
496 vdpa_unregister_device(&adapter->vdpa);
499 static struct pci_device_id ifcvf_pci_ids[] = {
500 { PCI_DEVICE_SUB(N3000_VENDOR_ID,
502 N3000_SUBSYS_VENDOR_ID,
503 N3000_SUBSYS_DEVICE_ID) },
504 { PCI_DEVICE_SUB(C5000X_PL_VENDOR_ID,
506 C5000X_PL_SUBSYS_VENDOR_ID,
507 C5000X_PL_SUBSYS_DEVICE_ID) },
511 MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
513 static struct pci_driver ifcvf_driver = {
514 .name = IFCVF_DRIVER_NAME,
515 .id_table = ifcvf_pci_ids,
516 .probe = ifcvf_probe,
517 .remove = ifcvf_remove,
520 module_pci_driver(ifcvf_driver);
522 MODULE_LICENSE("GPL v2");