drivers/dma/idxd/cdev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/sched/task.h>
   9 #include <linux/intel-svm.h>
  10 #include <linux/io-64-nonatomic-lo-hi.h>
  11 #include <linux/cdev.h>
  12 #include <linux/fs.h>
  13 #include <linux/poll.h>
  14 #include <linux/iommu.h>
  15 #include <uapi/linux/idxd.h>
  16 #include "registers.h"
  17 #include "idxd.h"
  18
  19 struct idxd_cdev_context {
  20         const char *name;
  21         dev_t devt;
  22         struct ida minor_ida;
  23 };
  24
  25 /*
  26  * ictx is an array based off of accelerator types. enum idxd_type
  27  * is used as index
  28  */
  29 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
  30         { .name = "dsa" },
  31         { .name = "iax" }
  32 };
  33
  34 struct idxd_user_context {
  35         struct idxd_wq *wq;
  36         struct task_struct *task;
  37         unsigned int pasid;
  38         unsigned int flags;
  39         struct iommu_sva *sva;
  40 };
  41
  42 static void idxd_cdev_dev_release(struct device *dev)
  43 {
  44         struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev);
  45         struct idxd_cdev_context *cdev_ctx;
  46         struct idxd_wq *wq = idxd_cdev->wq;
  47
  48         cdev_ctx = &ictx[wq->idxd->data->type];
  49         ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
  50         kfree(idxd_cdev);
  51 }
  52
  53 static struct device_type idxd_cdev_device_type = {
  54         .name = "idxd_cdev",
  55         .release = idxd_cdev_dev_release,
  56 };
  57
  58 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
  59 {
  60         struct cdev *cdev = inode->i_cdev;
  61
  62         return container_of(cdev, struct idxd_cdev, cdev);
  63 }
  64
  65 static inline struct idxd_wq *inode_wq(struct inode *inode)
  66 {
  67         struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
  68
  69         return idxd_cdev->wq;
  70 }
  71
  72 static int idxd_cdev_open(struct inode *inode, struct file *filp)
  73 {
  74         struct idxd_user_context *ctx;
  75         struct idxd_device *idxd;
  76         struct idxd_wq *wq;
  77         struct device *dev;
  78         int rc = 0;
  79         struct iommu_sva *sva;
  80         unsigned int pasid;
  81
  82         wq = inode_wq(inode);
  83         idxd = wq->idxd;
  84         dev = &idxd->pdev->dev;
  85
  86         dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
  87
  88         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  89         if (!ctx)
  90                 return -ENOMEM;
  91
  92         mutex_lock(&wq->wq_lock);
  93
  94         if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
  95                 rc = -EBUSY;
  96                 goto failed;
  97         }
  98
  99         ctx->wq = wq;
 100         filp->private_data = ctx;
 101
 102         if (device_pasid_enabled(idxd)) {
 103                 sva = iommu_sva_bind_device(dev, current->mm, NULL);
 104                 if (IS_ERR(sva)) {
 105                         rc = PTR_ERR(sva);
 106                         dev_err(dev, "pasid allocation failed: %d\n", rc);
 107                         goto failed;
 108                 }
 109
 110                 pasid = iommu_sva_get_pasid(sva);
 111                 if (pasid == IOMMU_PASID_INVALID) {
 112                         iommu_sva_unbind_device(sva);
 113                         rc = -EINVAL;
 114                         goto failed;
 115                 }
 116
 117                 ctx->sva = sva;
 118                 ctx->pasid = pasid;
 119
 120                 if (wq_dedicated(wq)) {
 121                         rc = idxd_wq_set_pasid(wq, pasid);
 122                         if (rc < 0) {
 123                                 iommu_sva_unbind_device(sva);
 124                                 dev_err(dev, "wq set pasid failed: %d\n", rc);
 125                                 goto failed;
 126                         }
 127                 }
 128         }
 129
 130         idxd_wq_get(wq);
 131         mutex_unlock(&wq->wq_lock);
 132         return 0;
 133
 134  failed:
 135         mutex_unlock(&wq->wq_lock);
 136         kfree(ctx);
 137         return rc;
 138 }
 139
 140 static int idxd_cdev_release(struct inode *node, struct file *filep)
 141 {
 142         struct idxd_user_context *ctx = filep->private_data;
 143         struct idxd_wq *wq = ctx->wq;
 144         struct idxd_device *idxd = wq->idxd;
 145         struct device *dev = &idxd->pdev->dev;
 146         int rc;
 147
 148         dev_dbg(dev, "%s called\n", __func__);
 149         filep->private_data = NULL;
 150
 151         /* Wait for in-flight operations to complete. */
 152         if (wq_shared(wq)) {
 153                 idxd_device_drain_pasid(idxd, ctx->pasid);
 154         } else {
 155                 if (device_pasid_enabled(idxd)) {
 156                         /* The wq disable in the disable pasid function will drain the wq */
 157                         rc = idxd_wq_disable_pasid(wq);
 158                         if (rc < 0)
 159                                 dev_err(dev, "wq disable pasid failed.\n");
 160                 } else {
 161                         idxd_wq_drain(wq);
 162                 }
 163         }
 164
 165         if (ctx->sva)
 166                 iommu_sva_unbind_device(ctx->sva);
 167         kfree(ctx);
 168         mutex_lock(&wq->wq_lock);
 169         idxd_wq_put(wq);
 170         mutex_unlock(&wq->wq_lock);
 171         return 0;
 172 }
 173
 174 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
 175                      const char *func)
 176 {
 177         struct device *dev = &wq->idxd->pdev->dev;
 178
 179         if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
 180                 dev_info_ratelimited(dev,
 181                                      "%s: %s: mapping too large: %lu\n",
 182                                      current->comm, func,
 183                                      vma->vm_end - vma->vm_start);
 184                 return -EINVAL;
 185         }
 186
 187         return 0;
 188 }
 189
 190 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
 191 {
 192         struct idxd_user_context *ctx = filp->private_data;
 193         struct idxd_wq *wq = ctx->wq;
 194         struct idxd_device *idxd = wq->idxd;
 195         struct pci_dev *pdev = idxd->pdev;
 196         phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
 197         unsigned long pfn;
 198         int rc;
 199
 200         dev_dbg(&pdev->dev, "%s called\n", __func__);
 201         rc = check_vma(wq, vma, __func__);
 202         if (rc < 0)
 203                 return rc;
 204
 205         vma->vm_flags |= VM_DONTCOPY;
 206         pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
 207                                 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
 208         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 209         vma->vm_private_data = ctx;
 210
 211         return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
 212                         vma->vm_page_prot);
 213 }
 214
 215 static __poll_t idxd_cdev_poll(struct file *filp,
 216                                struct poll_table_struct *wait)
 217 {
 218         struct idxd_user_context *ctx = filp->private_data;
 219         struct idxd_wq *wq = ctx->wq;
 220         struct idxd_device *idxd = wq->idxd;
 221         unsigned long flags;
 222         __poll_t out = 0;
 223
 224         poll_wait(filp, &wq->err_queue, wait);
 225         spin_lock_irqsave(&idxd->dev_lock, flags);
 226         if (idxd->sw_err.valid)
 227                 out = EPOLLIN | EPOLLRDNORM;
 228         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 229
 230         return out;
 231 }
 232
 233 static const struct file_operations idxd_cdev_fops = {
 234         .owner = THIS_MODULE,
 235         .open = idxd_cdev_open,
 236         .release = idxd_cdev_release,
 237         .mmap = idxd_cdev_mmap,
 238         .poll = idxd_cdev_poll,
 239 };
 240
 241 int idxd_cdev_get_major(struct idxd_device *idxd)
 242 {
 243         return MAJOR(ictx[idxd->data->type].devt);
 244 }
 245
 246 int idxd_wq_add_cdev(struct idxd_wq *wq)
 247 {
 248         struct idxd_device *idxd = wq->idxd;
 249         struct idxd_cdev *idxd_cdev;
 250         struct cdev *cdev;
 251         struct device *dev;
 252         struct idxd_cdev_context *cdev_ctx;
 253         int rc, minor;
 254
 255         idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
 256         if (!idxd_cdev)
 257                 return -ENOMEM;
 258
 259         idxd_cdev->wq = wq;
 260         cdev = &idxd_cdev->cdev;
 261         dev = &idxd_cdev->dev;
 262         cdev_ctx = &ictx[wq->idxd->data->type];
 263         minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
 264         if (minor < 0) {
 265                 kfree(idxd_cdev);
 266                 return minor;
 267         }
 268         idxd_cdev->minor = minor;
 269
 270         device_initialize(dev);
 271         dev->parent = &wq->conf_dev;
 272         dev->bus = &dsa_bus_type;
 273         dev->type = &idxd_cdev_device_type;
 274         dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
 275
 276         rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
 277         if (rc < 0)
 278                 goto err;
 279
 280         wq->idxd_cdev = idxd_cdev;
 281         cdev_init(cdev, &idxd_cdev_fops);
 282         rc = cdev_device_add(cdev, dev);
 283         if (rc) {
 284                 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
 285                 goto err;
 286         }
 287
 288         return 0;
 289
 290  err:
 291         put_device(dev);
 292         wq->idxd_cdev = NULL;
 293         return rc;
 294 }
 295
 296 void idxd_wq_del_cdev(struct idxd_wq *wq)
 297 {
 298         struct idxd_cdev *idxd_cdev;
 299         struct idxd_cdev_context *cdev_ctx;
 300
 301         cdev_ctx = &ictx[wq->idxd->data->type];
 302         idxd_cdev = wq->idxd_cdev;
 303         wq->idxd_cdev = NULL;
 304         cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev);
 305         put_device(&idxd_cdev->dev);
 306 }
 307
 308 int idxd_cdev_register(void)
 309 {
 310         int rc, i;
 311
 312         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 313                 ida_init(&ictx[i].minor_ida);
 314                 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
 315                                          ictx[i].name);
 316                 if (rc)
 317                         return rc;
 318         }
 319
 320         return 0;
 321 }
 322
 323 void idxd_cdev_remove(void)
 324 {
 325         int i;
 326
 327         for (i = 0; i < IDXD_TYPE_MAX; i++) {
 328                 unregister_chrdev_region(ictx[i].devt, MINORMASK);
 329                 ida_destroy(&ictx[i].minor_ida);
 330         }
 331 }