drivers/vdpa/vdpa_user/vduse_dev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VDUSE: vDPA Device in Userspace
   4  *
   5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
   6  *
   7  * Author: Xie Yongji <xieyongji@bytedance.com>
   8  *
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/cdev.h>
  14 #include <linux/device.h>
  15 #include <linux/eventfd.h>
  16 #include <linux/slab.h>
  17 #include <linux/wait.h>
  18 #include <linux/dma-map-ops.h>
  19 #include <linux/poll.h>
  20 #include <linux/file.h>
  21 #include <linux/uio.h>
  22 #include <linux/vdpa.h>
  23 #include <linux/nospec.h>
  24 #include <uapi/linux/vduse.h>
  25 #include <uapi/linux/vdpa.h>
  26 #include <uapi/linux/virtio_config.h>
  27 #include <uapi/linux/virtio_ids.h>
  28 #include <uapi/linux/virtio_blk.h>
  29 #include <linux/mod_devicetable.h>
  30
  31 #include "iova_domain.h"
  32
  33 #define DRV_AUTHOR   "Yongji Xie <xieyongji@bytedance.com>"
  34 #define DRV_DESC     "vDPA Device in Userspace"
  35 #define DRV_LICENSE  "GPL v2"
  36
  37 #define VDUSE_DEV_MAX (1U << MINORBITS)
  38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
  39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
  40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
  41
  42 struct vduse_virtqueue {
  43         u16 index;
  44         u16 num_max;
  45         u32 num;
  46         u64 desc_addr;
  47         u64 driver_addr;
  48         u64 device_addr;
  49         struct vdpa_vq_state state;
  50         bool ready;
  51         bool kicked;
  52         spinlock_t kick_lock;
  53         spinlock_t irq_lock;
  54         struct eventfd_ctx *kickfd;
  55         struct vdpa_callback cb;
  56         struct work_struct inject;
  57         struct work_struct kick;
  58 };
  59
  60 struct vduse_dev;
  61
  62 struct vduse_vdpa {
  63         struct vdpa_device vdpa;
  64         struct vduse_dev *dev;
  65 };
  66
  67 struct vduse_dev {
  68         struct vduse_vdpa *vdev;
  69         struct device *dev;
  70         struct vduse_virtqueue *vqs;
  71         struct vduse_iova_domain *domain;
  72         char *name;
  73         struct mutex lock;
  74         spinlock_t msg_lock;
  75         u64 msg_unique;
  76         u32 msg_timeout;
  77         wait_queue_head_t waitq;
  78         struct list_head send_list;
  79         struct list_head recv_list;
  80         struct vdpa_callback config_cb;
  81         struct work_struct inject;
  82         spinlock_t irq_lock;
  83         int minor;
  84         bool broken;
  85         bool connected;
  86         u64 api_version;
  87         u64 device_features;
  88         u64 driver_features;
  89         u32 device_id;
  90         u32 vendor_id;
  91         u32 generation;
  92         u32 config_size;
  93         void *config;
  94         u8 status;
  95         u32 vq_num;
  96         u32 vq_align;
  97 };
  98
  99 struct vduse_dev_msg {
 100         struct vduse_dev_request req;
 101         struct vduse_dev_response resp;
 102         struct list_head list;
 103         wait_queue_head_t waitq;
 104         bool completed;
 105 };
 106
 107 struct vduse_control {
 108         u64 api_version;
 109 };
 110
 111 static DEFINE_MUTEX(vduse_lock);
 112 static DEFINE_IDR(vduse_idr);
 113
 114 static dev_t vduse_major;
 115 static struct class *vduse_class;
 116 static struct cdev vduse_ctrl_cdev;
 117 static struct cdev vduse_cdev;
 118 static struct workqueue_struct *vduse_irq_wq;
 119
 120 static u32 allowed_device_id[] = {
 121         VIRTIO_ID_BLOCK,
 122 };
 123
 124 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
 125 {
 126         struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
 127
 128         return vdev->dev;
 129 }
 130
 131 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
 132 {
 133         struct vdpa_device *vdpa = dev_to_vdpa(dev);
 134
 135         return vdpa_to_vduse(vdpa);
 136 }
 137
 138 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
 139                                             uint32_t request_id)
 140 {
 141         struct vduse_dev_msg *msg;
 142
 143         list_for_each_entry(msg, head, list) {
 144                 if (msg->req.request_id == request_id) {
 145                         list_del(&msg->list);
 146                         return msg;
 147                 }
 148         }
 149
 150         return NULL;
 151 }
 152
 153 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
 154 {
 155         struct vduse_dev_msg *msg = NULL;
 156
 157         if (!list_empty(head)) {
 158                 msg = list_first_entry(head, struct vduse_dev_msg, list);
 159                 list_del(&msg->list);
 160         }
 161
 162         return msg;
 163 }
 164
 165 static void vduse_enqueue_msg(struct list_head *head,
 166                               struct vduse_dev_msg *msg)
 167 {
 168         list_add_tail(&msg->list, head);
 169 }
 170
 171 static void vduse_dev_broken(struct vduse_dev *dev)
 172 {
 173         struct vduse_dev_msg *msg, *tmp;
 174
 175         if (unlikely(dev->broken))
 176                 return;
 177
 178         list_splice_init(&dev->recv_list, &dev->send_list);
 179         list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
 180                 list_del(&msg->list);
 181                 msg->completed = 1;
 182                 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
 183                 wake_up(&msg->waitq);
 184         }
 185         dev->broken = true;
 186         wake_up(&dev->waitq);
 187 }
 188
 189 static int vduse_dev_msg_sync(struct vduse_dev *dev,
 190                               struct vduse_dev_msg *msg)
 191 {
 192         int ret;
 193
 194         if (unlikely(dev->broken))
 195                 return -EIO;
 196
 197         init_waitqueue_head(&msg->waitq);
 198         spin_lock(&dev->msg_lock);
 199         if (unlikely(dev->broken)) {
 200                 spin_unlock(&dev->msg_lock);
 201                 return -EIO;
 202         }
 203         msg->req.request_id = dev->msg_unique++;
 204         vduse_enqueue_msg(&dev->send_list, msg);
 205         wake_up(&dev->waitq);
 206         spin_unlock(&dev->msg_lock);
 207         if (dev->msg_timeout)
 208                 ret = wait_event_killable_timeout(msg->waitq, msg->completed,
 209                                                   (long)dev->msg_timeout * HZ);
 210         else
 211                 ret = wait_event_killable(msg->waitq, msg->completed);
 212
 213         spin_lock(&dev->msg_lock);
 214         if (!msg->completed) {
 215                 list_del(&msg->list);
 216                 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
 217                 /* Mark the device as malfunction when there is a timeout */
 218                 if (!ret)
 219                         vduse_dev_broken(dev);
 220         }
 221         ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
 222         spin_unlock(&dev->msg_lock);
 223
 224         return ret;
 225 }
 226
 227 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
 228                                          struct vduse_virtqueue *vq,
 229                                          struct vdpa_vq_state_packed *packed)
 230 {
 231         struct vduse_dev_msg msg = { 0 };
 232         int ret;
 233
 234         msg.req.type = VDUSE_GET_VQ_STATE;
 235         msg.req.vq_state.index = vq->index;
 236
 237         ret = vduse_dev_msg_sync(dev, &msg);
 238         if (ret)
 239                 return ret;
 240
 241         packed->last_avail_counter =
 242                         msg.resp.vq_state.packed.last_avail_counter & 0x0001;
 243         packed->last_avail_idx =
 244                         msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
 245         packed->last_used_counter =
 246                         msg.resp.vq_state.packed.last_used_counter & 0x0001;
 247         packed->last_used_idx =
 248                         msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
 249
 250         return 0;
 251 }
 252
 253 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
 254                                         struct vduse_virtqueue *vq,
 255                                         struct vdpa_vq_state_split *split)
 256 {
 257         struct vduse_dev_msg msg = { 0 };
 258         int ret;
 259
 260         msg.req.type = VDUSE_GET_VQ_STATE;
 261         msg.req.vq_state.index = vq->index;
 262
 263         ret = vduse_dev_msg_sync(dev, &msg);
 264         if (ret)
 265                 return ret;
 266
 267         split->avail_index = msg.resp.vq_state.split.avail_index;
 268
 269         return 0;
 270 }
 271
 272 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
 273 {
 274         struct vduse_dev_msg msg = { 0 };
 275
 276         msg.req.type = VDUSE_SET_STATUS;
 277         msg.req.s.status = status;
 278
 279         return vduse_dev_msg_sync(dev, &msg);
 280 }
 281
 282 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
 283                                   u64 start, u64 last)
 284 {
 285         struct vduse_dev_msg msg = { 0 };
 286
 287         if (last < start)
 288                 return -EINVAL;
 289
 290         msg.req.type = VDUSE_UPDATE_IOTLB;
 291         msg.req.iova.start = start;
 292         msg.req.iova.last = last;
 293
 294         return vduse_dev_msg_sync(dev, &msg);
 295 }
 296
 297 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
 298 {
 299         struct file *file = iocb->ki_filp;
 300         struct vduse_dev *dev = file->private_data;
 301         struct vduse_dev_msg *msg;
 302         int size = sizeof(struct vduse_dev_request);
 303         ssize_t ret;
 304
 305         if (iov_iter_count(to) < size)
 306                 return -EINVAL;
 307
 308         spin_lock(&dev->msg_lock);
 309         while (1) {
 310                 msg = vduse_dequeue_msg(&dev->send_list);
 311                 if (msg)
 312                         break;
 313
 314                 ret = -EAGAIN;
 315                 if (file->f_flags & O_NONBLOCK)
 316                         goto unlock;
 317
 318                 spin_unlock(&dev->msg_lock);
 319                 ret = wait_event_interruptible_exclusive(dev->waitq,
 320                                         !list_empty(&dev->send_list));
 321                 if (ret)
 322                         return ret;
 323
 324                 spin_lock(&dev->msg_lock);
 325         }
 326         spin_unlock(&dev->msg_lock);
 327         ret = copy_to_iter(&msg->req, size, to);
 328         spin_lock(&dev->msg_lock);
 329         if (ret != size) {
 330                 ret = -EFAULT;
 331                 vduse_enqueue_msg(&dev->send_list, msg);
 332                 goto unlock;
 333         }
 334         vduse_enqueue_msg(&dev->recv_list, msg);
 335 unlock:
 336         spin_unlock(&dev->msg_lock);
 337
 338         return ret;
 339 }
 340
 341 static bool is_mem_zero(const char *ptr, int size)
 342 {
 343         int i;
 344
 345         for (i = 0; i < size; i++) {
 346                 if (ptr[i])
 347                         return false;
 348         }
 349         return true;
 350 }
 351
 352 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 353 {
 354         struct file *file = iocb->ki_filp;
 355         struct vduse_dev *dev = file->private_data;
 356         struct vduse_dev_response resp;
 357         struct vduse_dev_msg *msg;
 358         size_t ret;
 359
 360         ret = copy_from_iter(&resp, sizeof(resp), from);
 361         if (ret != sizeof(resp))
 362                 return -EINVAL;
 363
 364         if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
 365                 return -EINVAL;
 366
 367         spin_lock(&dev->msg_lock);
 368         msg = vduse_find_msg(&dev->recv_list, resp.request_id);
 369         if (!msg) {
 370                 ret = -ENOENT;
 371                 goto unlock;
 372         }
 373
 374         memcpy(&msg->resp, &resp, sizeof(resp));
 375         msg->completed = 1;
 376         wake_up(&msg->waitq);
 377 unlock:
 378         spin_unlock(&dev->msg_lock);
 379
 380         return ret;
 381 }
 382
 383 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
 384 {
 385         struct vduse_dev *dev = file->private_data;
 386         __poll_t mask = 0;
 387
 388         poll_wait(file, &dev->waitq, wait);
 389
 390         spin_lock(&dev->msg_lock);
 391
 392         if (unlikely(dev->broken))
 393                 mask |= EPOLLERR;
 394         if (!list_empty(&dev->send_list))
 395                 mask |= EPOLLIN | EPOLLRDNORM;
 396         if (!list_empty(&dev->recv_list))
 397                 mask |= EPOLLOUT | EPOLLWRNORM;
 398
 399         spin_unlock(&dev->msg_lock);
 400
 401         return mask;
 402 }
 403
 404 static void vduse_dev_reset(struct vduse_dev *dev)
 405 {
 406         int i;
 407         struct vduse_iova_domain *domain = dev->domain;
 408
 409         /* The coherent mappings are handled in vduse_dev_free_coherent() */
 410         if (domain->bounce_map)
 411                 vduse_domain_reset_bounce_map(domain);
 412
 413         dev->status = 0;
 414         dev->driver_features = 0;
 415         dev->generation++;
 416         spin_lock(&dev->irq_lock);
 417         dev->config_cb.callback = NULL;
 418         dev->config_cb.private = NULL;
 419         spin_unlock(&dev->irq_lock);
 420         flush_work(&dev->inject);
 421
 422         for (i = 0; i < dev->vq_num; i++) {
 423                 struct vduse_virtqueue *vq = &dev->vqs[i];
 424
 425                 vq->ready = false;
 426                 vq->desc_addr = 0;
 427                 vq->driver_addr = 0;
 428                 vq->device_addr = 0;
 429                 vq->num = 0;
 430                 memset(&vq->state, 0, sizeof(vq->state));
 431
 432                 spin_lock(&vq->kick_lock);
 433                 vq->kicked = false;
 434                 if (vq->kickfd)
 435                         eventfd_ctx_put(vq->kickfd);
 436                 vq->kickfd = NULL;
 437                 spin_unlock(&vq->kick_lock);
 438
 439                 spin_lock(&vq->irq_lock);
 440                 vq->cb.callback = NULL;
 441                 vq->cb.private = NULL;
 442                 spin_unlock(&vq->irq_lock);
 443                 flush_work(&vq->inject);
 444                 flush_work(&vq->kick);
 445         }
 446 }
 447
 448 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
 449                                 u64 desc_area, u64 driver_area,
 450                                 u64 device_area)
 451 {
 452         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 453         struct vduse_virtqueue *vq = &dev->vqs[idx];
 454
 455         vq->desc_addr = desc_area;
 456         vq->driver_addr = driver_area;
 457         vq->device_addr = device_area;
 458
 459         return 0;
 460 }
 461
 462 static void vduse_vq_kick(struct vduse_virtqueue *vq)
 463 {
 464         spin_lock(&vq->kick_lock);
 465         if (!vq->ready)
 466                 goto unlock;
 467
 468         if (vq->kickfd)
 469                 eventfd_signal(vq->kickfd, 1);
 470         else
 471                 vq->kicked = true;
 472 unlock:
 473         spin_unlock(&vq->kick_lock);
 474 }
 475
 476 static void vduse_vq_kick_work(struct work_struct *work)
 477 {
 478         struct vduse_virtqueue *vq = container_of(work,
 479                                         struct vduse_virtqueue, kick);
 480
 481         vduse_vq_kick(vq);
 482 }
 483
 484 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
 485 {
 486         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 487         struct vduse_virtqueue *vq = &dev->vqs[idx];
 488
 489         if (!eventfd_signal_allowed()) {
 490                 schedule_work(&vq->kick);
 491                 return;
 492         }
 493         vduse_vq_kick(vq);
 494 }
 495
 496 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
 497                               struct vdpa_callback *cb)
 498 {
 499         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 500         struct vduse_virtqueue *vq = &dev->vqs[idx];
 501
 502         spin_lock(&vq->irq_lock);
 503         vq->cb.callback = cb->callback;
 504         vq->cb.private = cb->private;
 505         spin_unlock(&vq->irq_lock);
 506 }
 507
 508 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
 509 {
 510         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 511         struct vduse_virtqueue *vq = &dev->vqs[idx];
 512
 513         vq->num = num;
 514 }
 515
 516 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 517                                         u16 idx, bool ready)
 518 {
 519         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 520         struct vduse_virtqueue *vq = &dev->vqs[idx];
 521
 522         vq->ready = ready;
 523 }
 524
 525 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
 526 {
 527         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 528         struct vduse_virtqueue *vq = &dev->vqs[idx];
 529
 530         return vq->ready;
 531 }
 532
 533 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
 534                                 const struct vdpa_vq_state *state)
 535 {
 536         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 537         struct vduse_virtqueue *vq = &dev->vqs[idx];
 538
 539         if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
 540                 vq->state.packed.last_avail_counter =
 541                                 state->packed.last_avail_counter;
 542                 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
 543                 vq->state.packed.last_used_counter =
 544                                 state->packed.last_used_counter;
 545                 vq->state.packed.last_used_idx = state->packed.last_used_idx;
 546         } else
 547                 vq->state.split.avail_index = state->split.avail_index;
 548
 549         return 0;
 550 }
 551
 552 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
 553                                 struct vdpa_vq_state *state)
 554 {
 555         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 556         struct vduse_virtqueue *vq = &dev->vqs[idx];
 557
 558         if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
 559                 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
 560
 561         return vduse_dev_get_vq_state_split(dev, vq, &state->split);
 562 }
 563
 564 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
 565 {
 566         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 567
 568         return dev->vq_align;
 569 }
 570
 571 static u64 vduse_vdpa_get_features(struct vdpa_device *vdpa)
 572 {
 573         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 574
 575         return dev->device_features;
 576 }
 577
 578 static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features)
 579 {
 580         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 581
 582         dev->driver_features = features;
 583         return 0;
 584 }
 585
 586 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
 587                                   struct vdpa_callback *cb)
 588 {
 589         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 590
 591         spin_lock(&dev->irq_lock);
 592         dev->config_cb.callback = cb->callback;
 593         dev->config_cb.private = cb->private;
 594         spin_unlock(&dev->irq_lock);
 595 }
 596
 597 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
 598 {
 599         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 600         u16 num_max = 0;
 601         int i;
 602
 603         for (i = 0; i < dev->vq_num; i++)
 604                 if (num_max < dev->vqs[i].num_max)
 605                         num_max = dev->vqs[i].num_max;
 606
 607         return num_max;
 608 }
 609
 610 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
 611 {
 612         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 613
 614         return dev->device_id;
 615 }
 616
 617 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
 618 {
 619         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 620
 621         return dev->vendor_id;
 622 }
 623
 624 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
 625 {
 626         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 627
 628         return dev->status;
 629 }
 630
 631 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
 632 {
 633         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 634
 635         if (vduse_dev_set_status(dev, status))
 636                 return;
 637
 638         dev->status = status;
 639 }
 640
 641 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
 642 {
 643         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 644
 645         return dev->config_size;
 646 }
 647
 648 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
 649                                   void *buf, unsigned int len)
 650 {
 651         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 652
 653         if (len > dev->config_size - offset)
 654                 return;
 655
 656         memcpy(buf, dev->config + offset, len);
 657 }
 658
 659 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
 660                         const void *buf, unsigned int len)
 661 {
 662         /* Now we only support read-only configuration space */
 663 }
 664
 665 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
 666 {
 667         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 668
 669         if (vduse_dev_set_status(dev, 0))
 670                 return -EIO;
 671
 672         vduse_dev_reset(dev);
 673
 674         return 0;
 675 }
 676
 677 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
 678 {
 679         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 680
 681         return dev->generation;
 682 }
 683
 684 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
 685                                 struct vhost_iotlb *iotlb)
 686 {
 687         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 688         int ret;
 689
 690         ret = vduse_domain_set_map(dev->domain, iotlb);
 691         if (ret)
 692                 return ret;
 693
 694         ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
 695         if (ret) {
 696                 vduse_domain_clear_map(dev->domain, iotlb);
 697                 return ret;
 698         }
 699
 700         return 0;
 701 }
 702
 703 static void vduse_vdpa_free(struct vdpa_device *vdpa)
 704 {
 705         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 706
 707         dev->vdev = NULL;
 708 }
 709
 710 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
 711         .set_vq_address         = vduse_vdpa_set_vq_address,
 712         .kick_vq                = vduse_vdpa_kick_vq,
 713         .set_vq_cb              = vduse_vdpa_set_vq_cb,
 714         .set_vq_num             = vduse_vdpa_set_vq_num,
 715         .set_vq_ready           = vduse_vdpa_set_vq_ready,
 716         .get_vq_ready           = vduse_vdpa_get_vq_ready,
 717         .set_vq_state           = vduse_vdpa_set_vq_state,
 718         .get_vq_state           = vduse_vdpa_get_vq_state,
 719         .get_vq_align           = vduse_vdpa_get_vq_align,
 720         .get_features           = vduse_vdpa_get_features,
 721         .set_features           = vduse_vdpa_set_features,
 722         .set_config_cb          = vduse_vdpa_set_config_cb,
 723         .get_vq_num_max         = vduse_vdpa_get_vq_num_max,
 724         .get_device_id          = vduse_vdpa_get_device_id,
 725         .get_vendor_id          = vduse_vdpa_get_vendor_id,
 726         .get_status             = vduse_vdpa_get_status,
 727         .set_status             = vduse_vdpa_set_status,
 728         .get_config_size        = vduse_vdpa_get_config_size,
 729         .get_config             = vduse_vdpa_get_config,
 730         .set_config             = vduse_vdpa_set_config,
 731         .get_generation         = vduse_vdpa_get_generation,
 732         .reset                  = vduse_vdpa_reset,
 733         .set_map                = vduse_vdpa_set_map,
 734         .free                   = vduse_vdpa_free,
 735 };
 736
 737 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
 738                                      unsigned long offset, size_t size,
 739                                      enum dma_data_direction dir,
 740                                      unsigned long attrs)
 741 {
 742         struct vduse_dev *vdev = dev_to_vduse(dev);
 743         struct vduse_iova_domain *domain = vdev->domain;
 744
 745         return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
 746 }
 747
 748 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
 749                                 size_t size, enum dma_data_direction dir,
 750                                 unsigned long attrs)
 751 {
 752         struct vduse_dev *vdev = dev_to_vduse(dev);
 753         struct vduse_iova_domain *domain = vdev->domain;
 754
 755         return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
 756 }
 757
 758 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
 759                                         dma_addr_t *dma_addr, gfp_t flag,
 760                                         unsigned long attrs)
 761 {
 762         struct vduse_dev *vdev = dev_to_vduse(dev);
 763         struct vduse_iova_domain *domain = vdev->domain;
 764         unsigned long iova;
 765         void *addr;
 766
 767         *dma_addr = DMA_MAPPING_ERROR;
 768         addr = vduse_domain_alloc_coherent(domain, size,
 769                                 (dma_addr_t *)&iova, flag, attrs);
 770         if (!addr)
 771                 return NULL;
 772
 773         *dma_addr = (dma_addr_t)iova;
 774
 775         return addr;
 776 }
 777
 778 static void vduse_dev_free_coherent(struct device *dev, size_t size,
 779                                         void *vaddr, dma_addr_t dma_addr,
 780                                         unsigned long attrs)
 781 {
 782         struct vduse_dev *vdev = dev_to_vduse(dev);
 783         struct vduse_iova_domain *domain = vdev->domain;
 784
 785         vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
 786 }
 787
 788 static size_t vduse_dev_max_mapping_size(struct device *dev)
 789 {
 790         struct vduse_dev *vdev = dev_to_vduse(dev);
 791         struct vduse_iova_domain *domain = vdev->domain;
 792
 793         return domain->bounce_size;
 794 }
 795
 796 static const struct dma_map_ops vduse_dev_dma_ops = {
 797         .map_page = vduse_dev_map_page,
 798         .unmap_page = vduse_dev_unmap_page,
 799         .alloc = vduse_dev_alloc_coherent,
 800         .free = vduse_dev_free_coherent,
 801         .max_mapping_size = vduse_dev_max_mapping_size,
 802 };
 803
 804 static unsigned int perm_to_file_flags(u8 perm)
 805 {
 806         unsigned int flags = 0;
 807
 808         switch (perm) {
 809         case VDUSE_ACCESS_WO:
 810                 flags |= O_WRONLY;
 811                 break;
 812         case VDUSE_ACCESS_RO:
 813                 flags |= O_RDONLY;
 814                 break;
 815         case VDUSE_ACCESS_RW:
 816                 flags |= O_RDWR;
 817                 break;
 818         default:
 819                 WARN(1, "invalidate vhost IOTLB permission\n");
 820                 break;
 821         }
 822
 823         return flags;
 824 }
 825
 826 static int vduse_kickfd_setup(struct vduse_dev *dev,
 827                         struct vduse_vq_eventfd *eventfd)
 828 {
 829         struct eventfd_ctx *ctx = NULL;
 830         struct vduse_virtqueue *vq;
 831         u32 index;
 832
 833         if (eventfd->index >= dev->vq_num)
 834                 return -EINVAL;
 835
 836         index = array_index_nospec(eventfd->index, dev->vq_num);
 837         vq = &dev->vqs[index];
 838         if (eventfd->fd >= 0) {
 839                 ctx = eventfd_ctx_fdget(eventfd->fd);
 840                 if (IS_ERR(ctx))
 841                         return PTR_ERR(ctx);
 842         } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
 843                 return 0;
 844
 845         spin_lock(&vq->kick_lock);
 846         if (vq->kickfd)
 847                 eventfd_ctx_put(vq->kickfd);
 848         vq->kickfd = ctx;
 849         if (vq->ready && vq->kicked && vq->kickfd) {
 850                 eventfd_signal(vq->kickfd, 1);
 851                 vq->kicked = false;
 852         }
 853         spin_unlock(&vq->kick_lock);
 854
 855         return 0;
 856 }
 857
 858 static bool vduse_dev_is_ready(struct vduse_dev *dev)
 859 {
 860         int i;
 861
 862         for (i = 0; i < dev->vq_num; i++)
 863                 if (!dev->vqs[i].num_max)
 864                         return false;
 865
 866         return true;
 867 }
 868
 869 static void vduse_dev_irq_inject(struct work_struct *work)
 870 {
 871         struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
 872
 873         spin_lock_irq(&dev->irq_lock);
 874         if (dev->config_cb.callback)
 875                 dev->config_cb.callback(dev->config_cb.private);
 876         spin_unlock_irq(&dev->irq_lock);
 877 }
 878
 879 static void vduse_vq_irq_inject(struct work_struct *work)
 880 {
 881         struct vduse_virtqueue *vq = container_of(work,
 882                                         struct vduse_virtqueue, inject);
 883
 884         spin_lock_irq(&vq->irq_lock);
 885         if (vq->ready && vq->cb.callback)
 886                 vq->cb.callback(vq->cb.private);
 887         spin_unlock_irq(&vq->irq_lock);
 888 }
 889
 890 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 891                             unsigned long arg)
 892 {
 893         struct vduse_dev *dev = file->private_data;
 894         void __user *argp = (void __user *)arg;
 895         int ret;
 896
 897         if (unlikely(dev->broken))
 898                 return -EPERM;
 899
 900         switch (cmd) {
 901         case VDUSE_IOTLB_GET_FD: {
 902                 struct vduse_iotlb_entry entry;
 903                 struct vhost_iotlb_map *map;
 904                 struct vdpa_map_file *map_file;
 905                 struct vduse_iova_domain *domain = dev->domain;
 906                 struct file *f = NULL;
 907
 908                 ret = -EFAULT;
 909                 if (copy_from_user(&entry, argp, sizeof(entry)))
 910                         break;
 911
 912                 ret = -EINVAL;
 913                 if (entry.start > entry.last)
 914                         break;
 915
 916                 spin_lock(&domain->iotlb_lock);
 917                 map = vhost_iotlb_itree_first(domain->iotlb,
 918                                               entry.start, entry.last);
 919                 if (map) {
 920                         map_file = (struct vdpa_map_file *)map->opaque;
 921                         f = get_file(map_file->file);
 922                         entry.offset = map_file->offset;
 923                         entry.start = map->start;
 924                         entry.last = map->last;
 925                         entry.perm = map->perm;
 926                 }
 927                 spin_unlock(&domain->iotlb_lock);
 928                 ret = -EINVAL;
 929                 if (!f)
 930                         break;
 931
 932                 ret = -EFAULT;
 933                 if (copy_to_user(argp, &entry, sizeof(entry))) {
 934                         fput(f);
 935                         break;
 936                 }
 937                 ret = receive_fd(f, perm_to_file_flags(entry.perm));
 938                 fput(f);
 939                 break;
 940         }
 941         case VDUSE_DEV_GET_FEATURES:
 942                 /*
 943                  * Just mirror what driver wrote here.
 944                  * The driver is expected to check FEATURE_OK later.
 945                  */
 946                 ret = put_user(dev->driver_features, (u64 __user *)argp);
 947                 break;
 948         case VDUSE_DEV_SET_CONFIG: {
 949                 struct vduse_config_data config;
 950                 unsigned long size = offsetof(struct vduse_config_data,
 951                                               buffer);
 952
 953                 ret = -EFAULT;
 954                 if (copy_from_user(&config, argp, size))
 955                         break;
 956
 957                 ret = -EINVAL;
 958                 if (config.length == 0 ||
 959                     config.length > dev->config_size - config.offset)
 960                         break;
 961
 962                 ret = -EFAULT;
 963                 if (copy_from_user(dev->config + config.offset, argp + size,
 964                                    config.length))
 965                         break;
 966
 967                 ret = 0;
 968                 break;
 969         }
 970         case VDUSE_DEV_INJECT_CONFIG_IRQ:
 971                 ret = 0;
 972                 queue_work(vduse_irq_wq, &dev->inject);
 973                 break;
 974         case VDUSE_VQ_SETUP: {
 975                 struct vduse_vq_config config;
 976                 u32 index;
 977
 978                 ret = -EFAULT;
 979                 if (copy_from_user(&config, argp, sizeof(config)))
 980                         break;
 981
 982                 ret = -EINVAL;
 983                 if (config.index >= dev->vq_num)
 984                         break;
 985
 986                 if (!is_mem_zero((const char *)config.reserved,
 987                                  sizeof(config.reserved)))
 988                         break;
 989
 990                 index = array_index_nospec(config.index, dev->vq_num);
 991                 dev->vqs[index].num_max = config.max_size;
 992                 ret = 0;
 993                 break;
 994         }
 995         case VDUSE_VQ_GET_INFO: {
 996                 struct vduse_vq_info vq_info;
 997                 struct vduse_virtqueue *vq;
 998                 u32 index;
 999
1000                 ret = -EFAULT;
1001                 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1002                         break;
1003
1004                 ret = -EINVAL;
1005                 if (vq_info.index >= dev->vq_num)
1006                         break;
1007
1008                 index = array_index_nospec(vq_info.index, dev->vq_num);
1009                 vq = &dev->vqs[index];
1010                 vq_info.desc_addr = vq->desc_addr;
1011                 vq_info.driver_addr = vq->driver_addr;
1012                 vq_info.device_addr = vq->device_addr;
1013                 vq_info.num = vq->num;
1014
1015                 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1016                         vq_info.packed.last_avail_counter =
1017                                 vq->state.packed.last_avail_counter;
1018                         vq_info.packed.last_avail_idx =
1019                                 vq->state.packed.last_avail_idx;
1020                         vq_info.packed.last_used_counter =
1021                                 vq->state.packed.last_used_counter;
1022                         vq_info.packed.last_used_idx =
1023                                 vq->state.packed.last_used_idx;
1024                 } else
1025                         vq_info.split.avail_index =
1026                                 vq->state.split.avail_index;
1027
1028                 vq_info.ready = vq->ready;
1029
1030                 ret = -EFAULT;
1031                 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1032                         break;
1033
1034                 ret = 0;
1035                 break;
1036         }
1037         case VDUSE_VQ_SETUP_KICKFD: {
1038                 struct vduse_vq_eventfd eventfd;
1039
1040                 ret = -EFAULT;
1041                 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1042                         break;
1043
1044                 ret = vduse_kickfd_setup(dev, &eventfd);
1045                 break;
1046         }
1047         case VDUSE_VQ_INJECT_IRQ: {
1048                 u32 index;
1049
1050                 ret = -EFAULT;
1051                 if (get_user(index, (u32 __user *)argp))
1052                         break;
1053
1054                 ret = -EINVAL;
1055                 if (index >= dev->vq_num)
1056                         break;
1057
1058                 ret = 0;
1059                 index = array_index_nospec(index, dev->vq_num);
1060                 queue_work(vduse_irq_wq, &dev->vqs[index].inject);
1061                 break;
1062         }
1063         default:
1064                 ret = -ENOIOCTLCMD;
1065                 break;
1066         }
1067
1068         return ret;
1069 }
1070
1071 static int vduse_dev_release(struct inode *inode, struct file *file)
1072 {
1073         struct vduse_dev *dev = file->private_data;
1074
1075         spin_lock(&dev->msg_lock);
1076         /* Make sure the inflight messages can processed after reconncection */
1077         list_splice_init(&dev->recv_list, &dev->send_list);
1078         spin_unlock(&dev->msg_lock);
1079         dev->connected = false;
1080
1081         return 0;
1082 }
1083
1084 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1085 {
1086         struct vduse_dev *dev;
1087
1088         mutex_lock(&vduse_lock);
1089         dev = idr_find(&vduse_idr, minor);
1090         mutex_unlock(&vduse_lock);
1091
1092         return dev;
1093 }
1094
1095 static int vduse_dev_open(struct inode *inode, struct file *file)
1096 {
1097         int ret;
1098         struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1099
1100         if (!dev)
1101                 return -ENODEV;
1102
1103         ret = -EBUSY;
1104         mutex_lock(&dev->lock);
1105         if (dev->connected)
1106                 goto unlock;
1107
1108         ret = 0;
1109         dev->connected = true;
1110         file->private_data = dev;
1111 unlock:
1112         mutex_unlock(&dev->lock);
1113
1114         return ret;
1115 }
1116
1117 static const struct file_operations vduse_dev_fops = {
1118         .owner          = THIS_MODULE,
1119         .open           = vduse_dev_open,
1120         .release        = vduse_dev_release,
1121         .read_iter      = vduse_dev_read_iter,
1122         .write_iter     = vduse_dev_write_iter,
1123         .poll           = vduse_dev_poll,
1124         .unlocked_ioctl = vduse_dev_ioctl,
1125         .compat_ioctl   = compat_ptr_ioctl,
1126         .llseek         = noop_llseek,
1127 };
1128
1129 static struct vduse_dev *vduse_dev_create(void)
1130 {
1131         struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1132
1133         if (!dev)
1134                 return NULL;
1135
1136         mutex_init(&dev->lock);
1137         spin_lock_init(&dev->msg_lock);
1138         INIT_LIST_HEAD(&dev->send_list);
1139         INIT_LIST_HEAD(&dev->recv_list);
1140         spin_lock_init(&dev->irq_lock);
1141
1142         INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1143         init_waitqueue_head(&dev->waitq);
1144
1145         return dev;
1146 }
1147
1148 static void vduse_dev_destroy(struct vduse_dev *dev)
1149 {
1150         kfree(dev);
1151 }
1152
1153 static struct vduse_dev *vduse_find_dev(const char *name)
1154 {
1155         struct vduse_dev *dev;
1156         int id;
1157
1158         idr_for_each_entry(&vduse_idr, dev, id)
1159                 if (!strcmp(dev->name, name))
1160                         return dev;
1161
1162         return NULL;
1163 }
1164
1165 static int vduse_destroy_dev(char *name)
1166 {
1167         struct vduse_dev *dev = vduse_find_dev(name);
1168
1169         if (!dev)
1170                 return -EINVAL;
1171
1172         mutex_lock(&dev->lock);
1173         if (dev->vdev || dev->connected) {
1174                 mutex_unlock(&dev->lock);
1175                 return -EBUSY;
1176         }
1177         dev->connected = true;
1178         mutex_unlock(&dev->lock);
1179
1180         vduse_dev_reset(dev);
1181         device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1182         idr_remove(&vduse_idr, dev->minor);
1183         kvfree(dev->config);
1184         kfree(dev->vqs);
1185         vduse_domain_destroy(dev->domain);
1186         kfree(dev->name);
1187         vduse_dev_destroy(dev);
1188         module_put(THIS_MODULE);
1189
1190         return 0;
1191 }
1192
1193 static bool device_is_allowed(u32 device_id)
1194 {
1195         int i;
1196
1197         for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1198                 if (allowed_device_id[i] == device_id)
1199                         return true;
1200
1201         return false;
1202 }
1203
1204 static bool features_is_valid(u64 features)
1205 {
1206         if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1207                 return false;
1208
1209         /* Now we only support read-only configuration space */
1210         if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1211                 return false;
1212
1213         return true;
1214 }
1215
1216 static bool vduse_validate_config(struct vduse_dev_config *config)
1217 {
1218         if (!is_mem_zero((const char *)config->reserved,
1219                          sizeof(config->reserved)))
1220                 return false;
1221
1222         if (config->vq_align > PAGE_SIZE)
1223                 return false;
1224
1225         if (config->config_size > PAGE_SIZE)
1226                 return false;
1227
1228         if (!device_is_allowed(config->device_id))
1229                 return false;
1230
1231         if (!features_is_valid(config->features))
1232                 return false;
1233
1234         return true;
1235 }
1236
1237 static ssize_t msg_timeout_show(struct device *device,
1238                                 struct device_attribute *attr, char *buf)
1239 {
1240         struct vduse_dev *dev = dev_get_drvdata(device);
1241
1242         return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1243 }
1244
1245 static ssize_t msg_timeout_store(struct device *device,
1246                                  struct device_attribute *attr,
1247                                  const char *buf, size_t count)
1248 {
1249         struct vduse_dev *dev = dev_get_drvdata(device);
1250         int ret;
1251
1252         ret = kstrtouint(buf, 10, &dev->msg_timeout);
1253         if (ret < 0)
1254                 return ret;
1255
1256         return count;
1257 }
1258
1259 static DEVICE_ATTR_RW(msg_timeout);
1260
1261 static struct attribute *vduse_dev_attrs[] = {
1262         &dev_attr_msg_timeout.attr,
1263         NULL
1264 };
1265
1266 ATTRIBUTE_GROUPS(vduse_dev);
1267
1268 static int vduse_create_dev(struct vduse_dev_config *config,
1269                             void *config_buf, u64 api_version)
1270 {
1271         int i, ret;
1272         struct vduse_dev *dev;
1273
1274         ret = -EEXIST;
1275         if (vduse_find_dev(config->name))
1276                 goto err;
1277
1278         ret = -ENOMEM;
1279         dev = vduse_dev_create();
1280         if (!dev)
1281                 goto err;
1282
1283         dev->api_version = api_version;
1284         dev->device_features = config->features;
1285         dev->device_id = config->device_id;
1286         dev->vendor_id = config->vendor_id;
1287         dev->name = kstrdup(config->name, GFP_KERNEL);
1288         if (!dev->name)
1289                 goto err_str;
1290
1291         dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1292                                           VDUSE_BOUNCE_SIZE);
1293         if (!dev->domain)
1294                 goto err_domain;
1295
1296         dev->config = config_buf;
1297         dev->config_size = config->config_size;
1298         dev->vq_align = config->vq_align;
1299         dev->vq_num = config->vq_num;
1300         dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1301         if (!dev->vqs)
1302                 goto err_vqs;
1303
1304         for (i = 0; i < dev->vq_num; i++) {
1305                 dev->vqs[i].index = i;
1306                 INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
1307                 INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
1308                 spin_lock_init(&dev->vqs[i].kick_lock);
1309                 spin_lock_init(&dev->vqs[i].irq_lock);
1310         }
1311
1312         ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1313         if (ret < 0)
1314                 goto err_idr;
1315
1316         dev->minor = ret;
1317         dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1318         dev->dev = device_create(vduse_class, NULL,
1319                                  MKDEV(MAJOR(vduse_major), dev->minor),
1320                                  dev, "%s", config->name);
1321         if (IS_ERR(dev->dev)) {
1322                 ret = PTR_ERR(dev->dev);
1323                 goto err_dev;
1324         }
1325         __module_get(THIS_MODULE);
1326
1327         return 0;
1328 err_dev:
1329         idr_remove(&vduse_idr, dev->minor);
1330 err_idr:
1331         kfree(dev->vqs);
1332 err_vqs:
1333         vduse_domain_destroy(dev->domain);
1334 err_domain:
1335         kfree(dev->name);
1336 err_str:
1337         vduse_dev_destroy(dev);
1338 err:
1339         kvfree(config_buf);
1340         return ret;
1341 }
1342
1343 static long vduse_ioctl(struct file *file, unsigned int cmd,
1344                         unsigned long arg)
1345 {
1346         int ret;
1347         void __user *argp = (void __user *)arg;
1348         struct vduse_control *control = file->private_data;
1349
1350         mutex_lock(&vduse_lock);
1351         switch (cmd) {
1352         case VDUSE_GET_API_VERSION:
1353                 ret = put_user(control->api_version, (u64 __user *)argp);
1354                 break;
1355         case VDUSE_SET_API_VERSION: {
1356                 u64 api_version;
1357
1358                 ret = -EFAULT;
1359                 if (get_user(api_version, (u64 __user *)argp))
1360                         break;
1361
1362                 ret = -EINVAL;
1363                 if (api_version > VDUSE_API_VERSION)
1364                         break;
1365
1366                 ret = 0;
1367                 control->api_version = api_version;
1368                 break;
1369         }
1370         case VDUSE_CREATE_DEV: {
1371                 struct vduse_dev_config config;
1372                 unsigned long size = offsetof(struct vduse_dev_config, config);
1373                 void *buf;
1374
1375                 ret = -EFAULT;
1376                 if (copy_from_user(&config, argp, size))
1377                         break;
1378
1379                 ret = -EINVAL;
1380                 if (vduse_validate_config(&config) == false)
1381                         break;
1382
1383                 buf = vmemdup_user(argp + size, config.config_size);
1384                 if (IS_ERR(buf)) {
1385                         ret = PTR_ERR(buf);
1386                         break;
1387                 }
1388                 config.name[VDUSE_NAME_MAX - 1] = '\0';
1389                 ret = vduse_create_dev(&config, buf, control->api_version);
1390                 break;
1391         }
1392         case VDUSE_DESTROY_DEV: {
1393                 char name[VDUSE_NAME_MAX];
1394
1395                 ret = -EFAULT;
1396                 if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1397                         break;
1398
1399                 name[VDUSE_NAME_MAX - 1] = '\0';
1400                 ret = vduse_destroy_dev(name);
1401                 break;
1402         }
1403         default:
1404                 ret = -EINVAL;
1405                 break;
1406         }
1407         mutex_unlock(&vduse_lock);
1408
1409         return ret;
1410 }
1411
1412 static int vduse_release(struct inode *inode, struct file *file)
1413 {
1414         struct vduse_control *control = file->private_data;
1415
1416         kfree(control);
1417         return 0;
1418 }
1419
1420 static int vduse_open(struct inode *inode, struct file *file)
1421 {
1422         struct vduse_control *control;
1423
1424         control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1425         if (!control)
1426                 return -ENOMEM;
1427
1428         control->api_version = VDUSE_API_VERSION;
1429         file->private_data = control;
1430
1431         return 0;
1432 }
1433
1434 static const struct file_operations vduse_ctrl_fops = {
1435         .owner          = THIS_MODULE,
1436         .open           = vduse_open,
1437         .release        = vduse_release,
1438         .unlocked_ioctl = vduse_ioctl,
1439         .compat_ioctl   = compat_ptr_ioctl,
1440         .llseek         = noop_llseek,
1441 };
1442
1443 static char *vduse_devnode(struct device *dev, umode_t *mode)
1444 {
1445         return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1446 }
1447
1448 static void vduse_mgmtdev_release(struct device *dev)
1449 {
1450 }
1451
1452 static struct device vduse_mgmtdev = {
1453         .init_name = "vduse",
1454         .release = vduse_mgmtdev_release,
1455 };
1456
1457 static struct vdpa_mgmt_dev mgmt_dev;
1458
1459 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1460 {
1461         struct vduse_vdpa *vdev;
1462         int ret;
1463
1464         if (dev->vdev)
1465                 return -EEXIST;
1466
1467         vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1468                                  &vduse_vdpa_config_ops, name, true);
1469         if (IS_ERR(vdev))
1470                 return PTR_ERR(vdev);
1471
1472         dev->vdev = vdev;
1473         vdev->dev = dev;
1474         vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1475         ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1476         if (ret) {
1477                 put_device(&vdev->vdpa.dev);
1478                 return ret;
1479         }
1480         set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1481         vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1482         vdev->vdpa.mdev = &mgmt_dev;
1483
1484         return 0;
1485 }
1486
1487 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name)
1488 {
1489         struct vduse_dev *dev;
1490         int ret;
1491
1492         mutex_lock(&vduse_lock);
1493         dev = vduse_find_dev(name);
1494         if (!dev || !vduse_dev_is_ready(dev)) {
1495                 mutex_unlock(&vduse_lock);
1496                 return -EINVAL;
1497         }
1498         ret = vduse_dev_init_vdpa(dev, name);
1499         mutex_unlock(&vduse_lock);
1500         if (ret)
1501                 return ret;
1502
1503         ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1504         if (ret) {
1505                 put_device(&dev->vdev->vdpa.dev);
1506                 return ret;
1507         }
1508
1509         return 0;
1510 }
1511
1512 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
1513 {
1514         _vdpa_unregister_device(dev);
1515 }
1516
1517 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
1518         .dev_add = vdpa_dev_add,
1519         .dev_del = vdpa_dev_del,
1520 };
1521
1522 static struct virtio_device_id id_table[] = {
1523         { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1524         { 0 },
1525 };
1526
1527 static struct vdpa_mgmt_dev mgmt_dev = {
1528         .device = &vduse_mgmtdev,
1529         .id_table = id_table,
1530         .ops = &vdpa_dev_mgmtdev_ops,
1531 };
1532
1533 static int vduse_mgmtdev_init(void)
1534 {
1535         int ret;
1536
1537         ret = device_register(&vduse_mgmtdev);
1538         if (ret)
1539                 return ret;
1540
1541         ret = vdpa_mgmtdev_register(&mgmt_dev);
1542         if (ret)
1543                 goto err;
1544
1545         return 0;
1546 err:
1547         device_unregister(&vduse_mgmtdev);
1548         return ret;
1549 }
1550
1551 static void vduse_mgmtdev_exit(void)
1552 {
1553         vdpa_mgmtdev_unregister(&mgmt_dev);
1554         device_unregister(&vduse_mgmtdev);
1555 }
1556
1557 static int vduse_init(void)
1558 {
1559         int ret;
1560         struct device *dev;
1561
1562         vduse_class = class_create(THIS_MODULE, "vduse");
1563         if (IS_ERR(vduse_class))
1564                 return PTR_ERR(vduse_class);
1565
1566         vduse_class->devnode = vduse_devnode;
1567         vduse_class->dev_groups = vduse_dev_groups;
1568
1569         ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
1570         if (ret)
1571                 goto err_chardev_region;
1572
1573         /* /dev/vduse/control */
1574         cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
1575         vduse_ctrl_cdev.owner = THIS_MODULE;
1576         ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
1577         if (ret)
1578                 goto err_ctrl_cdev;
1579
1580         dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
1581         if (IS_ERR(dev)) {
1582                 ret = PTR_ERR(dev);
1583                 goto err_device;
1584         }
1585
1586         /* /dev/vduse/$DEVICE */
1587         cdev_init(&vduse_cdev, &vduse_dev_fops);
1588         vduse_cdev.owner = THIS_MODULE;
1589         ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
1590                        VDUSE_DEV_MAX - 1);
1591         if (ret)
1592                 goto err_cdev;
1593
1594         vduse_irq_wq = alloc_workqueue("vduse-irq",
1595                                 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
1596         if (!vduse_irq_wq)
1597                 goto err_wq;
1598
1599         ret = vduse_domain_init();
1600         if (ret)
1601                 goto err_domain;
1602
1603         ret = vduse_mgmtdev_init();
1604         if (ret)
1605                 goto err_mgmtdev;
1606
1607         return 0;
1608 err_mgmtdev:
1609         vduse_domain_exit();
1610 err_domain:
1611         destroy_workqueue(vduse_irq_wq);
1612 err_wq:
1613         cdev_del(&vduse_cdev);
1614 err_cdev:
1615         device_destroy(vduse_class, vduse_major);
1616 err_device:
1617         cdev_del(&vduse_ctrl_cdev);
1618 err_ctrl_cdev:
1619         unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1620 err_chardev_region:
1621         class_destroy(vduse_class);
1622         return ret;
1623 }
1624 module_init(vduse_init);
1625
1626 static void vduse_exit(void)
1627 {
1628         vduse_mgmtdev_exit();
1629         vduse_domain_exit();
1630         destroy_workqueue(vduse_irq_wq);
1631         cdev_del(&vduse_cdev);
1632         device_destroy(vduse_class, vduse_major);
1633         cdev_del(&vduse_ctrl_cdev);
1634         unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1635         class_destroy(vduse_class);
1636 }
1637 module_exit(vduse_exit);
1638
1639 MODULE_LICENSE(DRV_LICENSE);
1640 MODULE_AUTHOR(DRV_AUTHOR);
1641 MODULE_DESCRIPTION(DRV_DESC);