Merge tag 'docs-6.5-2' of git://git.lwn.net/linux
[linux-2.6-microblaze.git] / drivers / vhost / vdpa.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2018-2020 Intel Corporation.
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Author: Tiwei Bie <tiwei.bie@intel.com>
7  *         Jason Wang <jasowang@redhat.com>
8  *
9  * Thanks Michael S. Tsirkin for the valuable comments and
10  * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
11  * their supports.
12  */
13
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/cdev.h>
17 #include <linux/device.h>
18 #include <linux/mm.h>
19 #include <linux/slab.h>
20 #include <linux/iommu.h>
21 #include <linux/uuid.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vhost.h>
25
26 #include "vhost.h"
27
28 enum {
29         VHOST_VDPA_BACKEND_FEATURES =
30         (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
31         (1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
32         (1ULL << VHOST_BACKEND_F_IOTLB_ASID),
33 };
34
35 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
36
37 #define VHOST_VDPA_IOTLB_BUCKETS 16
38
39 struct vhost_vdpa_as {
40         struct hlist_node hash_link;
41         struct vhost_iotlb iotlb;
42         u32 id;
43 };
44
45 struct vhost_vdpa {
46         struct vhost_dev vdev;
47         struct iommu_domain *domain;
48         struct vhost_virtqueue *vqs;
49         struct completion completion;
50         struct vdpa_device *vdpa;
51         struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
52         struct device dev;
53         struct cdev cdev;
54         atomic_t opened;
55         u32 nvqs;
56         int virtio_id;
57         int minor;
58         struct eventfd_ctx *config_ctx;
59         int in_batch;
60         struct vdpa_iova_range range;
61         u32 batch_asid;
62 };
63
64 static DEFINE_IDA(vhost_vdpa_ida);
65
66 static dev_t vhost_vdpa_major;
67
68 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
69                                    struct vhost_iotlb *iotlb, u64 start,
70                                    u64 last, u32 asid);
71
72 static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
73 {
74         struct vhost_vdpa_as *as = container_of(iotlb, struct
75                                                 vhost_vdpa_as, iotlb);
76         return as->id;
77 }
78
79 static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
80 {
81         struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
82         struct vhost_vdpa_as *as;
83
84         hlist_for_each_entry(as, head, hash_link)
85                 if (as->id == asid)
86                         return as;
87
88         return NULL;
89 }
90
91 static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
92 {
93         struct vhost_vdpa_as *as = asid_to_as(v, asid);
94
95         if (!as)
96                 return NULL;
97
98         return &as->iotlb;
99 }
100
101 static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
102 {
103         struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
104         struct vhost_vdpa_as *as;
105
106         if (asid_to_as(v, asid))
107                 return NULL;
108
109         if (asid >= v->vdpa->nas)
110                 return NULL;
111
112         as = kmalloc(sizeof(*as), GFP_KERNEL);
113         if (!as)
114                 return NULL;
115
116         vhost_iotlb_init(&as->iotlb, 0, 0);
117         as->id = asid;
118         hlist_add_head(&as->hash_link, head);
119
120         return as;
121 }
122
123 static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
124                                                       u32 asid)
125 {
126         struct vhost_vdpa_as *as = asid_to_as(v, asid);
127
128         if (as)
129                 return as;
130
131         return vhost_vdpa_alloc_as(v, asid);
132 }
133
134 static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
135 {
136         struct vhost_vdpa_as *as = asid_to_as(v, asid);
137
138         if (!as)
139                 return -EINVAL;
140
141         hlist_del(&as->hash_link);
142         vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
143         kfree(as);
144
145         return 0;
146 }
147
148 static void handle_vq_kick(struct vhost_work *work)
149 {
150         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
151                                                   poll.work);
152         struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
153         const struct vdpa_config_ops *ops = v->vdpa->config;
154
155         ops->kick_vq(v->vdpa, vq - v->vqs);
156 }
157
158 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
159 {
160         struct vhost_virtqueue *vq = private;
161         struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
162
163         if (call_ctx)
164                 eventfd_signal(call_ctx, 1);
165
166         return IRQ_HANDLED;
167 }
168
169 static irqreturn_t vhost_vdpa_config_cb(void *private)
170 {
171         struct vhost_vdpa *v = private;
172         struct eventfd_ctx *config_ctx = v->config_ctx;
173
174         if (config_ctx)
175                 eventfd_signal(config_ctx, 1);
176
177         return IRQ_HANDLED;
178 }
179
180 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
181 {
182         struct vhost_virtqueue *vq = &v->vqs[qid];
183         const struct vdpa_config_ops *ops = v->vdpa->config;
184         struct vdpa_device *vdpa = v->vdpa;
185         int ret, irq;
186
187         if (!ops->get_vq_irq)
188                 return;
189
190         irq = ops->get_vq_irq(vdpa, qid);
191         if (irq < 0)
192                 return;
193
194         irq_bypass_unregister_producer(&vq->call_ctx.producer);
195         if (!vq->call_ctx.ctx)
196                 return;
197
198         vq->call_ctx.producer.token = vq->call_ctx.ctx;
199         vq->call_ctx.producer.irq = irq;
200         ret = irq_bypass_register_producer(&vq->call_ctx.producer);
201         if (unlikely(ret))
202                 dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",
203                          qid, vq->call_ctx.producer.token, ret);
204 }
205
206 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
207 {
208         struct vhost_virtqueue *vq = &v->vqs[qid];
209
210         irq_bypass_unregister_producer(&vq->call_ctx.producer);
211 }
212
213 static int vhost_vdpa_reset(struct vhost_vdpa *v)
214 {
215         struct vdpa_device *vdpa = v->vdpa;
216
217         v->in_batch = 0;
218
219         return vdpa_reset(vdpa);
220 }
221
222 static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
223 {
224         struct vdpa_device *vdpa = v->vdpa;
225         const struct vdpa_config_ops *ops = vdpa->config;
226
227         if (!vdpa->use_va || !ops->bind_mm)
228                 return 0;
229
230         return ops->bind_mm(vdpa, v->vdev.mm);
231 }
232
233 static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
234 {
235         struct vdpa_device *vdpa = v->vdpa;
236         const struct vdpa_config_ops *ops = vdpa->config;
237
238         if (!vdpa->use_va || !ops->unbind_mm)
239                 return;
240
241         ops->unbind_mm(vdpa);
242 }
243
244 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
245 {
246         struct vdpa_device *vdpa = v->vdpa;
247         const struct vdpa_config_ops *ops = vdpa->config;
248         u32 device_id;
249
250         device_id = ops->get_device_id(vdpa);
251
252         if (copy_to_user(argp, &device_id, sizeof(device_id)))
253                 return -EFAULT;
254
255         return 0;
256 }
257
258 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
259 {
260         struct vdpa_device *vdpa = v->vdpa;
261         const struct vdpa_config_ops *ops = vdpa->config;
262         u8 status;
263
264         status = ops->get_status(vdpa);
265
266         if (copy_to_user(statusp, &status, sizeof(status)))
267                 return -EFAULT;
268
269         return 0;
270 }
271
272 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
273 {
274         struct vdpa_device *vdpa = v->vdpa;
275         const struct vdpa_config_ops *ops = vdpa->config;
276         u8 status, status_old;
277         u32 nvqs = v->nvqs;
278         int ret;
279         u16 i;
280
281         if (copy_from_user(&status, statusp, sizeof(status)))
282                 return -EFAULT;
283
284         status_old = ops->get_status(vdpa);
285
286         /*
287          * Userspace shouldn't remove status bits unless reset the
288          * status to 0.
289          */
290         if (status != 0 && (status_old & ~status) != 0)
291                 return -EINVAL;
292
293         if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
294                 for (i = 0; i < nvqs; i++)
295                         vhost_vdpa_unsetup_vq_irq(v, i);
296
297         if (status == 0) {
298                 ret = vdpa_reset(vdpa);
299                 if (ret)
300                         return ret;
301         } else
302                 vdpa_set_status(vdpa, status);
303
304         if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
305                 for (i = 0; i < nvqs; i++)
306                         vhost_vdpa_setup_vq_irq(v, i);
307
308         return 0;
309 }
310
311 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
312                                       struct vhost_vdpa_config *c)
313 {
314         struct vdpa_device *vdpa = v->vdpa;
315         size_t size = vdpa->config->get_config_size(vdpa);
316
317         if (c->len == 0 || c->off > size)
318                 return -EINVAL;
319
320         if (c->len > size - c->off)
321                 return -E2BIG;
322
323         return 0;
324 }
325
326 static long vhost_vdpa_get_config(struct vhost_vdpa *v,
327                                   struct vhost_vdpa_config __user *c)
328 {
329         struct vdpa_device *vdpa = v->vdpa;
330         struct vhost_vdpa_config config;
331         unsigned long size = offsetof(struct vhost_vdpa_config, buf);
332         u8 *buf;
333
334         if (copy_from_user(&config, c, size))
335                 return -EFAULT;
336         if (vhost_vdpa_config_validate(v, &config))
337                 return -EINVAL;
338         buf = kvzalloc(config.len, GFP_KERNEL);
339         if (!buf)
340                 return -ENOMEM;
341
342         vdpa_get_config(vdpa, config.off, buf, config.len);
343
344         if (copy_to_user(c->buf, buf, config.len)) {
345                 kvfree(buf);
346                 return -EFAULT;
347         }
348
349         kvfree(buf);
350         return 0;
351 }
352
353 static long vhost_vdpa_set_config(struct vhost_vdpa *v,
354                                   struct vhost_vdpa_config __user *c)
355 {
356         struct vdpa_device *vdpa = v->vdpa;
357         struct vhost_vdpa_config config;
358         unsigned long size = offsetof(struct vhost_vdpa_config, buf);
359         u8 *buf;
360
361         if (copy_from_user(&config, c, size))
362                 return -EFAULT;
363         if (vhost_vdpa_config_validate(v, &config))
364                 return -EINVAL;
365
366         buf = vmemdup_user(c->buf, config.len);
367         if (IS_ERR(buf))
368                 return PTR_ERR(buf);
369
370         vdpa_set_config(vdpa, config.off, buf, config.len);
371
372         kvfree(buf);
373         return 0;
374 }
375
376 static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
377 {
378         struct vdpa_device *vdpa = v->vdpa;
379         const struct vdpa_config_ops *ops = vdpa->config;
380
381         return ops->suspend;
382 }
383
384 static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)
385 {
386         struct vdpa_device *vdpa = v->vdpa;
387         const struct vdpa_config_ops *ops = vdpa->config;
388
389         return ops->resume;
390 }
391
392 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
393 {
394         struct vdpa_device *vdpa = v->vdpa;
395         const struct vdpa_config_ops *ops = vdpa->config;
396         u64 features;
397
398         features = ops->get_device_features(vdpa);
399
400         if (copy_to_user(featurep, &features, sizeof(features)))
401                 return -EFAULT;
402
403         return 0;
404 }
405
406 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
407 {
408         struct vdpa_device *vdpa = v->vdpa;
409         const struct vdpa_config_ops *ops = vdpa->config;
410         struct vhost_dev *d = &v->vdev;
411         u64 actual_features;
412         u64 features;
413         int i;
414
415         /*
416          * It's not allowed to change the features after they have
417          * been negotiated.
418          */
419         if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
420                 return -EBUSY;
421
422         if (copy_from_user(&features, featurep, sizeof(features)))
423                 return -EFAULT;
424
425         if (vdpa_set_features(vdpa, features))
426                 return -EINVAL;
427
428         /* let the vqs know what has been configured */
429         actual_features = ops->get_driver_features(vdpa);
430         for (i = 0; i < d->nvqs; ++i) {
431                 struct vhost_virtqueue *vq = d->vqs[i];
432
433                 mutex_lock(&vq->mutex);
434                 vq->acked_features = actual_features;
435                 mutex_unlock(&vq->mutex);
436         }
437
438         return 0;
439 }
440
441 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
442 {
443         struct vdpa_device *vdpa = v->vdpa;
444         const struct vdpa_config_ops *ops = vdpa->config;
445         u16 num;
446
447         num = ops->get_vq_num_max(vdpa);
448
449         if (copy_to_user(argp, &num, sizeof(num)))
450                 return -EFAULT;
451
452         return 0;
453 }
454
455 static void vhost_vdpa_config_put(struct vhost_vdpa *v)
456 {
457         if (v->config_ctx) {
458                 eventfd_ctx_put(v->config_ctx);
459                 v->config_ctx = NULL;
460         }
461 }
462
463 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
464 {
465         struct vdpa_callback cb;
466         int fd;
467         struct eventfd_ctx *ctx;
468
469         cb.callback = vhost_vdpa_config_cb;
470         cb.private = v;
471         if (copy_from_user(&fd, argp, sizeof(fd)))
472                 return  -EFAULT;
473
474         ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
475         swap(ctx, v->config_ctx);
476
477         if (!IS_ERR_OR_NULL(ctx))
478                 eventfd_ctx_put(ctx);
479
480         if (IS_ERR(v->config_ctx)) {
481                 long ret = PTR_ERR(v->config_ctx);
482
483                 v->config_ctx = NULL;
484                 return ret;
485         }
486
487         v->vdpa->config->set_config_cb(v->vdpa, &cb);
488
489         return 0;
490 }
491
492 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
493 {
494         struct vhost_vdpa_iova_range range = {
495                 .first = v->range.first,
496                 .last = v->range.last,
497         };
498
499         if (copy_to_user(argp, &range, sizeof(range)))
500                 return -EFAULT;
501         return 0;
502 }
503
504 static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
505 {
506         struct vdpa_device *vdpa = v->vdpa;
507         const struct vdpa_config_ops *ops = vdpa->config;
508         u32 size;
509
510         size = ops->get_config_size(vdpa);
511
512         if (copy_to_user(argp, &size, sizeof(size)))
513                 return -EFAULT;
514
515         return 0;
516 }
517
518 static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
519 {
520         struct vdpa_device *vdpa = v->vdpa;
521
522         if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
523                 return -EFAULT;
524
525         return 0;
526 }
527
528 /* After a successful return of ioctl the device must not process more
529  * virtqueue descriptors. The device can answer to read or writes of config
530  * fields as if it were not suspended. In particular, writing to "queue_enable"
531  * with a value of 1 will not make the device start processing buffers.
532  */
533 static long vhost_vdpa_suspend(struct vhost_vdpa *v)
534 {
535         struct vdpa_device *vdpa = v->vdpa;
536         const struct vdpa_config_ops *ops = vdpa->config;
537
538         if (!ops->suspend)
539                 return -EOPNOTSUPP;
540
541         return ops->suspend(vdpa);
542 }
543
544 /* After a successful return of this ioctl the device resumes processing
545  * virtqueue descriptors. The device becomes fully operational the same way it
546  * was before it was suspended.
547  */
548 static long vhost_vdpa_resume(struct vhost_vdpa *v)
549 {
550         struct vdpa_device *vdpa = v->vdpa;
551         const struct vdpa_config_ops *ops = vdpa->config;
552
553         if (!ops->resume)
554                 return -EOPNOTSUPP;
555
556         return ops->resume(vdpa);
557 }
558
559 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
560                                    void __user *argp)
561 {
562         struct vdpa_device *vdpa = v->vdpa;
563         const struct vdpa_config_ops *ops = vdpa->config;
564         struct vdpa_vq_state vq_state;
565         struct vdpa_callback cb;
566         struct vhost_virtqueue *vq;
567         struct vhost_vring_state s;
568         u32 idx;
569         long r;
570
571         r = get_user(idx, (u32 __user *)argp);
572         if (r < 0)
573                 return r;
574
575         if (idx >= v->nvqs)
576                 return -ENOBUFS;
577
578         idx = array_index_nospec(idx, v->nvqs);
579         vq = &v->vqs[idx];
580
581         switch (cmd) {
582         case VHOST_VDPA_SET_VRING_ENABLE:
583                 if (copy_from_user(&s, argp, sizeof(s)))
584                         return -EFAULT;
585                 ops->set_vq_ready(vdpa, idx, s.num);
586                 return 0;
587         case VHOST_VDPA_GET_VRING_GROUP:
588                 if (!ops->get_vq_group)
589                         return -EOPNOTSUPP;
590                 s.index = idx;
591                 s.num = ops->get_vq_group(vdpa, idx);
592                 if (s.num >= vdpa->ngroups)
593                         return -EIO;
594                 else if (copy_to_user(argp, &s, sizeof(s)))
595                         return -EFAULT;
596                 return 0;
597         case VHOST_VDPA_SET_GROUP_ASID:
598                 if (copy_from_user(&s, argp, sizeof(s)))
599                         return -EFAULT;
600                 if (s.num >= vdpa->nas)
601                         return -EINVAL;
602                 if (!ops->set_group_asid)
603                         return -EOPNOTSUPP;
604                 return ops->set_group_asid(vdpa, idx, s.num);
605         case VHOST_GET_VRING_BASE:
606                 r = ops->get_vq_state(v->vdpa, idx, &vq_state);
607                 if (r)
608                         return r;
609
610                 if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
611                         vq->last_avail_idx = vq_state.packed.last_avail_idx |
612                                              (vq_state.packed.last_avail_counter << 15);
613                         vq->last_used_idx = vq_state.packed.last_used_idx |
614                                             (vq_state.packed.last_used_counter << 15);
615                 } else {
616                         vq->last_avail_idx = vq_state.split.avail_index;
617                 }
618                 break;
619         }
620
621         r = vhost_vring_ioctl(&v->vdev, cmd, argp);
622         if (r)
623                 return r;
624
625         switch (cmd) {
626         case VHOST_SET_VRING_ADDR:
627                 if (ops->set_vq_address(vdpa, idx,
628                                         (u64)(uintptr_t)vq->desc,
629                                         (u64)(uintptr_t)vq->avail,
630                                         (u64)(uintptr_t)vq->used))
631                         r = -EINVAL;
632                 break;
633
634         case VHOST_SET_VRING_BASE:
635                 if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
636                         vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff;
637                         vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000);
638                         vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff;
639                         vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000);
640                 } else {
641                         vq_state.split.avail_index = vq->last_avail_idx;
642                 }
643                 r = ops->set_vq_state(vdpa, idx, &vq_state);
644                 break;
645
646         case VHOST_SET_VRING_CALL:
647                 if (vq->call_ctx.ctx) {
648                         cb.callback = vhost_vdpa_virtqueue_cb;
649                         cb.private = vq;
650                         cb.trigger = vq->call_ctx.ctx;
651                 } else {
652                         cb.callback = NULL;
653                         cb.private = NULL;
654                         cb.trigger = NULL;
655                 }
656                 ops->set_vq_cb(vdpa, idx, &cb);
657                 vhost_vdpa_setup_vq_irq(v, idx);
658                 break;
659
660         case VHOST_SET_VRING_NUM:
661                 ops->set_vq_num(vdpa, idx, vq->num);
662                 break;
663         }
664
665         return r;
666 }
667
668 static long vhost_vdpa_unlocked_ioctl(struct file *filep,
669                                       unsigned int cmd, unsigned long arg)
670 {
671         struct vhost_vdpa *v = filep->private_data;
672         struct vhost_dev *d = &v->vdev;
673         void __user *argp = (void __user *)arg;
674         u64 __user *featurep = argp;
675         u64 features;
676         long r = 0;
677
678         if (cmd == VHOST_SET_BACKEND_FEATURES) {
679                 if (copy_from_user(&features, featurep, sizeof(features)))
680                         return -EFAULT;
681                 if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
682                                  BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
683                                  BIT_ULL(VHOST_BACKEND_F_RESUME)))
684                         return -EOPNOTSUPP;
685                 if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
686                      !vhost_vdpa_can_suspend(v))
687                         return -EOPNOTSUPP;
688                 if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) &&
689                      !vhost_vdpa_can_resume(v))
690                         return -EOPNOTSUPP;
691                 vhost_set_backend_features(&v->vdev, features);
692                 return 0;
693         }
694
695         mutex_lock(&d->mutex);
696
697         switch (cmd) {
698         case VHOST_VDPA_GET_DEVICE_ID:
699                 r = vhost_vdpa_get_device_id(v, argp);
700                 break;
701         case VHOST_VDPA_GET_STATUS:
702                 r = vhost_vdpa_get_status(v, argp);
703                 break;
704         case VHOST_VDPA_SET_STATUS:
705                 r = vhost_vdpa_set_status(v, argp);
706                 break;
707         case VHOST_VDPA_GET_CONFIG:
708                 r = vhost_vdpa_get_config(v, argp);
709                 break;
710         case VHOST_VDPA_SET_CONFIG:
711                 r = vhost_vdpa_set_config(v, argp);
712                 break;
713         case VHOST_GET_FEATURES:
714                 r = vhost_vdpa_get_features(v, argp);
715                 break;
716         case VHOST_SET_FEATURES:
717                 r = vhost_vdpa_set_features(v, argp);
718                 break;
719         case VHOST_VDPA_GET_VRING_NUM:
720                 r = vhost_vdpa_get_vring_num(v, argp);
721                 break;
722         case VHOST_VDPA_GET_GROUP_NUM:
723                 if (copy_to_user(argp, &v->vdpa->ngroups,
724                                  sizeof(v->vdpa->ngroups)))
725                         r = -EFAULT;
726                 break;
727         case VHOST_VDPA_GET_AS_NUM:
728                 if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
729                         r = -EFAULT;
730                 break;
731         case VHOST_SET_LOG_BASE:
732         case VHOST_SET_LOG_FD:
733                 r = -ENOIOCTLCMD;
734                 break;
735         case VHOST_VDPA_SET_CONFIG_CALL:
736                 r = vhost_vdpa_set_config_call(v, argp);
737                 break;
738         case VHOST_GET_BACKEND_FEATURES:
739                 features = VHOST_VDPA_BACKEND_FEATURES;
740                 if (vhost_vdpa_can_suspend(v))
741                         features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
742                 if (vhost_vdpa_can_resume(v))
743                         features |= BIT_ULL(VHOST_BACKEND_F_RESUME);
744                 if (copy_to_user(featurep, &features, sizeof(features)))
745                         r = -EFAULT;
746                 break;
747         case VHOST_VDPA_GET_IOVA_RANGE:
748                 r = vhost_vdpa_get_iova_range(v, argp);
749                 break;
750         case VHOST_VDPA_GET_CONFIG_SIZE:
751                 r = vhost_vdpa_get_config_size(v, argp);
752                 break;
753         case VHOST_VDPA_GET_VQS_COUNT:
754                 r = vhost_vdpa_get_vqs_count(v, argp);
755                 break;
756         case VHOST_VDPA_SUSPEND:
757                 r = vhost_vdpa_suspend(v);
758                 break;
759         case VHOST_VDPA_RESUME:
760                 r = vhost_vdpa_resume(v);
761                 break;
762         default:
763                 r = vhost_dev_ioctl(&v->vdev, cmd, argp);
764                 if (r == -ENOIOCTLCMD)
765                         r = vhost_vdpa_vring_ioctl(v, cmd, argp);
766                 break;
767         }
768
769         if (r)
770                 goto out;
771
772         switch (cmd) {
773         case VHOST_SET_OWNER:
774                 r = vhost_vdpa_bind_mm(v);
775                 if (r)
776                         vhost_dev_reset_owner(d, NULL);
777                 break;
778         }
779 out:
780         mutex_unlock(&d->mutex);
781         return r;
782 }
783 static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
784                                      struct vhost_iotlb_map *map, u32 asid)
785 {
786         struct vdpa_device *vdpa = v->vdpa;
787         const struct vdpa_config_ops *ops = vdpa->config;
788         if (ops->dma_map) {
789                 ops->dma_unmap(vdpa, asid, map->start, map->size);
790         } else if (ops->set_map == NULL) {
791                 iommu_unmap(v->domain, map->start, map->size);
792         }
793 }
794
795 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
796                                 u64 start, u64 last, u32 asid)
797 {
798         struct vhost_dev *dev = &v->vdev;
799         struct vhost_iotlb_map *map;
800         struct page *page;
801         unsigned long pfn, pinned;
802
803         while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
804                 pinned = PFN_DOWN(map->size);
805                 for (pfn = PFN_DOWN(map->addr);
806                      pinned > 0; pfn++, pinned--) {
807                         page = pfn_to_page(pfn);
808                         if (map->perm & VHOST_ACCESS_WO)
809                                 set_page_dirty_lock(page);
810                         unpin_user_page(page);
811                 }
812                 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
813                 vhost_vdpa_general_unmap(v, map, asid);
814                 vhost_iotlb_map_free(iotlb, map);
815         }
816 }
817
818 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
819                                 u64 start, u64 last, u32 asid)
820 {
821         struct vhost_iotlb_map *map;
822         struct vdpa_map_file *map_file;
823
824         while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
825                 map_file = (struct vdpa_map_file *)map->opaque;
826                 fput(map_file->file);
827                 kfree(map_file);
828                 vhost_vdpa_general_unmap(v, map, asid);
829                 vhost_iotlb_map_free(iotlb, map);
830         }
831 }
832
833 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
834                                    struct vhost_iotlb *iotlb, u64 start,
835                                    u64 last, u32 asid)
836 {
837         struct vdpa_device *vdpa = v->vdpa;
838
839         if (vdpa->use_va)
840                 return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
841
842         return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
843 }
844
845 static int perm_to_iommu_flags(u32 perm)
846 {
847         int flags = 0;
848
849         switch (perm) {
850         case VHOST_ACCESS_WO:
851                 flags |= IOMMU_WRITE;
852                 break;
853         case VHOST_ACCESS_RO:
854                 flags |= IOMMU_READ;
855                 break;
856         case VHOST_ACCESS_RW:
857                 flags |= (IOMMU_WRITE | IOMMU_READ);
858                 break;
859         default:
860                 WARN(1, "invalidate vhost IOTLB permission\n");
861                 break;
862         }
863
864         return flags | IOMMU_CACHE;
865 }
866
867 static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
868                           u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
869 {
870         struct vhost_dev *dev = &v->vdev;
871         struct vdpa_device *vdpa = v->vdpa;
872         const struct vdpa_config_ops *ops = vdpa->config;
873         u32 asid = iotlb_to_asid(iotlb);
874         int r = 0;
875
876         r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
877                                       pa, perm, opaque);
878         if (r)
879                 return r;
880
881         if (ops->dma_map) {
882                 r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
883         } else if (ops->set_map) {
884                 if (!v->in_batch)
885                         r = ops->set_map(vdpa, asid, iotlb);
886         } else {
887                 r = iommu_map(v->domain, iova, pa, size,
888                               perm_to_iommu_flags(perm), GFP_KERNEL);
889         }
890         if (r) {
891                 vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
892                 return r;
893         }
894
895         if (!vdpa->use_va)
896                 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
897
898         return 0;
899 }
900
901 static void vhost_vdpa_unmap(struct vhost_vdpa *v,
902                              struct vhost_iotlb *iotlb,
903                              u64 iova, u64 size)
904 {
905         struct vdpa_device *vdpa = v->vdpa;
906         const struct vdpa_config_ops *ops = vdpa->config;
907         u32 asid = iotlb_to_asid(iotlb);
908
909         vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
910
911         if (ops->set_map) {
912                 if (!v->in_batch)
913                         ops->set_map(vdpa, asid, iotlb);
914         }
915
916 }
917
918 static int vhost_vdpa_va_map(struct vhost_vdpa *v,
919                              struct vhost_iotlb *iotlb,
920                              u64 iova, u64 size, u64 uaddr, u32 perm)
921 {
922         struct vhost_dev *dev = &v->vdev;
923         u64 offset, map_size, map_iova = iova;
924         struct vdpa_map_file *map_file;
925         struct vm_area_struct *vma;
926         int ret = 0;
927
928         mmap_read_lock(dev->mm);
929
930         while (size) {
931                 vma = find_vma(dev->mm, uaddr);
932                 if (!vma) {
933                         ret = -EINVAL;
934                         break;
935                 }
936                 map_size = min(size, vma->vm_end - uaddr);
937                 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
938                         !(vma->vm_flags & (VM_IO | VM_PFNMAP))))
939                         goto next;
940
941                 map_file = kzalloc(sizeof(*map_file), GFP_KERNEL);
942                 if (!map_file) {
943                         ret = -ENOMEM;
944                         break;
945                 }
946                 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
947                 map_file->offset = offset;
948                 map_file->file = get_file(vma->vm_file);
949                 ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
950                                      perm, map_file);
951                 if (ret) {
952                         fput(map_file->file);
953                         kfree(map_file);
954                         break;
955                 }
956 next:
957                 size -= map_size;
958                 uaddr += map_size;
959                 map_iova += map_size;
960         }
961         if (ret)
962                 vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
963
964         mmap_read_unlock(dev->mm);
965
966         return ret;
967 }
968
969 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
970                              struct vhost_iotlb *iotlb,
971                              u64 iova, u64 size, u64 uaddr, u32 perm)
972 {
973         struct vhost_dev *dev = &v->vdev;
974         struct page **page_list;
975         unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
976         unsigned int gup_flags = FOLL_LONGTERM;
977         unsigned long npages, cur_base, map_pfn, last_pfn = 0;
978         unsigned long lock_limit, sz2pin, nchunks, i;
979         u64 start = iova;
980         long pinned;
981         int ret = 0;
982
983         /* Limit the use of memory for bookkeeping */
984         page_list = (struct page **) __get_free_page(GFP_KERNEL);
985         if (!page_list)
986                 return -ENOMEM;
987
988         if (perm & VHOST_ACCESS_WO)
989                 gup_flags |= FOLL_WRITE;
990
991         npages = PFN_UP(size + (iova & ~PAGE_MASK));
992         if (!npages) {
993                 ret = -EINVAL;
994                 goto free;
995         }
996
997         mmap_read_lock(dev->mm);
998
999         lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1000         if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
1001                 ret = -ENOMEM;
1002                 goto unlock;
1003         }
1004
1005         cur_base = uaddr & PAGE_MASK;
1006         iova &= PAGE_MASK;
1007         nchunks = 0;
1008
1009         while (npages) {
1010                 sz2pin = min_t(unsigned long, npages, list_size);
1011                 pinned = pin_user_pages(cur_base, sz2pin,
1012                                         gup_flags, page_list);
1013                 if (sz2pin != pinned) {
1014                         if (pinned < 0) {
1015                                 ret = pinned;
1016                         } else {
1017                                 unpin_user_pages(page_list, pinned);
1018                                 ret = -ENOMEM;
1019                         }
1020                         goto out;
1021                 }
1022                 nchunks++;
1023
1024                 if (!last_pfn)
1025                         map_pfn = page_to_pfn(page_list[0]);
1026
1027                 for (i = 0; i < pinned; i++) {
1028                         unsigned long this_pfn = page_to_pfn(page_list[i]);
1029                         u64 csize;
1030
1031                         if (last_pfn && (this_pfn != last_pfn + 1)) {
1032                                 /* Pin a contiguous chunk of memory */
1033                                 csize = PFN_PHYS(last_pfn - map_pfn + 1);
1034                                 ret = vhost_vdpa_map(v, iotlb, iova, csize,
1035                                                      PFN_PHYS(map_pfn),
1036                                                      perm, NULL);
1037                                 if (ret) {
1038                                         /*
1039                                          * Unpin the pages that are left unmapped
1040                                          * from this point on in the current
1041                                          * page_list. The remaining outstanding
1042                                          * ones which may stride across several
1043                                          * chunks will be covered in the common
1044                                          * error path subsequently.
1045                                          */
1046                                         unpin_user_pages(&page_list[i],
1047                                                          pinned - i);
1048                                         goto out;
1049                                 }
1050
1051                                 map_pfn = this_pfn;
1052                                 iova += csize;
1053                                 nchunks = 0;
1054                         }
1055
1056                         last_pfn = this_pfn;
1057                 }
1058
1059                 cur_base += PFN_PHYS(pinned);
1060                 npages -= pinned;
1061         }
1062
1063         /* Pin the rest chunk */
1064         ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
1065                              PFN_PHYS(map_pfn), perm, NULL);
1066 out:
1067         if (ret) {
1068                 if (nchunks) {
1069                         unsigned long pfn;
1070
1071                         /*
1072                          * Unpin the outstanding pages which are yet to be
1073                          * mapped but haven't due to vdpa_map() or
1074                          * pin_user_pages() failure.
1075                          *
1076                          * Mapped pages are accounted in vdpa_map(), hence
1077                          * the corresponding unpinning will be handled by
1078                          * vdpa_unmap().
1079                          */
1080                         WARN_ON(!last_pfn);
1081                         for (pfn = map_pfn; pfn <= last_pfn; pfn++)
1082                                 unpin_user_page(pfn_to_page(pfn));
1083                 }
1084                 vhost_vdpa_unmap(v, iotlb, start, size);
1085         }
1086 unlock:
1087         mmap_read_unlock(dev->mm);
1088 free:
1089         free_page((unsigned long)page_list);
1090         return ret;
1091
1092 }
1093
1094 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1095                                            struct vhost_iotlb *iotlb,
1096                                            struct vhost_iotlb_msg *msg)
1097 {
1098         struct vdpa_device *vdpa = v->vdpa;
1099
1100         if (msg->iova < v->range.first || !msg->size ||
1101             msg->iova > U64_MAX - msg->size + 1 ||
1102             msg->iova + msg->size - 1 > v->range.last)
1103                 return -EINVAL;
1104
1105         if (vhost_iotlb_itree_first(iotlb, msg->iova,
1106                                     msg->iova + msg->size - 1))
1107                 return -EEXIST;
1108
1109         if (vdpa->use_va)
1110                 return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1111                                          msg->uaddr, msg->perm);
1112
1113         return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1114                                  msg->perm);
1115 }
1116
1117 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1118                                         struct vhost_iotlb_msg *msg)
1119 {
1120         struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1121         struct vdpa_device *vdpa = v->vdpa;
1122         const struct vdpa_config_ops *ops = vdpa->config;
1123         struct vhost_iotlb *iotlb = NULL;
1124         struct vhost_vdpa_as *as = NULL;
1125         int r = 0;
1126
1127         mutex_lock(&dev->mutex);
1128
1129         r = vhost_dev_check_owner(dev);
1130         if (r)
1131                 goto unlock;
1132
1133         if (msg->type == VHOST_IOTLB_UPDATE ||
1134             msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1135                 as = vhost_vdpa_find_alloc_as(v, asid);
1136                 if (!as) {
1137                         dev_err(&v->dev, "can't find and alloc asid %d\n",
1138                                 asid);
1139                         r = -EINVAL;
1140                         goto unlock;
1141                 }
1142                 iotlb = &as->iotlb;
1143         } else
1144                 iotlb = asid_to_iotlb(v, asid);
1145
1146         if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1147                 if (v->in_batch && v->batch_asid != asid) {
1148                         dev_info(&v->dev, "batch id %d asid %d\n",
1149                                  v->batch_asid, asid);
1150                 }
1151                 if (!iotlb)
1152                         dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1153                 r = -EINVAL;
1154                 goto unlock;
1155         }
1156
1157         switch (msg->type) {
1158         case VHOST_IOTLB_UPDATE:
1159                 r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1160                 break;
1161         case VHOST_IOTLB_INVALIDATE:
1162                 vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1163                 break;
1164         case VHOST_IOTLB_BATCH_BEGIN:
1165                 v->batch_asid = asid;
1166                 v->in_batch = true;
1167                 break;
1168         case VHOST_IOTLB_BATCH_END:
1169                 if (v->in_batch && ops->set_map)
1170                         ops->set_map(vdpa, asid, iotlb);
1171                 v->in_batch = false;
1172                 break;
1173         default:
1174                 r = -EINVAL;
1175                 break;
1176         }
1177 unlock:
1178         mutex_unlock(&dev->mutex);
1179
1180         return r;
1181 }
1182
1183 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1184                                          struct iov_iter *from)
1185 {
1186         struct file *file = iocb->ki_filp;
1187         struct vhost_vdpa *v = file->private_data;
1188         struct vhost_dev *dev = &v->vdev;
1189
1190         return vhost_chr_write_iter(dev, from);
1191 }
1192
1193 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1194 {
1195         struct vdpa_device *vdpa = v->vdpa;
1196         const struct vdpa_config_ops *ops = vdpa->config;
1197         struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1198         const struct bus_type *bus;
1199         int ret;
1200
1201         /* Device want to do DMA by itself */
1202         if (ops->set_map || ops->dma_map)
1203                 return 0;
1204
1205         bus = dma_dev->bus;
1206         if (!bus)
1207                 return -EFAULT;
1208
1209         if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) {
1210                 dev_warn_once(&v->dev,
1211                               "Failed to allocate domain, device is not IOMMU cache coherent capable\n");
1212                 return -ENOTSUPP;
1213         }
1214
1215         v->domain = iommu_domain_alloc(bus);
1216         if (!v->domain)
1217                 return -EIO;
1218
1219         ret = iommu_attach_device(v->domain, dma_dev);
1220         if (ret)
1221                 goto err_attach;
1222
1223         return 0;
1224
1225 err_attach:
1226         iommu_domain_free(v->domain);
1227         v->domain = NULL;
1228         return ret;
1229 }
1230
1231 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1232 {
1233         struct vdpa_device *vdpa = v->vdpa;
1234         struct device *dma_dev = vdpa_get_dma_dev(vdpa);
1235
1236         if (v->domain) {
1237                 iommu_detach_device(v->domain, dma_dev);
1238                 iommu_domain_free(v->domain);
1239         }
1240
1241         v->domain = NULL;
1242 }
1243
1244 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1245 {
1246         struct vdpa_iova_range *range = &v->range;
1247         struct vdpa_device *vdpa = v->vdpa;
1248         const struct vdpa_config_ops *ops = vdpa->config;
1249
1250         if (ops->get_iova_range) {
1251                 *range = ops->get_iova_range(vdpa);
1252         } else if (v->domain && v->domain->geometry.force_aperture) {
1253                 range->first = v->domain->geometry.aperture_start;
1254                 range->last = v->domain->geometry.aperture_end;
1255         } else {
1256                 range->first = 0;
1257                 range->last = ULLONG_MAX;
1258         }
1259 }
1260
1261 static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1262 {
1263         struct vhost_vdpa_as *as;
1264         u32 asid;
1265
1266         for (asid = 0; asid < v->vdpa->nas; asid++) {
1267                 as = asid_to_as(v, asid);
1268                 if (as)
1269                         vhost_vdpa_remove_as(v, asid);
1270         }
1271
1272         vhost_vdpa_free_domain(v);
1273         vhost_dev_cleanup(&v->vdev);
1274         kfree(v->vdev.vqs);
1275 }
1276
1277 static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1278 {
1279         struct vhost_vdpa *v;
1280         struct vhost_dev *dev;
1281         struct vhost_virtqueue **vqs;
1282         int r, opened;
1283         u32 i, nvqs;
1284
1285         v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1286
1287         opened = atomic_cmpxchg(&v->opened, 0, 1);
1288         if (opened)
1289                 return -EBUSY;
1290
1291         nvqs = v->nvqs;
1292         r = vhost_vdpa_reset(v);
1293         if (r)
1294                 goto err;
1295
1296         vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1297         if (!vqs) {
1298                 r = -ENOMEM;
1299                 goto err;
1300         }
1301
1302         dev = &v->vdev;
1303         for (i = 0; i < nvqs; i++) {
1304                 vqs[i] = &v->vqs[i];
1305                 vqs[i]->handle_kick = handle_vq_kick;
1306         }
1307         vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1308                        vhost_vdpa_process_iotlb_msg);
1309
1310         r = vhost_vdpa_alloc_domain(v);
1311         if (r)
1312                 goto err_alloc_domain;
1313
1314         vhost_vdpa_set_iova_range(v);
1315
1316         filep->private_data = v;
1317
1318         return 0;
1319
1320 err_alloc_domain:
1321         vhost_vdpa_cleanup(v);
1322 err:
1323         atomic_dec(&v->opened);
1324         return r;
1325 }
1326
1327 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1328 {
1329         u32 i;
1330
1331         for (i = 0; i < v->nvqs; i++)
1332                 vhost_vdpa_unsetup_vq_irq(v, i);
1333 }
1334
1335 static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1336 {
1337         struct vhost_vdpa *v = filep->private_data;
1338         struct vhost_dev *d = &v->vdev;
1339
1340         mutex_lock(&d->mutex);
1341         filep->private_data = NULL;
1342         vhost_vdpa_clean_irq(v);
1343         vhost_vdpa_reset(v);
1344         vhost_dev_stop(&v->vdev);
1345         vhost_vdpa_unbind_mm(v);
1346         vhost_vdpa_config_put(v);
1347         vhost_vdpa_cleanup(v);
1348         mutex_unlock(&d->mutex);
1349
1350         atomic_dec(&v->opened);
1351         complete(&v->completion);
1352
1353         return 0;
1354 }
1355
1356 #ifdef CONFIG_MMU
1357 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1358 {
1359         struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1360         struct vdpa_device *vdpa = v->vdpa;
1361         const struct vdpa_config_ops *ops = vdpa->config;
1362         struct vdpa_notification_area notify;
1363         struct vm_area_struct *vma = vmf->vma;
1364         u16 index = vma->vm_pgoff;
1365
1366         notify = ops->get_vq_notification(vdpa, index);
1367
1368         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1369         if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
1370                             PFN_DOWN(notify.addr), PAGE_SIZE,
1371                             vma->vm_page_prot))
1372                 return VM_FAULT_SIGBUS;
1373
1374         return VM_FAULT_NOPAGE;
1375 }
1376
1377 static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1378         .fault = vhost_vdpa_fault,
1379 };
1380
1381 static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1382 {
1383         struct vhost_vdpa *v = vma->vm_file->private_data;
1384         struct vdpa_device *vdpa = v->vdpa;
1385         const struct vdpa_config_ops *ops = vdpa->config;
1386         struct vdpa_notification_area notify;
1387         unsigned long index = vma->vm_pgoff;
1388
1389         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1390                 return -EINVAL;
1391         if ((vma->vm_flags & VM_SHARED) == 0)
1392                 return -EINVAL;
1393         if (vma->vm_flags & VM_READ)
1394                 return -EINVAL;
1395         if (index > 65535)
1396                 return -EINVAL;
1397         if (!ops->get_vq_notification)
1398                 return -ENOTSUPP;
1399
1400         /* To be safe and easily modelled by userspace, We only
1401          * support the doorbell which sits on the page boundary and
1402          * does not share the page with other registers.
1403          */
1404         notify = ops->get_vq_notification(vdpa, index);
1405         if (notify.addr & (PAGE_SIZE - 1))
1406                 return -EINVAL;
1407         if (vma->vm_end - vma->vm_start != notify.size)
1408                 return -ENOTSUPP;
1409
1410         vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1411         vma->vm_ops = &vhost_vdpa_vm_ops;
1412         return 0;
1413 }
1414 #endif /* CONFIG_MMU */
1415
1416 static const struct file_operations vhost_vdpa_fops = {
1417         .owner          = THIS_MODULE,
1418         .open           = vhost_vdpa_open,
1419         .release        = vhost_vdpa_release,
1420         .write_iter     = vhost_vdpa_chr_write_iter,
1421         .unlocked_ioctl = vhost_vdpa_unlocked_ioctl,
1422 #ifdef CONFIG_MMU
1423         .mmap           = vhost_vdpa_mmap,
1424 #endif /* CONFIG_MMU */
1425         .compat_ioctl   = compat_ptr_ioctl,
1426 };
1427
1428 static void vhost_vdpa_release_dev(struct device *device)
1429 {
1430         struct vhost_vdpa *v =
1431                container_of(device, struct vhost_vdpa, dev);
1432
1433         ida_simple_remove(&vhost_vdpa_ida, v->minor);
1434         kfree(v->vqs);
1435         kfree(v);
1436 }
1437
1438 static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1439 {
1440         const struct vdpa_config_ops *ops = vdpa->config;
1441         struct vhost_vdpa *v;
1442         int minor;
1443         int i, r;
1444
1445         /* We can't support platform IOMMU device with more than 1
1446          * group or as
1447          */
1448         if (!ops->set_map && !ops->dma_map &&
1449             (vdpa->ngroups > 1 || vdpa->nas > 1))
1450                 return -EOPNOTSUPP;
1451
1452         v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1453         if (!v)
1454                 return -ENOMEM;
1455
1456         minor = ida_simple_get(&vhost_vdpa_ida, 0,
1457                                VHOST_VDPA_DEV_MAX, GFP_KERNEL);
1458         if (minor < 0) {
1459                 kfree(v);
1460                 return minor;
1461         }
1462
1463         atomic_set(&v->opened, 0);
1464         v->minor = minor;
1465         v->vdpa = vdpa;
1466         v->nvqs = vdpa->nvqs;
1467         v->virtio_id = ops->get_device_id(vdpa);
1468
1469         device_initialize(&v->dev);
1470         v->dev.release = vhost_vdpa_release_dev;
1471         v->dev.parent = &vdpa->dev;
1472         v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1473         v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),
1474                                GFP_KERNEL);
1475         if (!v->vqs) {
1476                 r = -ENOMEM;
1477                 goto err;
1478         }
1479
1480         r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1481         if (r)
1482                 goto err;
1483
1484         cdev_init(&v->cdev, &vhost_vdpa_fops);
1485         v->cdev.owner = THIS_MODULE;
1486
1487         r = cdev_device_add(&v->cdev, &v->dev);
1488         if (r)
1489                 goto err;
1490
1491         init_completion(&v->completion);
1492         vdpa_set_drvdata(vdpa, v);
1493
1494         for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1495                 INIT_HLIST_HEAD(&v->as[i]);
1496
1497         return 0;
1498
1499 err:
1500         put_device(&v->dev);
1501         ida_simple_remove(&vhost_vdpa_ida, v->minor);
1502         return r;
1503 }
1504
1505 static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1506 {
1507         struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1508         int opened;
1509
1510         cdev_device_del(&v->cdev, &v->dev);
1511
1512         do {
1513                 opened = atomic_cmpxchg(&v->opened, 0, 1);
1514                 if (!opened)
1515                         break;
1516                 wait_for_completion(&v->completion);
1517         } while (1);
1518
1519         put_device(&v->dev);
1520 }
1521
1522 static struct vdpa_driver vhost_vdpa_driver = {
1523         .driver = {
1524                 .name   = "vhost_vdpa",
1525         },
1526         .probe  = vhost_vdpa_probe,
1527         .remove = vhost_vdpa_remove,
1528 };
1529
1530 static int __init vhost_vdpa_init(void)
1531 {
1532         int r;
1533
1534         r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1535                                 "vhost-vdpa");
1536         if (r)
1537                 goto err_alloc_chrdev;
1538
1539         r = vdpa_register_driver(&vhost_vdpa_driver);
1540         if (r)
1541                 goto err_vdpa_register_driver;
1542
1543         return 0;
1544
1545 err_vdpa_register_driver:
1546         unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1547 err_alloc_chrdev:
1548         return r;
1549 }
1550 module_init(vhost_vdpa_init);
1551
1552 static void __exit vhost_vdpa_exit(void)
1553 {
1554         vdpa_unregister_driver(&vhost_vdpa_driver);
1555         unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1556 }
1557 module_exit(vhost_vdpa_exit);
1558
1559 MODULE_VERSION("0.0.1");
1560 MODULE_LICENSE("GPL v2");
1561 MODULE_AUTHOR("Intel Corporation");
1562 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");