2 * Copyright © 2012-2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <drm/i915_drm.h>
27 #include "i915_trace.h"
28 #include "intel_drv.h"
29 #include <linux/mmu_context.h>
30 #include <linux/mmu_notifier.h>
31 #include <linux/mempolicy.h>
32 #include <linux/swap.h>
33 #include <linux/sched/mm.h>
35 struct i915_mm_struct {
37 struct drm_i915_private *i915;
38 struct i915_mmu_notifier *mn;
39 struct hlist_node node;
41 struct work_struct work;
44 #if defined(CONFIG_MMU_NOTIFIER)
45 #include <linux/interval_tree.h>
47 struct i915_mmu_notifier {
49 struct hlist_node node;
50 struct mmu_notifier mn;
51 struct rb_root_cached objects;
52 struct i915_mm_struct *mm;
55 struct i915_mmu_object {
56 struct i915_mmu_notifier *mn;
57 struct drm_i915_gem_object *obj;
58 struct interval_tree_node it;
61 static void add_object(struct i915_mmu_object *mo)
63 GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb));
64 interval_tree_insert(&mo->it, &mo->mn->objects);
67 static void del_object(struct i915_mmu_object *mo)
69 if (RB_EMPTY_NODE(&mo->it.rb))
72 interval_tree_remove(&mo->it, &mo->mn->objects);
73 RB_CLEAR_NODE(&mo->it.rb);
77 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
79 struct i915_mmu_object *mo = obj->userptr.mmu_object;
82 * During mm_invalidate_range we need to cancel any userptr that
83 * overlaps the range being invalidated. Doing so requires the
84 * struct_mutex, and that risks recursion. In order to cause
85 * recursion, the user must alias the userptr address space with
86 * a GTT mmapping (possible with a MAP_FIXED) - then when we have
87 * to invalidate that mmaping, mm_invalidate_range is called with
88 * the userptr address *and* the struct_mutex held. To prevent that
89 * we set a flag under the i915_mmu_notifier spinlock to indicate
90 * whether this object is valid.
95 spin_lock(&mo->mn->lock);
100 spin_unlock(&mo->mn->lock);
104 userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
105 const struct mmu_notifier_range *range)
107 struct i915_mmu_notifier *mn =
108 container_of(_mn, struct i915_mmu_notifier, mn);
109 struct interval_tree_node *it;
110 struct mutex *unlock = NULL;
114 if (RB_EMPTY_ROOT(&mn->objects.rb_root))
117 /* interval ranges are inclusive, but invalidate range is exclusive */
118 end = range->end - 1;
120 spin_lock(&mn->lock);
121 it = interval_tree_iter_first(&mn->objects, range->start, end);
123 struct drm_i915_gem_object *obj;
125 if (!range->blockable) {
131 * The mmu_object is released late when destroying the
132 * GEM object so it is entirely possible to gain a
133 * reference on an object in the process of being freed
134 * since our serialisation is via the spinlock and not
135 * the struct_mutex - and consequently use it after it
136 * is freed and then double free it. To prevent that
137 * use-after-free we only acquire a reference on the
138 * object if it is not in the process of being destroyed.
140 obj = container_of(it, struct i915_mmu_object, it)->obj;
141 if (!kref_get_unless_zero(&obj->base.refcount)) {
142 it = interval_tree_iter_next(it, range->start, end);
145 spin_unlock(&mn->lock);
148 unlock = &mn->mm->i915->drm.struct_mutex;
150 switch (mutex_trylock_recursive(unlock)) {
152 case MUTEX_TRYLOCK_FAILED:
153 if (mutex_lock_killable_nested(unlock, I915_MM_SHRINKER)) {
154 i915_gem_object_put(obj);
158 case MUTEX_TRYLOCK_SUCCESS:
161 case MUTEX_TRYLOCK_RECURSIVE:
162 unlock = ERR_PTR(-EEXIST);
167 ret = i915_gem_object_unbind(obj);
169 ret = __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
170 i915_gem_object_put(obj);
174 spin_lock(&mn->lock);
177 * As we do not (yet) protect the mmu from concurrent insertion
178 * over this range, there is no guarantee that this search will
179 * terminate given a pathologic workload.
181 it = interval_tree_iter_first(&mn->objects, range->start, end);
183 spin_unlock(&mn->lock);
186 if (!IS_ERR_OR_NULL(unlock))
187 mutex_unlock(unlock);
193 static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
194 .invalidate_range_start = userptr_mn_invalidate_range_start,
197 static struct i915_mmu_notifier *
198 i915_mmu_notifier_create(struct i915_mm_struct *mm)
200 struct i915_mmu_notifier *mn;
202 mn = kmalloc(sizeof(*mn), GFP_KERNEL);
204 return ERR_PTR(-ENOMEM);
206 spin_lock_init(&mn->lock);
207 mn->mn.ops = &i915_gem_userptr_notifier;
208 mn->objects = RB_ROOT_CACHED;
215 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
217 struct i915_mmu_object *mo;
219 mo = fetch_and_zero(&obj->userptr.mmu_object);
223 spin_lock(&mo->mn->lock);
225 spin_unlock(&mo->mn->lock);
229 static struct i915_mmu_notifier *
230 i915_mmu_notifier_find(struct i915_mm_struct *mm)
232 struct i915_mmu_notifier *mn;
239 mn = i915_mmu_notifier_create(mm);
243 down_write(&mm->mm->mmap_sem);
244 mutex_lock(&mm->i915->mm_lock);
245 if (mm->mn == NULL && !err) {
246 /* Protected by mmap_sem (write-lock) */
247 err = __mmu_notifier_register(&mn->mn, mm->mm);
249 /* Protected by mm_lock */
250 mm->mn = fetch_and_zero(&mn);
254 * Someone else raced and successfully installed the mmu
255 * notifier, we can cancel our own errors.
259 mutex_unlock(&mm->i915->mm_lock);
260 up_write(&mm->mm->mmap_sem);
262 if (mn && !IS_ERR(mn))
265 return err ? ERR_PTR(err) : mm->mn;
269 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
272 struct i915_mmu_notifier *mn;
273 struct i915_mmu_object *mo;
275 if (flags & I915_USERPTR_UNSYNCHRONIZED)
276 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
278 if (WARN_ON(obj->userptr.mm == NULL))
281 mn = i915_mmu_notifier_find(obj->userptr.mm);
285 mo = kzalloc(sizeof(*mo), GFP_KERNEL);
291 mo->it.start = obj->userptr.ptr;
292 mo->it.last = obj->userptr.ptr + obj->base.size - 1;
293 RB_CLEAR_NODE(&mo->it.rb);
295 obj->userptr.mmu_object = mo;
300 i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
301 struct mm_struct *mm)
306 mmu_notifier_unregister(&mn->mn, mm);
313 __i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
318 i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
323 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
326 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0)
329 if (!capable(CAP_SYS_ADMIN))
336 i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
337 struct mm_struct *mm)
343 static struct i915_mm_struct *
344 __i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real)
346 struct i915_mm_struct *mm;
348 /* Protected by dev_priv->mm_lock */
349 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real)
357 i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)
359 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
360 struct i915_mm_struct *mm;
363 /* During release of the GEM object we hold the struct_mutex. This
364 * precludes us from calling mmput() at that time as that may be
365 * the last reference and so call exit_mmap(). exit_mmap() will
366 * attempt to reap the vma, and if we were holding a GTT mmap
367 * would then call drm_gem_vm_close() and attempt to reacquire
368 * the struct mutex. So in order to avoid that recursion, we have
369 * to defer releasing the mm reference until after we drop the
370 * struct_mutex, i.e. we need to schedule a worker to do the clean
373 mutex_lock(&dev_priv->mm_lock);
374 mm = __i915_mm_struct_find(dev_priv, current->mm);
376 mm = kmalloc(sizeof(*mm), GFP_KERNEL);
382 kref_init(&mm->kref);
383 mm->i915 = to_i915(obj->base.dev);
385 mm->mm = current->mm;
390 /* Protected by dev_priv->mm_lock */
391 hash_add(dev_priv->mm_structs,
392 &mm->node, (unsigned long)mm->mm);
396 obj->userptr.mm = mm;
398 mutex_unlock(&dev_priv->mm_lock);
403 __i915_mm_struct_free__worker(struct work_struct *work)
405 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work);
406 i915_mmu_notifier_free(mm->mn, mm->mm);
412 __i915_mm_struct_free(struct kref *kref)
414 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref);
416 /* Protected by dev_priv->mm_lock */
418 mutex_unlock(&mm->i915->mm_lock);
420 INIT_WORK(&mm->work, __i915_mm_struct_free__worker);
421 queue_work(mm->i915->mm.userptr_wq, &mm->work);
425 i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)
427 if (obj->userptr.mm == NULL)
430 kref_put_mutex(&obj->userptr.mm->kref,
431 __i915_mm_struct_free,
432 &to_i915(obj->base.dev)->mm_lock);
433 obj->userptr.mm = NULL;
436 struct get_pages_work {
437 struct work_struct work;
438 struct drm_i915_gem_object *obj;
439 struct task_struct *task;
442 static struct sg_table *
443 __i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
444 struct page **pvec, int num_pages)
446 unsigned int max_segment = i915_sg_segment_size();
448 unsigned int sg_page_sizes;
451 st = kmalloc(sizeof(*st), GFP_KERNEL);
453 return ERR_PTR(-ENOMEM);
456 ret = __sg_alloc_table_from_pages(st, pvec, num_pages,
457 0, num_pages << PAGE_SHIFT,
465 ret = i915_gem_gtt_prepare_pages(obj, st);
469 if (max_segment > PAGE_SIZE) {
470 max_segment = PAGE_SIZE;
478 sg_page_sizes = i915_sg_page_sizes(st->sgl);
480 __i915_gem_object_set_pages(obj, st, sg_page_sizes);
486 __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
488 struct get_pages_work *work = container_of(_work, typeof(*work), work);
489 struct drm_i915_gem_object *obj = work->obj;
490 const int npages = obj->base.size >> PAGE_SHIFT;
497 pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
499 struct mm_struct *mm = obj->userptr.mm->mm;
500 unsigned int flags = 0;
502 if (!i915_gem_object_is_readonly(obj))
506 if (mmget_not_zero(mm)) {
507 down_read(&mm->mmap_sem);
508 while (pinned < npages) {
509 ret = get_user_pages_remote
511 obj->userptr.ptr + pinned * PAGE_SIZE,
514 pvec + pinned, NULL, NULL);
520 up_read(&mm->mmap_sem);
525 mutex_lock(&obj->mm.lock);
526 if (obj->userptr.work == &work->work) {
527 struct sg_table *pages = ERR_PTR(ret);
529 if (pinned == npages) {
530 pages = __i915_gem_userptr_alloc_pages(obj, pvec,
532 if (!IS_ERR(pages)) {
538 obj->userptr.work = ERR_CAST(pages);
540 __i915_gem_userptr_set_active(obj, false);
542 mutex_unlock(&obj->mm.lock);
544 release_pages(pvec, pinned);
547 i915_gem_object_put(obj);
548 put_task_struct(work->task);
552 static struct sg_table *
553 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj)
555 struct get_pages_work *work;
557 /* Spawn a worker so that we can acquire the
558 * user pages without holding our mutex. Access
559 * to the user pages requires mmap_sem, and we have
560 * a strict lock ordering of mmap_sem, struct_mutex -
561 * we already hold struct_mutex here and so cannot
562 * call gup without encountering a lock inversion.
564 * Userspace will keep on repeating the operation
565 * (thanks to EAGAIN) until either we hit the fast
566 * path or the worker completes. If the worker is
567 * cancelled or superseded, the task is still run
568 * but the results ignored. (This leads to
569 * complications that we may have a stray object
570 * refcount that we need to be wary of when
571 * checking for existing objects during creation.)
572 * If the worker encounters an error, it reports
573 * that error back to this function through
574 * obj->userptr.work = ERR_PTR.
576 work = kmalloc(sizeof(*work), GFP_KERNEL);
578 return ERR_PTR(-ENOMEM);
580 obj->userptr.work = &work->work;
582 work->obj = i915_gem_object_get(obj);
584 work->task = current;
585 get_task_struct(work->task);
587 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
588 queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work);
590 return ERR_PTR(-EAGAIN);
593 static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
595 const int num_pages = obj->base.size >> PAGE_SHIFT;
596 struct mm_struct *mm = obj->userptr.mm->mm;
598 struct sg_table *pages;
602 /* If userspace should engineer that these pages are replaced in
603 * the vma between us binding this page into the GTT and completion
604 * of rendering... Their loss. If they change the mapping of their
605 * pages they need to create a new bo to point to the new vma.
607 * However, that still leaves open the possibility of the vma
608 * being copied upon fork. Which falls under the same userspace
609 * synchronisation issue as a regular bo, except that this time
610 * the process may not be expecting that a particular piece of
611 * memory is tied to the GPU.
613 * Fortunately, we can hook into the mmu_notifier in order to
614 * discard the page references prior to anything nasty happening
615 * to the vma (discard or cloning) which should prevent the more
616 * egregious cases from causing harm.
619 if (obj->userptr.work) {
620 /* active flag should still be held for the pending work */
621 if (IS_ERR(obj->userptr.work))
622 return PTR_ERR(obj->userptr.work);
630 if (mm == current->mm) {
631 pvec = kvmalloc_array(num_pages, sizeof(struct page *),
635 if (pvec) /* defer to worker if malloc fails */
636 pinned = __get_user_pages_fast(obj->userptr.ptr,
638 !i915_gem_object_is_readonly(obj),
644 pages = ERR_PTR(pinned);
646 } else if (pinned < num_pages) {
647 pages = __i915_gem_userptr_get_pages_schedule(obj);
648 active = pages == ERR_PTR(-EAGAIN);
650 pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages);
651 active = !IS_ERR(pages);
654 __i915_gem_userptr_set_active(obj, true);
657 release_pages(pvec, pinned);
660 return PTR_ERR_OR_ZERO(pages);
664 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
665 struct sg_table *pages)
667 struct sgt_iter sgt_iter;
670 /* Cancel any inflight work and force them to restart their gup */
671 obj->userptr.work = NULL;
672 __i915_gem_userptr_set_active(obj, false);
676 if (obj->mm.madv != I915_MADV_WILLNEED)
677 obj->mm.dirty = false;
679 i915_gem_gtt_finish_pages(obj, pages);
681 for_each_sgt_page(page, sgt_iter, pages) {
683 set_page_dirty(page);
685 mark_page_accessed(page);
688 obj->mm.dirty = false;
690 sg_free_table(pages);
695 i915_gem_userptr_release(struct drm_i915_gem_object *obj)
697 i915_gem_userptr_release__mmu_notifier(obj);
698 i915_gem_userptr_release__mm_struct(obj);
702 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
704 if (obj->userptr.mmu_object)
707 return i915_gem_userptr_init__mmu_notifier(obj, 0);
710 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
711 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
712 I915_GEM_OBJECT_IS_SHRINKABLE |
713 I915_GEM_OBJECT_ASYNC_CANCEL,
714 .get_pages = i915_gem_userptr_get_pages,
715 .put_pages = i915_gem_userptr_put_pages,
716 .dmabuf_export = i915_gem_userptr_dmabuf_export,
717 .release = i915_gem_userptr_release,
721 * Creates a new mm object that wraps some normal memory from the process
722 * context - user memory.
724 * We impose several restrictions upon the memory being mapped
726 * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
727 * 2. It must be normal system memory, not a pointer into another map of IO
728 * space (e.g. it must not be a GTT mmapping of another object).
729 * 3. We only allow a bo as large as we could in theory map into the GTT,
730 * that is we limit the size to the total size of the GTT.
731 * 4. The bo is marked as being snoopable. The backing pages are left
732 * accessible directly by the CPU, but reads and writes by the GPU may
733 * incur the cost of a snoop (unless you have an LLC architecture).
735 * Synchronisation between multiple users and the GPU is left to userspace
736 * through the normal set-domain-ioctl. The kernel will enforce that the
737 * GPU relinquishes the VMA before it is returned back to the system
738 * i.e. upon free(), munmap() or process termination. However, the userspace
739 * malloc() library may not immediately relinquish the VMA after free() and
740 * instead reuse it whilst the GPU is still reading and writing to the VMA.
743 * Also note, that the object created here is not currently a "first class"
744 * object, in that several ioctls are banned. These are the CPU access
745 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
746 * direct access via your pointer rather than use those ioctls. Another
747 * restriction is that we do not allow userptr surfaces to be pinned to the
748 * hardware and so we reject any attempt to create a framebuffer out of a
751 * If you think this is a good interface to use to pass GPU memory between
752 * drivers, please use dma-buf instead. In fact, wherever possible use
756 i915_gem_userptr_ioctl(struct drm_device *dev,
758 struct drm_file *file)
760 struct drm_i915_private *dev_priv = to_i915(dev);
761 struct drm_i915_gem_userptr *args = data;
762 struct drm_i915_gem_object *obj;
766 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
767 /* We cannot support coherent userptr objects on hw without
768 * LLC and broken snooping.
773 if (args->flags & ~(I915_USERPTR_READ_ONLY |
774 I915_USERPTR_UNSYNCHRONIZED))
777 if (!args->user_size)
780 if (offset_in_page(args->user_ptr | args->user_size))
783 if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
786 if (args->flags & I915_USERPTR_READ_ONLY) {
787 struct i915_hw_ppgtt *ppgtt;
790 * On almost all of the older hw, we cannot tell the GPU that
791 * a page is readonly.
793 ppgtt = dev_priv->kernel_context->ppgtt;
794 if (!ppgtt || !ppgtt->vm.has_read_only)
798 obj = i915_gem_object_alloc(dev_priv);
802 drm_gem_private_object_init(dev, &obj->base, args->user_size);
803 i915_gem_object_init(obj, &i915_gem_userptr_ops);
804 obj->read_domains = I915_GEM_DOMAIN_CPU;
805 obj->write_domain = I915_GEM_DOMAIN_CPU;
806 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
808 obj->userptr.ptr = args->user_ptr;
809 if (args->flags & I915_USERPTR_READ_ONLY)
810 i915_gem_object_set_readonly(obj);
812 /* And keep a pointer to the current->mm for resolving the user pages
813 * at binding. This means that we need to hook into the mmu_notifier
814 * in order to detect if the mmu is destroyed.
816 ret = i915_gem_userptr_init__mm_struct(obj);
818 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);
820 ret = drm_gem_handle_create(file, &obj->base, &handle);
822 /* drop reference from allocate - handle holds it now */
823 i915_gem_object_put(obj);
827 args->handle = handle;
831 int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
833 mutex_init(&dev_priv->mm_lock);
834 hash_init(dev_priv->mm_structs);
836 dev_priv->mm.userptr_wq =
837 alloc_workqueue("i915-userptr-acquire",
838 WQ_HIGHPRI | WQ_UNBOUND,
840 if (!dev_priv->mm.userptr_wq)
846 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
848 destroy_workqueue(dev_priv->mm.userptr_wq);