2 * Copyright © 2008,2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
25 * Chris Wilson <chris@chris-wilson.co.uk>
30 #include <drm/i915_drm.h>
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include <linux/dma_remapping.h>
36 #define __EXEC_OBJECT_HAS_PIN (1<<31)
37 #define __EXEC_OBJECT_HAS_FENCE (1<<30)
40 struct list_head vmas;
43 struct i915_vma *lut[0];
44 struct hlist_head buckets[0];
48 static struct eb_vmas *
49 eb_create(struct drm_i915_gem_execbuffer2 *args)
51 struct eb_vmas *eb = NULL;
53 if (args->flags & I915_EXEC_HANDLE_LUT) {
54 unsigned size = args->buffer_count;
55 size *= sizeof(struct i915_vma *);
56 size += sizeof(struct eb_vmas);
57 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
61 unsigned size = args->buffer_count;
62 unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
63 BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
64 while (count > 2*size)
66 eb = kzalloc(count*sizeof(struct hlist_head) +
67 sizeof(struct eb_vmas),
74 eb->and = -args->buffer_count;
76 INIT_LIST_HEAD(&eb->vmas);
81 eb_reset(struct eb_vmas *eb)
84 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
88 eb_lookup_vmas(struct eb_vmas *eb,
89 struct drm_i915_gem_exec_object2 *exec,
90 const struct drm_i915_gem_execbuffer2 *args,
91 struct i915_address_space *vm,
92 struct drm_file *file)
94 struct drm_i915_private *dev_priv = vm->dev->dev_private;
95 struct drm_i915_gem_object *obj;
96 struct list_head objects;
99 INIT_LIST_HEAD(&objects);
100 spin_lock(&file->table_lock);
101 /* Grab a reference to the object and release the lock so we can lookup
102 * or create the VMA without using GFP_ATOMIC */
103 for (i = 0; i < args->buffer_count; i++) {
104 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
106 spin_unlock(&file->table_lock);
107 DRM_DEBUG("Invalid object handle %d at index %d\n",
113 if (!list_empty(&obj->obj_exec_link)) {
114 spin_unlock(&file->table_lock);
115 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
116 obj, exec[i].handle, i);
121 drm_gem_object_reference(&obj->base);
122 list_add_tail(&obj->obj_exec_link, &objects);
124 spin_unlock(&file->table_lock);
127 list_for_each_entry(obj, &objects, obj_exec_link) {
128 struct i915_vma *vma;
129 struct i915_address_space *bind_vm = vm;
131 /* If we have secure dispatch, or the userspace assures us that
132 * they know what they're doing, use the GGTT VM.
134 if (exec[i].flags & EXEC_OBJECT_NEEDS_GTT ||
135 ((args->flags & I915_EXEC_SECURE) &&
136 (i == (args->buffer_count - 1))))
137 bind_vm = &dev_priv->gtt.base;
140 * NOTE: We can leak any vmas created here when something fails
141 * later on. But that's no issue since vma_unbind can deal with
142 * vmas which are not actually bound. And since only
143 * lookup_or_create exists as an interface to get at the vma
144 * from the (obj, vm) we don't run the risk of creating
145 * duplicated vmas for the same vm.
147 vma = i915_gem_obj_lookup_or_create_vma(obj, bind_vm);
149 DRM_DEBUG("Failed to lookup VMA\n");
154 list_add_tail(&vma->exec_list, &eb->vmas);
156 vma->exec_entry = &exec[i];
160 uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
161 vma->exec_handle = handle;
162 hlist_add_head(&vma->exec_node,
163 &eb->buckets[handle & eb->and]);
170 while (!list_empty(&objects)) {
171 obj = list_first_entry(&objects,
172 struct drm_i915_gem_object,
174 list_del_init(&obj->obj_exec_link);
176 drm_gem_object_unreference(&obj->base);
181 static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
184 if (handle >= -eb->and)
186 return eb->lut[handle];
188 struct hlist_head *head;
189 struct hlist_node *node;
191 head = &eb->buckets[handle & eb->and];
192 hlist_for_each(node, head) {
193 struct i915_vma *vma;
195 vma = hlist_entry(node, struct i915_vma, exec_node);
196 if (vma->exec_handle == handle)
204 i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
206 struct drm_i915_gem_exec_object2 *entry;
207 struct drm_i915_gem_object *obj = vma->obj;
209 if (!drm_mm_node_allocated(&vma->node))
212 entry = vma->exec_entry;
214 if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
215 i915_gem_object_unpin_fence(obj);
217 if (entry->flags & __EXEC_OBJECT_HAS_PIN)
220 entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
223 static void eb_destroy(struct eb_vmas *eb)
225 while (!list_empty(&eb->vmas)) {
226 struct i915_vma *vma;
228 vma = list_first_entry(&eb->vmas,
231 list_del_init(&vma->exec_list);
232 i915_gem_execbuffer_unreserve_vma(vma);
233 drm_gem_object_unreference(&vma->obj->base);
238 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
240 return (HAS_LLC(obj->base.dev) ||
241 obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
242 !obj->map_and_fenceable ||
243 obj->cache_level != I915_CACHE_NONE);
247 relocate_entry_cpu(struct drm_i915_gem_object *obj,
248 struct drm_i915_gem_relocation_entry *reloc)
250 struct drm_device *dev = obj->base.dev;
251 uint32_t page_offset = offset_in_page(reloc->offset);
255 ret = i915_gem_object_set_to_cpu_domain(obj, true);
259 vaddr = kmap_atomic(i915_gem_object_get_page(obj,
260 reloc->offset >> PAGE_SHIFT));
261 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
263 if (INTEL_INFO(dev)->gen >= 8) {
264 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
266 if (page_offset == 0) {
267 kunmap_atomic(vaddr);
268 vaddr = kmap_atomic(i915_gem_object_get_page(obj,
269 (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
272 *(uint32_t *)(vaddr + page_offset) = 0;
275 kunmap_atomic(vaddr);
281 relocate_entry_gtt(struct drm_i915_gem_object *obj,
282 struct drm_i915_gem_relocation_entry *reloc)
284 struct drm_device *dev = obj->base.dev;
285 struct drm_i915_private *dev_priv = dev->dev_private;
286 uint32_t __iomem *reloc_entry;
287 void __iomem *reloc_page;
290 ret = i915_gem_object_set_to_gtt_domain(obj, true);
294 ret = i915_gem_object_put_fence(obj);
298 /* Map the page containing the relocation we're going to perform. */
299 reloc->offset += i915_gem_obj_ggtt_offset(obj);
300 reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
301 reloc->offset & PAGE_MASK);
302 reloc_entry = (uint32_t __iomem *)
303 (reloc_page + offset_in_page(reloc->offset));
304 iowrite32(reloc->delta, reloc_entry);
306 if (INTEL_INFO(dev)->gen >= 8) {
309 if (offset_in_page(reloc->offset + sizeof(uint32_t)) == 0) {
310 io_mapping_unmap_atomic(reloc_page);
311 reloc_page = io_mapping_map_atomic_wc(
312 dev_priv->gtt.mappable,
313 reloc->offset + sizeof(uint32_t));
314 reloc_entry = reloc_page;
317 iowrite32(0, reloc_entry);
320 io_mapping_unmap_atomic(reloc_page);
326 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
328 struct drm_i915_gem_relocation_entry *reloc)
330 struct drm_device *dev = obj->base.dev;
331 struct drm_gem_object *target_obj;
332 struct drm_i915_gem_object *target_i915_obj;
333 struct i915_vma *target_vma;
334 uint32_t target_offset;
337 /* we've already hold a reference to all valid objects */
338 target_vma = eb_get_vma(eb, reloc->target_handle);
339 if (unlikely(target_vma == NULL))
341 target_i915_obj = target_vma->obj;
342 target_obj = &target_vma->obj->base;
344 target_offset = target_vma->node.start;
346 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
347 * pipe_control writes because the gpu doesn't properly redirect them
348 * through the ppgtt for non_secure batchbuffers. */
349 if (unlikely(IS_GEN6(dev) &&
350 reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
351 !target_i915_obj->has_global_gtt_mapping)) {
352 struct i915_vma *vma =
353 list_first_entry(&target_i915_obj->vma_list,
354 typeof(*vma), vma_link);
355 vma->bind_vma(vma, target_i915_obj->cache_level, GLOBAL_BIND);
358 /* Validate that the target is in a valid r/w GPU domain */
359 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
360 DRM_DEBUG("reloc with multiple write domains: "
361 "obj %p target %d offset %d "
362 "read %08x write %08x",
363 obj, reloc->target_handle,
366 reloc->write_domain);
369 if (unlikely((reloc->write_domain | reloc->read_domains)
370 & ~I915_GEM_GPU_DOMAINS)) {
371 DRM_DEBUG("reloc with read/write non-GPU domains: "
372 "obj %p target %d offset %d "
373 "read %08x write %08x",
374 obj, reloc->target_handle,
377 reloc->write_domain);
381 target_obj->pending_read_domains |= reloc->read_domains;
382 target_obj->pending_write_domain |= reloc->write_domain;
384 /* If the relocation already has the right value in it, no
385 * more work needs to be done.
387 if (target_offset == reloc->presumed_offset)
390 /* Check that the relocation address is valid... */
391 if (unlikely(reloc->offset >
392 obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
393 DRM_DEBUG("Relocation beyond object bounds: "
394 "obj %p target %d offset %d size %d.\n",
395 obj, reloc->target_handle,
397 (int) obj->base.size);
400 if (unlikely(reloc->offset & 3)) {
401 DRM_DEBUG("Relocation not 4-byte aligned: "
402 "obj %p target %d offset %d.\n",
403 obj, reloc->target_handle,
404 (int) reloc->offset);
408 /* We can't wait for rendering with pagefaults disabled */
409 if (obj->active && in_atomic())
412 reloc->delta += target_offset;
413 if (use_cpu_reloc(obj))
414 ret = relocate_entry_cpu(obj, reloc);
416 ret = relocate_entry_gtt(obj, reloc);
421 /* and update the user's relocation entry */
422 reloc->presumed_offset = target_offset;
428 i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
431 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
432 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
433 struct drm_i915_gem_relocation_entry __user *user_relocs;
434 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
437 user_relocs = to_user_ptr(entry->relocs_ptr);
439 remain = entry->relocation_count;
441 struct drm_i915_gem_relocation_entry *r = stack_reloc;
443 if (count > ARRAY_SIZE(stack_reloc))
444 count = ARRAY_SIZE(stack_reloc);
447 if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
451 u64 offset = r->presumed_offset;
453 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
457 if (r->presumed_offset != offset &&
458 __copy_to_user_inatomic(&user_relocs->presumed_offset,
460 sizeof(r->presumed_offset))) {
474 i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
476 struct drm_i915_gem_relocation_entry *relocs)
478 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
481 for (i = 0; i < entry->relocation_count; i++) {
482 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
491 i915_gem_execbuffer_relocate(struct eb_vmas *eb)
493 struct i915_vma *vma;
496 /* This is the fast path and we cannot handle a pagefault whilst
497 * holding the struct mutex lest the user pass in the relocations
498 * contained within a mmaped bo. For in such a case we, the page
499 * fault handler would call i915_gem_fault() and we would try to
500 * acquire the struct mutex again. Obviously this is bad and so
501 * lockdep complains vehemently.
504 list_for_each_entry(vma, &eb->vmas, exec_list) {
505 ret = i915_gem_execbuffer_relocate_vma(vma, eb);
515 need_reloc_mappable(struct i915_vma *vma)
517 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
518 return entry->relocation_count && !use_cpu_reloc(vma->obj) &&
519 i915_is_ggtt(vma->vm);
523 i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
524 struct intel_ring_buffer *ring,
527 struct drm_i915_gem_object *obj = vma->obj;
528 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
529 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
530 bool need_fence, need_mappable;
531 u32 flags = (entry->flags & EXEC_OBJECT_NEEDS_GTT) &&
532 !vma->obj->has_global_gtt_mapping ? GLOBAL_BIND : 0;
536 has_fenced_gpu_access &&
537 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
538 obj->tiling_mode != I915_TILING_NONE;
539 need_mappable = need_fence || need_reloc_mappable(vma);
541 ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, need_mappable,
546 entry->flags |= __EXEC_OBJECT_HAS_PIN;
548 if (has_fenced_gpu_access) {
549 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
550 ret = i915_gem_object_get_fence(obj);
554 if (i915_gem_object_pin_fence(obj))
555 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
557 obj->pending_fenced_gpu_access = true;
561 if (entry->offset != vma->node.start) {
562 entry->offset = vma->node.start;
566 if (entry->flags & EXEC_OBJECT_WRITE) {
567 obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
568 obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
571 vma->bind_vma(vma, obj->cache_level, flags);
577 i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
578 struct list_head *vmas,
581 struct drm_i915_gem_object *obj;
582 struct i915_vma *vma;
583 struct i915_address_space *vm;
584 struct list_head ordered_vmas;
585 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
588 if (list_empty(vmas))
591 vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
593 INIT_LIST_HEAD(&ordered_vmas);
594 while (!list_empty(vmas)) {
595 struct drm_i915_gem_exec_object2 *entry;
596 bool need_fence, need_mappable;
598 vma = list_first_entry(vmas, struct i915_vma, exec_list);
600 entry = vma->exec_entry;
603 has_fenced_gpu_access &&
604 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
605 obj->tiling_mode != I915_TILING_NONE;
606 need_mappable = need_fence || need_reloc_mappable(vma);
609 list_move(&vma->exec_list, &ordered_vmas);
611 list_move_tail(&vma->exec_list, &ordered_vmas);
613 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
614 obj->base.pending_write_domain = 0;
615 obj->pending_fenced_gpu_access = false;
617 list_splice(&ordered_vmas, vmas);
619 /* Attempt to pin all of the buffers into the GTT.
620 * This is done in 3 phases:
622 * 1a. Unbind all objects that do not match the GTT constraints for
623 * the execbuffer (fenceable, mappable, alignment etc).
624 * 1b. Increment pin count for already bound objects.
625 * 2. Bind new objects.
626 * 3. Decrement pin count.
628 * This avoid unnecessary unbinding of later objects in order to make
629 * room for the earlier objects *unless* we need to defragment.
635 /* Unbind any ill-fitting objects or pin. */
636 list_for_each_entry(vma, vmas, exec_list) {
637 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
638 bool need_fence, need_mappable;
642 if (!drm_mm_node_allocated(&vma->node))
646 has_fenced_gpu_access &&
647 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
648 obj->tiling_mode != I915_TILING_NONE;
649 need_mappable = need_fence || need_reloc_mappable(vma);
651 WARN_ON((need_mappable || need_fence) &&
652 !i915_is_ggtt(vma->vm));
654 if ((entry->alignment &&
655 vma->node.start & (entry->alignment - 1)) ||
656 (need_mappable && !obj->map_and_fenceable))
657 ret = i915_vma_unbind(vma);
659 ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
664 /* Bind fresh objects */
665 list_for_each_entry(vma, vmas, exec_list) {
666 if (drm_mm_node_allocated(&vma->node))
669 ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
675 if (ret != -ENOSPC || retry++)
678 /* Decrement pin count for bound objects */
679 list_for_each_entry(vma, vmas, exec_list)
680 i915_gem_execbuffer_unreserve_vma(vma);
682 ret = i915_gem_evict_vm(vm, true);
689 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
690 struct drm_i915_gem_execbuffer2 *args,
691 struct drm_file *file,
692 struct intel_ring_buffer *ring,
694 struct drm_i915_gem_exec_object2 *exec)
696 struct drm_i915_gem_relocation_entry *reloc;
697 struct i915_address_space *vm;
698 struct i915_vma *vma;
702 unsigned count = args->buffer_count;
704 if (WARN_ON(list_empty(&eb->vmas)))
707 vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
709 /* We may process another execbuffer during the unlock... */
710 while (!list_empty(&eb->vmas)) {
711 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
712 list_del_init(&vma->exec_list);
713 i915_gem_execbuffer_unreserve_vma(vma);
714 drm_gem_object_unreference(&vma->obj->base);
717 mutex_unlock(&dev->struct_mutex);
720 for (i = 0; i < count; i++)
721 total += exec[i].relocation_count;
723 reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
724 reloc = drm_malloc_ab(total, sizeof(*reloc));
725 if (reloc == NULL || reloc_offset == NULL) {
726 drm_free_large(reloc);
727 drm_free_large(reloc_offset);
728 mutex_lock(&dev->struct_mutex);
733 for (i = 0; i < count; i++) {
734 struct drm_i915_gem_relocation_entry __user *user_relocs;
735 u64 invalid_offset = (u64)-1;
738 user_relocs = to_user_ptr(exec[i].relocs_ptr);
740 if (copy_from_user(reloc+total, user_relocs,
741 exec[i].relocation_count * sizeof(*reloc))) {
743 mutex_lock(&dev->struct_mutex);
747 /* As we do not update the known relocation offsets after
748 * relocating (due to the complexities in lock handling),
749 * we need to mark them as invalid now so that we force the
750 * relocation processing next time. Just in case the target
751 * object is evicted and then rebound into its old
752 * presumed_offset before the next execbuffer - if that
753 * happened we would make the mistake of assuming that the
754 * relocations were valid.
756 for (j = 0; j < exec[i].relocation_count; j++) {
757 if (copy_to_user(&user_relocs[j].presumed_offset,
759 sizeof(invalid_offset))) {
761 mutex_lock(&dev->struct_mutex);
766 reloc_offset[i] = total;
767 total += exec[i].relocation_count;
770 ret = i915_mutex_lock_interruptible(dev);
772 mutex_lock(&dev->struct_mutex);
776 /* reacquire the objects */
778 ret = eb_lookup_vmas(eb, exec, args, vm, file);
782 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
783 ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);
787 list_for_each_entry(vma, &eb->vmas, exec_list) {
788 int offset = vma->exec_entry - exec;
789 ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
790 reloc + reloc_offset[offset]);
795 /* Leave the user relocations as are, this is the painfully slow path,
796 * and we want to avoid the complication of dropping the lock whilst
797 * having buffers reserved in the aperture and so causing spurious
798 * ENOSPC for random operations.
802 drm_free_large(reloc);
803 drm_free_large(reloc_offset);
808 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
809 struct list_head *vmas)
811 struct i915_vma *vma;
812 uint32_t flush_domains = 0;
813 bool flush_chipset = false;
816 list_for_each_entry(vma, vmas, exec_list) {
817 struct drm_i915_gem_object *obj = vma->obj;
818 ret = i915_gem_object_sync(obj, ring);
822 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
823 flush_chipset |= i915_gem_clflush_object(obj, false);
825 flush_domains |= obj->base.write_domain;
829 i915_gem_chipset_flush(ring->dev);
831 if (flush_domains & I915_GEM_DOMAIN_GTT)
834 /* Unconditionally invalidate gpu caches and ensure that we do flush
835 * any residual writes from the previous batch.
837 return intel_ring_invalidate_all_caches(ring);
841 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
843 if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
846 return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
850 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
854 unsigned relocs_total = 0;
855 unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
857 for (i = 0; i < count; i++) {
858 char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
859 int length; /* limited by fault_in_pages_readable() */
861 if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS)
864 /* First check for malicious input causing overflow in
865 * the worst case where we need to allocate the entire
866 * relocation tree as a single array.
868 if (exec[i].relocation_count > relocs_max - relocs_total)
870 relocs_total += exec[i].relocation_count;
872 length = exec[i].relocation_count *
873 sizeof(struct drm_i915_gem_relocation_entry);
875 * We must check that the entire relocation array is safe
876 * to read, but since we may need to update the presumed
877 * offsets during execution, check for full write access.
879 if (!access_ok(VERIFY_WRITE, ptr, length))
882 if (likely(!i915_prefault_disable)) {
883 if (fault_in_multipages_readable(ptr, length))
891 static struct i915_hw_context *
892 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
895 struct i915_hw_context *ctx = NULL;
896 struct i915_ctx_hang_stats *hs;
898 ctx = i915_gem_context_get(file->driver_priv, ctx_id);
899 if (IS_ERR_OR_NULL(ctx))
902 hs = &ctx->hang_stats;
904 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
905 return ERR_PTR(-EIO);
912 i915_gem_execbuffer_move_to_active(struct list_head *vmas,
913 struct intel_ring_buffer *ring)
915 struct i915_vma *vma;
917 list_for_each_entry(vma, vmas, exec_list) {
918 struct drm_i915_gem_object *obj = vma->obj;
919 u32 old_read = obj->base.read_domains;
920 u32 old_write = obj->base.write_domain;
922 obj->base.write_domain = obj->base.pending_write_domain;
923 if (obj->base.write_domain == 0)
924 obj->base.pending_read_domains |= obj->base.read_domains;
925 obj->base.read_domains = obj->base.pending_read_domains;
926 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
928 i915_vma_move_to_active(vma, ring);
929 if (obj->base.write_domain) {
931 obj->last_write_seqno = intel_ring_get_seqno(ring);
932 /* check for potential scanout */
933 if (i915_gem_obj_ggtt_bound(obj) &&
934 i915_gem_obj_to_ggtt(obj)->pin_count)
935 intel_mark_fb_busy(obj, ring);
938 trace_i915_gem_object_change_domain(obj, old_read, old_write);
943 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
944 struct drm_file *file,
945 struct intel_ring_buffer *ring,
946 struct drm_i915_gem_object *obj)
948 /* Unconditionally force add_request to emit a full flush. */
949 ring->gpu_caches_dirty = true;
951 /* Add a breadcrumb for the completion of the batch buffer */
952 (void)__i915_add_request(ring, file, obj, NULL);
956 i915_reset_gen7_sol_offsets(struct drm_device *dev,
957 struct intel_ring_buffer *ring)
959 drm_i915_private_t *dev_priv = dev->dev_private;
962 if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
965 ret = intel_ring_begin(ring, 4 * 3);
969 for (i = 0; i < 4; i++) {
970 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
971 intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
972 intel_ring_emit(ring, 0);
975 intel_ring_advance(ring);
981 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
982 struct drm_file *file,
983 struct drm_i915_gem_execbuffer2 *args,
984 struct drm_i915_gem_exec_object2 *exec)
986 drm_i915_private_t *dev_priv = dev->dev_private;
988 struct drm_i915_gem_object *batch_obj;
989 struct drm_clip_rect *cliprects = NULL;
990 struct intel_ring_buffer *ring;
991 struct i915_hw_context *ctx;
992 struct i915_address_space *vm;
993 const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
994 u32 exec_start = args->batch_start_offset, exec_len;
999 if (!i915_gem_check_execbuffer(args))
1002 ret = validate_exec_list(exec, args->buffer_count);
1007 if (args->flags & I915_EXEC_SECURE) {
1008 if (!file->is_master || !capable(CAP_SYS_ADMIN))
1011 flags |= I915_DISPATCH_SECURE;
1013 if (args->flags & I915_EXEC_IS_PINNED)
1014 flags |= I915_DISPATCH_PINNED;
1016 if ((args->flags & I915_EXEC_RING_MASK) > I915_NUM_RINGS) {
1017 DRM_DEBUG("execbuf with unknown ring: %d\n",
1018 (int)(args->flags & I915_EXEC_RING_MASK));
1022 if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
1023 ring = &dev_priv->ring[RCS];
1025 ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
1027 if (!intel_ring_initialized(ring)) {
1028 DRM_DEBUG("execbuf with invalid ring: %d\n",
1029 (int)(args->flags & I915_EXEC_RING_MASK));
1033 mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1034 mask = I915_EXEC_CONSTANTS_MASK;
1036 case I915_EXEC_CONSTANTS_REL_GENERAL:
1037 case I915_EXEC_CONSTANTS_ABSOLUTE:
1038 case I915_EXEC_CONSTANTS_REL_SURFACE:
1039 if (ring == &dev_priv->ring[RCS] &&
1040 mode != dev_priv->relative_constants_mode) {
1041 if (INTEL_INFO(dev)->gen < 4)
1044 if (INTEL_INFO(dev)->gen > 5 &&
1045 mode == I915_EXEC_CONSTANTS_REL_SURFACE)
1048 /* The HW changed the meaning on this bit on gen6 */
1049 if (INTEL_INFO(dev)->gen >= 6)
1050 mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
1054 DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
1058 if (args->buffer_count < 1) {
1059 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1063 if (args->num_cliprects != 0) {
1064 if (ring != &dev_priv->ring[RCS]) {
1065 DRM_DEBUG("clip rectangles are only valid with the render ring\n");
1069 if (INTEL_INFO(dev)->gen >= 5) {
1070 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
1074 if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
1075 DRM_DEBUG("execbuf with %u cliprects\n",
1076 args->num_cliprects);
1080 cliprects = kcalloc(args->num_cliprects,
1083 if (cliprects == NULL) {
1088 if (copy_from_user(cliprects,
1089 to_user_ptr(args->cliprects_ptr),
1090 sizeof(*cliprects)*args->num_cliprects)) {
1096 ret = i915_mutex_lock_interruptible(dev);
1100 if (dev_priv->ums.mm_suspended) {
1101 mutex_unlock(&dev->struct_mutex);
1106 ctx = i915_gem_validate_context(dev, file, ctx_id);
1107 if (IS_ERR_OR_NULL(ctx)) {
1108 mutex_unlock(&dev->struct_mutex);
1113 i915_gem_context_reference(ctx);
1116 if (!USES_FULL_PPGTT(dev))
1117 vm = &dev_priv->gtt.base;
1119 eb = eb_create(args);
1121 mutex_unlock(&dev->struct_mutex);
1126 /* Look up object handles */
1127 ret = eb_lookup_vmas(eb, exec, args, vm, file);
1131 /* take note of the batch buffer before we might reorder the lists */
1132 batch_obj = list_entry(eb->vmas.prev, struct i915_vma, exec_list)->obj;
1134 /* Move the objects en-masse into the GTT, evicting if necessary. */
1135 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1136 ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);
1140 /* The objects are in their final locations, apply the relocations. */
1142 ret = i915_gem_execbuffer_relocate(eb);
1144 if (ret == -EFAULT) {
1145 ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
1147 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1153 /* Set the pending read domains for the batch buffer to COMMAND */
1154 if (batch_obj->base.pending_write_domain) {
1155 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1159 batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1161 /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1162 * batch" bit. Hence we need to pin secure batches into the global gtt.
1163 * hsw should have this fixed, but bdw mucks it up again. */
1164 if (flags & I915_DISPATCH_SECURE &&
1165 !batch_obj->has_global_gtt_mapping) {
1166 /* When we have multiple VMs, we'll need to make sure that we
1167 * allocate space first */
1168 struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj);
1170 vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND);
1173 if (flags & I915_DISPATCH_SECURE)
1174 exec_start += i915_gem_obj_ggtt_offset(batch_obj);
1176 exec_start += i915_gem_obj_offset(batch_obj, vm);
1178 ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->vmas);
1182 ret = i915_switch_context(ring, file, ctx);
1186 if (ring == &dev_priv->ring[RCS] &&
1187 mode != dev_priv->relative_constants_mode) {
1188 ret = intel_ring_begin(ring, 4);
1192 intel_ring_emit(ring, MI_NOOP);
1193 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1194 intel_ring_emit(ring, INSTPM);
1195 intel_ring_emit(ring, mask << 16 | mode);
1196 intel_ring_advance(ring);
1198 dev_priv->relative_constants_mode = mode;
1201 if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1202 ret = i915_reset_gen7_sol_offsets(dev, ring);
1208 exec_len = args->batch_len;
1210 for (i = 0; i < args->num_cliprects; i++) {
1211 ret = i915_emit_box(dev, &cliprects[i],
1212 args->DR1, args->DR4);
1216 ret = ring->dispatch_execbuffer(ring,
1217 exec_start, exec_len,
1223 ret = ring->dispatch_execbuffer(ring,
1224 exec_start, exec_len,
1230 trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
1232 i915_gem_execbuffer_move_to_active(&eb->vmas, ring);
1233 i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
1236 /* the request owns the ref now */
1237 i915_gem_context_unreference(ctx);
1240 mutex_unlock(&dev->struct_mutex);
1248 * Legacy execbuffer just creates an exec2 list from the original exec object
1249 * list array and passes it to the real function.
1252 i915_gem_execbuffer(struct drm_device *dev, void *data,
1253 struct drm_file *file)
1255 struct drm_i915_gem_execbuffer *args = data;
1256 struct drm_i915_gem_execbuffer2 exec2;
1257 struct drm_i915_gem_exec_object *exec_list = NULL;
1258 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1261 if (args->buffer_count < 1) {
1262 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1266 /* Copy in the exec list from userland */
1267 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1268 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1269 if (exec_list == NULL || exec2_list == NULL) {
1270 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1271 args->buffer_count);
1272 drm_free_large(exec_list);
1273 drm_free_large(exec2_list);
1276 ret = copy_from_user(exec_list,
1277 to_user_ptr(args->buffers_ptr),
1278 sizeof(*exec_list) * args->buffer_count);
1280 DRM_DEBUG("copy %d exec entries failed %d\n",
1281 args->buffer_count, ret);
1282 drm_free_large(exec_list);
1283 drm_free_large(exec2_list);
1287 for (i = 0; i < args->buffer_count; i++) {
1288 exec2_list[i].handle = exec_list[i].handle;
1289 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1290 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1291 exec2_list[i].alignment = exec_list[i].alignment;
1292 exec2_list[i].offset = exec_list[i].offset;
1293 if (INTEL_INFO(dev)->gen < 4)
1294 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1296 exec2_list[i].flags = 0;
1299 exec2.buffers_ptr = args->buffers_ptr;
1300 exec2.buffer_count = args->buffer_count;
1301 exec2.batch_start_offset = args->batch_start_offset;
1302 exec2.batch_len = args->batch_len;
1303 exec2.DR1 = args->DR1;
1304 exec2.DR4 = args->DR4;
1305 exec2.num_cliprects = args->num_cliprects;
1306 exec2.cliprects_ptr = args->cliprects_ptr;
1307 exec2.flags = I915_EXEC_RENDER;
1308 i915_execbuffer2_set_context_id(exec2, 0);
1310 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1312 /* Copy the new buffer offsets back to the user's exec list. */
1313 for (i = 0; i < args->buffer_count; i++)
1314 exec_list[i].offset = exec2_list[i].offset;
1315 /* ... and back out to userspace */
1316 ret = copy_to_user(to_user_ptr(args->buffers_ptr),
1318 sizeof(*exec_list) * args->buffer_count);
1321 DRM_DEBUG("failed to copy %d exec entries "
1322 "back to user (%d)\n",
1323 args->buffer_count, ret);
1327 drm_free_large(exec_list);
1328 drm_free_large(exec2_list);
1333 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1334 struct drm_file *file)
1336 struct drm_i915_gem_execbuffer2 *args = data;
1337 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1340 if (args->buffer_count < 1 ||
1341 args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1342 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1346 exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
1347 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
1348 if (exec2_list == NULL)
1349 exec2_list = drm_malloc_ab(sizeof(*exec2_list),
1350 args->buffer_count);
1351 if (exec2_list == NULL) {
1352 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1353 args->buffer_count);
1356 ret = copy_from_user(exec2_list,
1357 to_user_ptr(args->buffers_ptr),
1358 sizeof(*exec2_list) * args->buffer_count);
1360 DRM_DEBUG("copy %d exec entries failed %d\n",
1361 args->buffer_count, ret);
1362 drm_free_large(exec2_list);
1366 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1368 /* Copy the new buffer offsets back to the user's exec list. */
1369 ret = copy_to_user(to_user_ptr(args->buffers_ptr),
1371 sizeof(*exec2_list) * args->buffer_count);
1374 DRM_DEBUG("failed to copy %d exec entries "
1375 "back to user (%d)\n",
1376 args->buffer_count, ret);
1380 drm_free_large(exec2_list);