2 * SPDX-License-Identifier: MIT
4 * Copyright © 2014-2016 Intel Corporation
8 #include "i915_gem_object.h"
9 #include "i915_scatterlist.h"
10 #include "i915_gem_lmem.h"
11 #include "i915_gem_mman.h"
13 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
14 struct sg_table *pages,
15 unsigned int sg_page_sizes)
17 struct drm_i915_private *i915 = to_i915(obj->base.dev);
18 unsigned long supported = INTEL_INFO(i915)->page_sizes;
21 lockdep_assert_held(&obj->mm.lock);
23 if (i915_gem_object_is_volatile(obj))
24 obj->mm.madv = I915_MADV_DONTNEED;
26 /* Make the pages coherent with the GPU (flushing any swapin). */
27 if (obj->cache_dirty) {
28 obj->write_domain = 0;
29 if (i915_gem_object_has_struct_page(obj))
30 drm_clflush_sg(pages);
31 obj->cache_dirty = false;
34 obj->mm.get_page.sg_pos = pages->sgl;
35 obj->mm.get_page.sg_idx = 0;
37 obj->mm.pages = pages;
39 if (i915_gem_object_is_tiled(obj) &&
40 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
41 GEM_BUG_ON(obj->mm.quirked);
42 __i915_gem_object_pin_pages(obj);
43 obj->mm.quirked = true;
46 GEM_BUG_ON(!sg_page_sizes);
47 obj->mm.page_sizes.phys = sg_page_sizes;
50 * Calculate the supported page-sizes which fit into the given
51 * sg_page_sizes. This will give us the page-sizes which we may be able
52 * to use opportunistically when later inserting into the GTT. For
53 * example if phys=2G, then in theory we should be able to use 1G, 2M,
54 * 64K or 4K pages, although in practice this will depend on a number of
57 obj->mm.page_sizes.sg = 0;
58 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
59 if (obj->mm.page_sizes.phys & ~0u << i)
60 obj->mm.page_sizes.sg |= BIT(i);
62 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
64 if (i915_gem_object_is_shrinkable(obj)) {
65 struct list_head *list;
68 spin_lock_irqsave(&i915->mm.obj_lock, flags);
70 i915->mm.shrink_count++;
71 i915->mm.shrink_memory += obj->base.size;
73 if (obj->mm.madv != I915_MADV_WILLNEED)
74 list = &i915->mm.purge_list;
76 list = &i915->mm.shrink_list;
77 list_add_tail(&obj->mm.link, list);
79 atomic_set(&obj->mm.shrink_pin, 0);
80 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
84 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
86 struct drm_i915_private *i915 = to_i915(obj->base.dev);
89 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
91 "Attempting to obtain a purgeable object\n");
95 err = obj->ops->get_pages(obj);
96 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
101 /* Ensure that the associated pages are gathered from the backing storage
102 * and pinned into our object. i915_gem_object_pin_pages() may be called
103 * multiple times before they are released by a single call to
104 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
105 * either as a result of memory pressure (reaping pages under the shrinker)
106 * or as the object is itself released.
108 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
112 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
116 if (unlikely(!i915_gem_object_has_pages(obj))) {
117 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
119 err = ____i915_gem_object_get_pages(obj);
123 smp_mb__before_atomic();
125 atomic_inc(&obj->mm.pages_pin_count);
128 mutex_unlock(&obj->mm.lock);
132 /* Immediately discard the backing storage */
133 void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
135 drm_gem_free_mmap_offset(&obj->base);
136 if (obj->ops->truncate)
137 obj->ops->truncate(obj);
140 /* Try to discard unwanted pages */
141 void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
143 lockdep_assert_held(&obj->mm.lock);
144 GEM_BUG_ON(i915_gem_object_has_pages(obj));
146 if (obj->ops->writeback)
147 obj->ops->writeback(obj);
150 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
152 struct radix_tree_iter iter;
156 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
157 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
161 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
163 if (is_vmalloc_addr(ptr))
166 kunmap(kmap_to_page(ptr));
170 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
172 struct sg_table *pages;
174 pages = fetch_and_zero(&obj->mm.pages);
175 if (IS_ERR_OR_NULL(pages))
178 if (i915_gem_object_is_volatile(obj))
179 obj->mm.madv = I915_MADV_WILLNEED;
181 i915_gem_object_make_unshrinkable(obj);
183 if (obj->mm.mapping) {
184 unmap_object(obj, page_mask_bits(obj->mm.mapping));
185 obj->mm.mapping = NULL;
188 __i915_gem_object_reset_page_iter(obj);
189 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
194 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
196 struct sg_table *pages;
199 if (i915_gem_object_has_pinned_pages(obj))
202 /* May be called by shrinker from within get_pages() (on another bo) */
203 mutex_lock(&obj->mm.lock);
204 if (unlikely(atomic_read(&obj->mm.pages_pin_count))) {
209 i915_gem_object_release_mmap_offset(obj);
212 * ->put_pages might need to allocate memory for the bit17 swizzle
213 * array, hence protect them from being reaped by removing them from gtt
216 pages = __i915_gem_object_unset_pages(obj);
219 * XXX Temporary hijinx to avoid updating all backends to handle
220 * NULL pages. In the future, when we have more asynchronous
221 * get_pages backends we should be better able to handle the
222 * cancellation of the async task in a more uniform manner.
224 if (!pages && !i915_gem_object_needs_async_cancel(obj))
225 pages = ERR_PTR(-EINVAL);
228 obj->ops->put_pages(obj, pages);
232 mutex_unlock(&obj->mm.lock);
237 static inline pte_t iomap_pte(resource_size_t base,
241 return pte_mkspecial(pfn_pte((base + offset) >> PAGE_SHIFT, prot));
244 /* The 'mapping' part of i915_gem_object_pin_map() below */
245 static void *i915_gem_object_map(struct drm_i915_gem_object *obj,
246 enum i915_map_type type)
248 unsigned long n_pte = obj->base.size >> PAGE_SHIFT;
249 struct sg_table *sgt = obj->mm.pages;
250 pte_t *stack[32], **mem;
251 struct vm_struct *area;
254 if (!i915_gem_object_has_struct_page(obj) && type != I915_MAP_WC)
257 /* A single page can always be kmapped */
258 if (n_pte == 1 && type == I915_MAP_WB)
259 return kmap(sg_page(sgt->sgl));
262 if (n_pte > ARRAY_SIZE(stack)) {
263 /* Too big for stack -- allocate temporary array instead */
264 mem = kvmalloc_array(n_pte, sizeof(*mem), GFP_KERNEL);
269 area = alloc_vm_area(obj->base.size, mem);
279 fallthrough; /* to use PAGE_KERNEL anyway */
281 pgprot = PAGE_KERNEL;
284 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
288 if (i915_gem_object_has_struct_page(obj)) {
289 struct sgt_iter iter;
293 for_each_sgt_page(page, iter, sgt)
294 **ptes++ = mk_pte(page, pgprot);
296 resource_size_t iomap;
297 struct sgt_iter iter;
301 iomap = obj->mm.region->iomap.base;
302 iomap -= obj->mm.region->region.start;
304 for_each_sgt_daddr(addr, iter, sgt)
305 **ptes++ = iomap_pte(iomap, addr, pgprot);
314 /* get, pin, and map the pages of the object into kernel space */
315 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
316 enum i915_map_type type)
318 enum i915_map_type has_type;
324 flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM;
325 if (!i915_gem_object_type_has(obj, flags))
326 return ERR_PTR(-ENXIO);
328 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
332 pinned = !(type & I915_MAP_OVERRIDE);
333 type &= ~I915_MAP_OVERRIDE;
335 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
336 if (unlikely(!i915_gem_object_has_pages(obj))) {
337 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
339 err = ____i915_gem_object_get_pages(obj);
343 smp_mb__before_atomic();
345 atomic_inc(&obj->mm.pages_pin_count);
348 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
350 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
351 if (ptr && has_type != type) {
357 unmap_object(obj, ptr);
359 ptr = obj->mm.mapping = NULL;
363 ptr = i915_gem_object_map(obj, type);
369 obj->mm.mapping = page_pack_bits(ptr, type);
373 mutex_unlock(&obj->mm.lock);
377 atomic_dec(&obj->mm.pages_pin_count);
383 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
384 unsigned long offset,
387 enum i915_map_type has_type;
390 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
391 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
392 offset, size, obj->base.size));
394 wmb(); /* let all previous writes be visible to coherent partners */
395 obj->mm.dirty = true;
397 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
400 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
401 if (has_type == I915_MAP_WC)
404 drm_clflush_virt_range(ptr + offset, size);
405 if (size == obj->base.size) {
406 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
407 obj->cache_dirty = false;
411 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
413 GEM_BUG_ON(!obj->mm.mapping);
416 * We allow removing the mapping from underneath pinned pages!
418 * Furthermore, since this is an unsafe operation reserved only
419 * for construction time manipulation, we ignore locking prudence.
421 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
423 i915_gem_object_unpin_map(obj);
427 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
429 unsigned int *offset)
431 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
432 struct scatterlist *sg;
433 unsigned int idx, count;
436 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
437 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
439 /* As we iterate forward through the sg, we record each entry in a
440 * radixtree for quick repeated (backwards) lookups. If we have seen
441 * this index previously, we will have an entry for it.
443 * Initial lookup is O(N), but this is amortized to O(1) for
444 * sequential page access (where each new request is consecutive
445 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
446 * i.e. O(1) with a large constant!
448 if (n < READ_ONCE(iter->sg_idx))
451 mutex_lock(&iter->lock);
453 /* We prefer to reuse the last sg so that repeated lookup of this
454 * (or the subsequent) sg are fast - comparing against the last
455 * sg is faster than going through the radixtree.
460 count = __sg_page_count(sg);
462 while (idx + count <= n) {
467 /* If we cannot allocate and insert this entry, or the
468 * individual pages from this range, cancel updating the
469 * sg_idx so that on this lookup we are forced to linearly
470 * scan onwards, but on future lookups we will try the
471 * insertion again (in which case we need to be careful of
472 * the error return reporting that we have already inserted
475 ret = radix_tree_insert(&iter->radix, idx, sg);
476 if (ret && ret != -EEXIST)
479 entry = xa_mk_value(idx);
480 for (i = 1; i < count; i++) {
481 ret = radix_tree_insert(&iter->radix, idx + i, entry);
482 if (ret && ret != -EEXIST)
487 sg = ____sg_next(sg);
488 count = __sg_page_count(sg);
495 mutex_unlock(&iter->lock);
497 if (unlikely(n < idx)) /* insertion completed by another thread */
500 /* In case we failed to insert the entry into the radixtree, we need
501 * to look beyond the current sg.
503 while (idx + count <= n) {
505 sg = ____sg_next(sg);
506 count = __sg_page_count(sg);
515 sg = radix_tree_lookup(&iter->radix, n);
518 /* If this index is in the middle of multi-page sg entry,
519 * the radix tree will contain a value entry that points
520 * to the start of that range. We will return the pointer to
521 * the base page and the offset of this page within the
525 if (unlikely(xa_is_value(sg))) {
526 unsigned long base = xa_to_value(sg);
528 sg = radix_tree_lookup(&iter->radix, base);
540 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
542 struct scatterlist *sg;
545 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
547 sg = i915_gem_object_get_sg(obj, n, &offset);
548 return nth_page(sg_page(sg), offset);
552 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
556 struct scatterlist *sg;
559 sg = i915_gem_object_get_sg(obj, n, &offset);
562 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
564 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
568 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
571 return i915_gem_object_get_dma_address_len(obj, n, NULL);