1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
4 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 **************************************************************************/
29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
32 #include <drm/ttm/ttm_bo_driver.h>
33 #include <drm/ttm/ttm_placement.h>
34 #include <drm/drm_vma_manager.h>
35 #include <linux/dma-buf-map.h>
37 #include <linux/highmem.h>
38 #include <linux/wait.h>
39 #include <linux/slab.h>
40 #include <linux/vmalloc.h>
41 #include <linux/module.h>
42 #include <linux/dma-resv.h>
44 struct ttm_transfer_obj {
45 struct ttm_buffer_object base;
46 struct ttm_buffer_object *bo;
49 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
50 struct ttm_resource *mem)
52 if (mem->bus.offset || mem->bus.addr)
55 mem->bus.is_iomem = false;
56 if (!bdev->driver->io_mem_reserve)
59 return bdev->driver->io_mem_reserve(bdev, mem);
62 void ttm_mem_io_free(struct ttm_bo_device *bdev,
63 struct ttm_resource *mem)
65 if (!mem->bus.offset && !mem->bus.addr)
68 if (bdev->driver->io_mem_free)
69 bdev->driver->io_mem_free(bdev, mem);
75 static int ttm_resource_ioremap(struct ttm_bo_device *bdev,
76 struct ttm_resource *mem,
83 ret = ttm_mem_io_reserve(bdev, mem);
84 if (ret || !mem->bus.is_iomem)
90 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
92 if (mem->bus.caching == ttm_write_combined)
93 addr = ioremap_wc(mem->bus.offset, bus_size);
95 addr = ioremap(mem->bus.offset, bus_size);
97 ttm_mem_io_free(bdev, mem);
105 static void ttm_resource_iounmap(struct ttm_bo_device *bdev,
106 struct ttm_resource *mem,
109 if (virtual && mem->bus.addr == NULL)
111 ttm_mem_io_free(bdev, mem);
114 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
117 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
119 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
122 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
123 iowrite32(ioread32(srcP++), dstP++);
127 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
131 struct page *d = ttm->pages[page];
137 src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
138 dst = kmap_atomic_prot(d, prot);
142 memcpy_fromio(dst, src, PAGE_SIZE);
149 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
153 struct page *s = ttm->pages[page];
159 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
160 src = kmap_atomic_prot(s, prot);
164 memcpy_toio(dst, src, PAGE_SIZE);
171 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
172 struct ttm_operation_ctx *ctx,
173 struct ttm_resource *new_mem)
175 struct ttm_bo_device *bdev = bo->bdev;
176 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
177 struct ttm_tt *ttm = bo->ttm;
178 struct ttm_resource *old_mem = &bo->mem;
179 struct ttm_resource old_copy = *old_mem;
185 ret = ttm_bo_wait_ctx(bo, ctx);
189 ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap);
192 ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap);
197 * Single TTM move. NOP.
199 if (old_iomap == NULL && new_iomap == NULL)
203 * Don't move nonexistent data. Clear destination instead.
205 if (old_iomap == NULL &&
206 (ttm == NULL || (!ttm_tt_is_populated(ttm) &&
207 !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
208 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
213 * TTM might be null for moves within the same region.
216 ret = ttm_tt_populate(bdev, ttm, ctx);
221 for (i = 0; i < new_mem->num_pages; ++i) {
222 if (old_iomap == NULL) {
223 pgprot_t prot = ttm_io_prot(bo, old_mem, PAGE_KERNEL);
224 ret = ttm_copy_ttm_io_page(ttm, new_iomap, i,
226 } else if (new_iomap == NULL) {
227 pgprot_t prot = ttm_io_prot(bo, new_mem, PAGE_KERNEL);
228 ret = ttm_copy_io_ttm_page(ttm, old_iomap, i,
231 ret = ttm_copy_io_page(new_iomap, old_iomap, i);
240 ttm_bo_assign_mem(bo, new_mem);
243 ttm_bo_tt_destroy(bo);
246 ttm_resource_iounmap(bdev, old_mem, new_iomap);
248 ttm_resource_iounmap(bdev, &old_copy, old_iomap);
251 * On error, keep the mm node!
254 ttm_resource_free(bo, &old_copy);
257 EXPORT_SYMBOL(ttm_bo_move_memcpy);
259 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
261 struct ttm_transfer_obj *fbo;
263 fbo = container_of(bo, struct ttm_transfer_obj, base);
269 * ttm_buffer_object_transfer
271 * @bo: A pointer to a struct ttm_buffer_object.
272 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
273 * holding the data of @bo with the old placement.
275 * This is a utility function that may be called after an accelerated move
276 * has been scheduled. A new buffer object is created as a placeholder for
277 * the old data while it's being copied. When that buffer object is idle,
278 * it can be destroyed, releasing the space of the old placement.
283 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
284 struct ttm_buffer_object **new_obj)
286 struct ttm_transfer_obj *fbo;
289 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
299 * Fix up members that we shouldn't copy directly:
300 * TODO: Explicit member copy would probably be better here.
303 atomic_inc(&ttm_bo_glob.bo_count);
304 INIT_LIST_HEAD(&fbo->base.ddestroy);
305 INIT_LIST_HEAD(&fbo->base.lru);
306 INIT_LIST_HEAD(&fbo->base.swap);
307 fbo->base.moving = NULL;
308 drm_vma_node_reset(&fbo->base.base.vma_node);
310 kref_init(&fbo->base.kref);
311 fbo->base.destroy = &ttm_transfered_destroy;
312 fbo->base.acc_size = 0;
313 fbo->base.pin_count = 1;
314 if (bo->type != ttm_bo_type_sg)
315 fbo->base.base.resv = &fbo->base.base._resv;
317 dma_resv_init(&fbo->base.base._resv);
318 fbo->base.base.dev = NULL;
319 ret = dma_resv_trylock(&fbo->base.base._resv);
322 *new_obj = &fbo->base;
326 pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res,
329 struct ttm_resource_manager *man;
330 enum ttm_caching caching;
332 man = ttm_manager_type(bo->bdev, res->mem_type);
333 caching = man->use_tt ? bo->ttm->caching : res->bus.caching;
335 /* Cached mappings need no adjustment */
336 if (caching == ttm_cached)
339 #if defined(__i386__) || defined(__x86_64__)
340 if (caching == ttm_write_combined)
341 tmp = pgprot_writecombine(tmp);
342 else if (boot_cpu_data.x86 > 3)
343 tmp = pgprot_noncached(tmp);
345 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
346 defined(__powerpc__) || defined(__mips__)
347 if (caching == ttm_write_combined)
348 tmp = pgprot_writecombine(tmp);
350 tmp = pgprot_noncached(tmp);
352 #if defined(__sparc__)
353 tmp = pgprot_noncached(tmp);
357 EXPORT_SYMBOL(ttm_io_prot);
359 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
360 unsigned long offset,
362 struct ttm_bo_kmap_obj *map)
364 struct ttm_resource *mem = &bo->mem;
366 if (bo->mem.bus.addr) {
367 map->bo_kmap_type = ttm_bo_map_premapped;
368 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
370 map->bo_kmap_type = ttm_bo_map_iomap;
371 if (mem->bus.caching == ttm_write_combined)
372 map->virtual = ioremap_wc(bo->mem.bus.offset + offset,
375 map->virtual = ioremap(bo->mem.bus.offset + offset,
378 return (!map->virtual) ? -ENOMEM : 0;
381 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
382 unsigned long start_page,
383 unsigned long num_pages,
384 struct ttm_bo_kmap_obj *map)
386 struct ttm_resource *mem = &bo->mem;
387 struct ttm_operation_ctx ctx = {
388 .interruptible = false,
391 struct ttm_tt *ttm = bo->ttm;
397 ret = ttm_tt_populate(bo->bdev, ttm, &ctx);
401 if (num_pages == 1 && ttm->caching == ttm_cached) {
403 * We're mapping a single page, and the desired
404 * page protection is consistent with the bo.
407 map->bo_kmap_type = ttm_bo_map_kmap;
408 map->page = ttm->pages[start_page];
409 map->virtual = kmap(map->page);
412 * We need to use vmap to get the desired page protection
413 * or to make the buffer object look contiguous.
415 prot = ttm_io_prot(bo, mem, PAGE_KERNEL);
416 map->bo_kmap_type = ttm_bo_map_vmap;
417 map->virtual = vmap(ttm->pages + start_page, num_pages,
420 return (!map->virtual) ? -ENOMEM : 0;
423 int ttm_bo_kmap(struct ttm_buffer_object *bo,
424 unsigned long start_page, unsigned long num_pages,
425 struct ttm_bo_kmap_obj *map)
427 unsigned long offset, size;
432 if (num_pages > bo->num_pages)
434 if (start_page > bo->num_pages)
437 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
440 if (!bo->mem.bus.is_iomem) {
441 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
443 offset = start_page << PAGE_SHIFT;
444 size = num_pages << PAGE_SHIFT;
445 return ttm_bo_ioremap(bo, offset, size, map);
448 EXPORT_SYMBOL(ttm_bo_kmap);
450 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
454 switch (map->bo_kmap_type) {
455 case ttm_bo_map_iomap:
456 iounmap(map->virtual);
458 case ttm_bo_map_vmap:
459 vunmap(map->virtual);
461 case ttm_bo_map_kmap:
464 case ttm_bo_map_premapped:
469 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
473 EXPORT_SYMBOL(ttm_bo_kunmap);
475 int ttm_bo_vmap(struct ttm_buffer_object *bo, struct dma_buf_map *map)
477 struct ttm_resource *mem = &bo->mem;
480 ret = ttm_mem_io_reserve(bo->bdev, mem);
484 if (mem->bus.is_iomem) {
485 void __iomem *vaddr_iomem;
486 size_t size = bo->num_pages << PAGE_SHIFT;
489 vaddr_iomem = (void __iomem *)mem->bus.addr;
490 else if (mem->bus.caching == ttm_write_combined)
491 vaddr_iomem = ioremap_wc(mem->bus.offset, size);
493 vaddr_iomem = ioremap(mem->bus.offset, size);
498 dma_buf_map_set_vaddr_iomem(map, vaddr_iomem);
501 struct ttm_operation_ctx ctx = {
502 .interruptible = false,
505 struct ttm_tt *ttm = bo->ttm;
509 ret = ttm_tt_populate(bo->bdev, ttm, &ctx);
514 * We need to use vmap to get the desired page protection
515 * or to make the buffer object look contiguous.
517 prot = ttm_io_prot(bo, mem, PAGE_KERNEL);
518 vaddr = vmap(ttm->pages, bo->num_pages, 0, prot);
522 dma_buf_map_set_vaddr(map, vaddr);
527 EXPORT_SYMBOL(ttm_bo_vmap);
529 void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct dma_buf_map *map)
531 struct ttm_resource *mem = &bo->mem;
533 if (dma_buf_map_is_null(map))
538 else if (!mem->bus.addr)
539 iounmap(map->vaddr_iomem);
540 dma_buf_map_clear(map);
542 ttm_mem_io_free(bo->bdev, &bo->mem);
544 EXPORT_SYMBOL(ttm_bo_vunmap);
546 static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo,
550 ret = ttm_bo_wait(bo, false, false);
555 ttm_bo_tt_destroy(bo);
556 ttm_resource_free(bo, &bo->mem);
560 static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo,
561 struct dma_fence *fence,
564 struct ttm_buffer_object *ghost_obj;
568 * This should help pipeline ordinary buffer moves.
570 * Hang old buffer memory on a new buffer object,
571 * and leave it to be released when the GPU
572 * operation has completed.
575 dma_fence_put(bo->moving);
576 bo->moving = dma_fence_get(fence);
578 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
582 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
585 * If we're not moving to fixed memory, the TTM object
586 * needs to stay alive. Otherwhise hang it on the ghost
587 * bo to be unbound and destroyed.
591 ghost_obj->ttm = NULL;
595 dma_resv_unlock(&ghost_obj->base._resv);
596 ttm_bo_put(ghost_obj);
600 static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo,
601 struct dma_fence *fence)
603 struct ttm_bo_device *bdev = bo->bdev;
604 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type);
607 * BO doesn't have a TTM we need to bind/unbind. Just remember
608 * this eviction and free up the allocation
610 spin_lock(&from->move_lock);
611 if (!from->move || dma_fence_is_later(fence, from->move)) {
612 dma_fence_put(from->move);
613 from->move = dma_fence_get(fence);
615 spin_unlock(&from->move_lock);
617 ttm_resource_free(bo, &bo->mem);
619 dma_fence_put(bo->moving);
620 bo->moving = dma_fence_get(fence);
623 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
624 struct dma_fence *fence,
627 struct ttm_resource *new_mem)
629 struct ttm_bo_device *bdev = bo->bdev;
630 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type);
631 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
634 dma_resv_add_excl_fence(bo->base.resv, fence);
636 ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt);
637 else if (!from->use_tt && pipeline)
638 ttm_bo_move_pipeline_evict(bo, fence);
640 ret = ttm_bo_wait_free_node(bo, man->use_tt);
645 ttm_bo_assign_mem(bo, new_mem);
649 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
651 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
653 struct ttm_buffer_object *ghost;
656 ret = ttm_buffer_object_transfer(bo, &ghost);
660 ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);
661 /* Last resort, wait for the BO to be idle when we are OOM */
663 ttm_bo_wait(bo, false, false);
665 memset(&bo->mem, 0, sizeof(bo->mem));
666 bo->mem.mem_type = TTM_PL_SYSTEM;
669 dma_resv_unlock(&ghost->base._resv);