drivers/gpu/drm/ttm/ttm_resource.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/dma-buf-map.h>
  26 #include <linux/io-mapping.h>
  27 #include <linux/scatterlist.h>
  28
  29 #include <drm/ttm/ttm_resource.h>
  30 #include <drm/ttm/ttm_bo_driver.h>
  31
  32 /**
  33  * ttm_lru_bulk_move_init - initialize a bulk move structure
  34  * @bulk: the structure to init
  35  *
  36  * For now just memset the structure to zero.
  37  */
  38 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
  39 {
  40         memset(bulk, 0, sizeof(*bulk));
  41 }
  42 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
  43
  44 /**
  45  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
  46  *
  47  * @bulk: bulk move structure
  48  *
  49  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
  50  * resource order never changes. Should be called with &ttm_device.lru_lock held.
  51  */
  52 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
  53 {
  54         unsigned i;
  55
  56         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
  57                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
  58                 struct ttm_resource_manager *man;
  59
  60                 if (!pos->first)
  61                         continue;
  62
  63                 lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
  64                 dma_resv_assert_held(pos->first->bo->base.resv);
  65                 dma_resv_assert_held(pos->last->bo->base.resv);
  66
  67                 man = ttm_manager_type(pos->first->bo->bdev, TTM_PL_TT);
  68                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
  69                                     &pos->last->lru);
  70         }
  71
  72         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
  73                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
  74                 struct ttm_resource_manager *man;
  75
  76                 if (!pos->first)
  77                         continue;
  78
  79                 lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
  80                 dma_resv_assert_held(pos->first->bo->base.resv);
  81                 dma_resv_assert_held(pos->last->bo->base.resv);
  82
  83                 man = ttm_manager_type(pos->first->bo->bdev, TTM_PL_VRAM);
  84                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
  85                                     &pos->last->lru);
  86         }
  87 }
  88 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
  89
  90 /* Record a resource position in a bulk move structure */
  91 static void ttm_lru_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
  92                                       struct ttm_resource *res)
  93 {
  94         if (!pos->first)
  95                 pos->first = res;
  96         pos->last = res;
  97 }
  98
  99 /* Move a resource to the LRU tail and track the bulk position */
 100 void ttm_resource_move_to_lru_tail(struct ttm_resource *res,
 101                                    struct ttm_lru_bulk_move *bulk)
 102 {
 103         struct ttm_buffer_object *bo = res->bo;
 104         struct ttm_device *bdev = bo->bdev;
 105         struct ttm_resource_manager *man;
 106
 107         lockdep_assert_held(&bo->bdev->lru_lock);
 108
 109         if (bo->pin_count) {
 110                 list_move_tail(&res->lru, &bdev->pinned);
 111                 if (bdev->funcs->del_from_lru_notify)
 112                         bdev->funcs->del_from_lru_notify(res->bo);
 113                 return;
 114         }
 115
 116         man = ttm_manager_type(bdev, res->mem_type);
 117         list_move_tail(&res->lru, &man->lru[bo->priority]);
 118
 119         if (bdev->funcs->del_from_lru_notify)
 120                 bdev->funcs->del_from_lru_notify(bo);
 121
 122         if (!bulk)
 123                 return;
 124
 125         switch (res->mem_type) {
 126         case TTM_PL_TT:
 127                 ttm_lru_bulk_move_set_pos(&bulk->tt[bo->priority], res);
 128                 break;
 129
 130         case TTM_PL_VRAM:
 131                 ttm_lru_bulk_move_set_pos(&bulk->vram[bo->priority], res);
 132                 break;
 133         }
 134 }
 135
 136 /**
 137  * ttm_resource_init - resource object constructure
 138  * @bo: buffer object this resources is allocated for
 139  * @place: placement of the resource
 140  * @res: the resource object to inistilize
 141  *
 142  * Initialize a new resource object. Counterpart of ttm_resource_fini().
 143  */
 144 void ttm_resource_init(struct ttm_buffer_object *bo,
 145                        const struct ttm_place *place,
 146                        struct ttm_resource *res)
 147 {
 148         struct ttm_resource_manager *man;
 149
 150         res->start = 0;
 151         res->num_pages = PFN_UP(bo->base.size);
 152         res->mem_type = place->mem_type;
 153         res->placement = place->flags;
 154         res->bus.addr = NULL;
 155         res->bus.offset = 0;
 156         res->bus.is_iomem = false;
 157         res->bus.caching = ttm_cached;
 158         res->bo = bo;
 159         INIT_LIST_HEAD(&res->lru);
 160
 161         man = ttm_manager_type(bo->bdev, place->mem_type);
 162         spin_lock(&bo->bdev->lru_lock);
 163         man->usage += res->num_pages << PAGE_SHIFT;
 164         ttm_resource_move_to_lru_tail(res, NULL);
 165         spin_unlock(&bo->bdev->lru_lock);
 166 }
 167 EXPORT_SYMBOL(ttm_resource_init);
 168
 169 /**
 170  * ttm_resource_fini - resource destructor
 171  * @man: the resource manager this resource belongs to
 172  * @res: the resource to clean up
 173  *
 174  * Should be used by resource manager backends to clean up the TTM resource
 175  * objects before freeing the underlying structure. Makes sure the resource is
 176  * removed from the LRU before destruction.
 177  * Counterpart of ttm_resource_init().
 178  */
 179 void ttm_resource_fini(struct ttm_resource_manager *man,
 180                        struct ttm_resource *res)
 181 {
 182         struct ttm_device *bdev = man->bdev;
 183
 184         spin_lock(&bdev->lru_lock);
 185         list_del_init(&res->lru);
 186         if (res->bo && bdev->funcs->del_from_lru_notify)
 187                 bdev->funcs->del_from_lru_notify(res->bo);
 188         man->usage -= res->num_pages << PAGE_SHIFT;
 189         spin_unlock(&bdev->lru_lock);
 190 }
 191 EXPORT_SYMBOL(ttm_resource_fini);
 192
 193 int ttm_resource_alloc(struct ttm_buffer_object *bo,
 194                        const struct ttm_place *place,
 195                        struct ttm_resource **res_ptr)
 196 {
 197         struct ttm_resource_manager *man =
 198                 ttm_manager_type(bo->bdev, place->mem_type);
 199
 200         return man->func->alloc(man, bo, place, res_ptr);
 201 }
 202
 203 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
 204 {
 205         struct ttm_resource_manager *man;
 206
 207         if (!*res)
 208                 return;
 209
 210         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
 211         man->func->free(man, *res);
 212         *res = NULL;
 213 }
 214 EXPORT_SYMBOL(ttm_resource_free);
 215
 216 static bool ttm_resource_places_compat(struct ttm_resource *res,
 217                                        const struct ttm_place *places,
 218                                        unsigned num_placement)
 219 {
 220         unsigned i;
 221
 222         if (res->placement & TTM_PL_FLAG_TEMPORARY)
 223                 return false;
 224
 225         for (i = 0; i < num_placement; i++) {
 226                 const struct ttm_place *heap = &places[i];
 227
 228                 if (res->start < heap->fpfn || (heap->lpfn &&
 229                     (res->start + res->num_pages) > heap->lpfn))
 230                         continue;
 231
 232                 if ((res->mem_type == heap->mem_type) &&
 233                     (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
 234                      (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
 235                         return true;
 236         }
 237         return false;
 238 }
 239
 240 /**
 241  * ttm_resource_compat - check if resource is compatible with placement
 242  *
 243  * @res: the resource to check
 244  * @placement: the placement to check against
 245  *
 246  * Returns true if the placement is compatible.
 247  */
 248 bool ttm_resource_compat(struct ttm_resource *res,
 249                          struct ttm_placement *placement)
 250 {
 251         if (ttm_resource_places_compat(res, placement->placement,
 252                                        placement->num_placement))
 253                 return true;
 254
 255         if ((placement->busy_placement != placement->placement ||
 256              placement->num_busy_placement > placement->num_placement) &&
 257             ttm_resource_places_compat(res, placement->busy_placement,
 258                                        placement->num_busy_placement))
 259                 return true;
 260
 261         return false;
 262 }
 263 EXPORT_SYMBOL(ttm_resource_compat);
 264
 265 void ttm_resource_set_bo(struct ttm_resource *res,
 266                          struct ttm_buffer_object *bo)
 267 {
 268         spin_lock(&bo->bdev->lru_lock);
 269         res->bo = bo;
 270         spin_unlock(&bo->bdev->lru_lock);
 271 }
 272
 273 /**
 274  * ttm_resource_manager_init
 275  *
 276  * @man: memory manager object to init
 277  * @bdev: ttm device this manager belongs to
 278  * @size: size of managed resources in arbitrary units
 279  *
 280  * Initialise core parts of a manager object.
 281  */
 282 void ttm_resource_manager_init(struct ttm_resource_manager *man,
 283                                struct ttm_device *bdev,
 284                                uint64_t size)
 285 {
 286         unsigned i;
 287
 288         spin_lock_init(&man->move_lock);
 289         man->bdev = bdev;
 290         man->size = size;
 291         man->usage = 0;
 292
 293         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 294                 INIT_LIST_HEAD(&man->lru[i]);
 295         man->move = NULL;
 296 }
 297 EXPORT_SYMBOL(ttm_resource_manager_init);
 298
 299 /*
 300  * ttm_resource_manager_evict_all
 301  *
 302  * @bdev - device to use
 303  * @man - manager to use
 304  *
 305  * Evict all the objects out of a memory manager until it is empty.
 306  * Part of memory manager cleanup sequence.
 307  */
 308 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 309                                    struct ttm_resource_manager *man)
 310 {
 311         struct ttm_operation_ctx ctx = {
 312                 .interruptible = false,
 313                 .no_wait_gpu = false,
 314                 .force_alloc = true
 315         };
 316         struct dma_fence *fence;
 317         int ret;
 318         unsigned i;
 319
 320         /*
 321          * Can't use standard list traversal since we're unlocking.
 322          */
 323
 324         spin_lock(&bdev->lru_lock);
 325         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 326                 while (!list_empty(&man->lru[i])) {
 327                         spin_unlock(&bdev->lru_lock);
 328                         ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
 329                                                   NULL);
 330                         if (ret)
 331                                 return ret;
 332                         spin_lock(&bdev->lru_lock);
 333                 }
 334         }
 335         spin_unlock(&bdev->lru_lock);
 336
 337         spin_lock(&man->move_lock);
 338         fence = dma_fence_get(man->move);
 339         spin_unlock(&man->move_lock);
 340
 341         if (fence) {
 342                 ret = dma_fence_wait(fence, false);
 343                 dma_fence_put(fence);
 344                 if (ret)
 345                         return ret;
 346         }
 347
 348         return 0;
 349 }
 350 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
 351
 352 /**
 353  * ttm_resource_manager_usage
 354  *
 355  * @man: A memory manager object.
 356  *
 357  * Return how many resources are currently used.
 358  */
 359 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
 360 {
 361         uint64_t usage;
 362
 363         spin_lock(&man->bdev->lru_lock);
 364         usage = man->usage;
 365         spin_unlock(&man->bdev->lru_lock);
 366         return usage;
 367 }
 368 EXPORT_SYMBOL(ttm_resource_manager_usage);
 369
 370 /**
 371  * ttm_resource_manager_debug
 372  *
 373  * @man: manager type to dump.
 374  * @p: printer to use for debug.
 375  */
 376 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
 377                                 struct drm_printer *p)
 378 {
 379         drm_printf(p, "  use_type: %d\n", man->use_type);
 380         drm_printf(p, "  use_tt: %d\n", man->use_tt);
 381         drm_printf(p, "  size: %llu\n", man->size);
 382         drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
 383         if (man->func->debug)
 384                 man->func->debug(man, p);
 385 }
 386 EXPORT_SYMBOL(ttm_resource_manager_debug);
 387
 388 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
 389                                           struct dma_buf_map *dmap,
 390                                           pgoff_t i)
 391 {
 392         struct ttm_kmap_iter_iomap *iter_io =
 393                 container_of(iter, typeof(*iter_io), base);
 394         void __iomem *addr;
 395
 396 retry:
 397         while (i >= iter_io->cache.end) {
 398                 iter_io->cache.sg = iter_io->cache.sg ?
 399                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
 400                 iter_io->cache.i = iter_io->cache.end;
 401                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
 402                         PAGE_SHIFT;
 403                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
 404                         iter_io->start;
 405         }
 406
 407         if (i < iter_io->cache.i) {
 408                 iter_io->cache.end = 0;
 409                 iter_io->cache.sg = NULL;
 410                 goto retry;
 411         }
 412
 413         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
 414                                        (((resource_size_t)i - iter_io->cache.i)
 415                                         << PAGE_SHIFT));
 416         dma_buf_map_set_vaddr_iomem(dmap, addr);
 417 }
 418
 419 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
 420                                             struct dma_buf_map *map)
 421 {
 422         io_mapping_unmap_local(map->vaddr_iomem);
 423 }
 424
 425 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
 426         .map_local =  ttm_kmap_iter_iomap_map_local,
 427         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
 428         .maps_tt = false,
 429 };
 430
 431 /**
 432  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
 433  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
 434  * @iomap: The struct io_mapping representing the underlying linear io_memory.
 435  * @st: sg_table into @iomap, representing the memory of the struct
 436  * ttm_resource.
 437  * @start: Offset that needs to be subtracted from @st to make
 438  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 439  *
 440  * Return: Pointer to the embedded struct ttm_kmap_iter.
 441  */
 442 struct ttm_kmap_iter *
 443 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
 444                          struct io_mapping *iomap,
 445                          struct sg_table *st,
 446                          resource_size_t start)
 447 {
 448         iter_io->base.ops = &ttm_kmap_iter_io_ops;
 449         iter_io->iomap = iomap;
 450         iter_io->st = st;
 451         iter_io->start = start;
 452         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
 453
 454         return &iter_io->base;
 455 }
 456 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
 457
 458 /**
 459  * DOC: Linear io iterator
 460  *
 461  * This code should die in the not too near future. Best would be if we could
 462  * make io-mapping use memremap for all io memory, and have memremap
 463  * implement a kmap_local functionality. We could then strip a huge amount of
 464  * code. These linear io iterators are implemented to mimic old functionality,
 465  * and they don't use kmap_local semantics at all internally. Rather ioremap or
 466  * friends, and at least on 32-bit they add global TLB flushes and points
 467  * of failure.
 468  */
 469
 470 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
 471                                               struct dma_buf_map *dmap,
 472                                               pgoff_t i)
 473 {
 474         struct ttm_kmap_iter_linear_io *iter_io =
 475                 container_of(iter, typeof(*iter_io), base);
 476
 477         *dmap = iter_io->dmap;
 478         dma_buf_map_incr(dmap, i * PAGE_SIZE);
 479 }
 480
 481 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
 482         .map_local =  ttm_kmap_iter_linear_io_map_local,
 483         .maps_tt = false,
 484 };
 485
 486 /**
 487  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
 488  * @iter_io: The iterator to initialize
 489  * @bdev: The TTM device
 490  * @mem: The ttm resource representing the iomap.
 491  *
 492  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
 493  * pointing at a linear chunk of io memory.
 494  *
 495  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
 496  * failure.
 497  */
 498 struct ttm_kmap_iter *
 499 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 500                              struct ttm_device *bdev,
 501                              struct ttm_resource *mem)
 502 {
 503         int ret;
 504
 505         ret = ttm_mem_io_reserve(bdev, mem);
 506         if (ret)
 507                 goto out_err;
 508         if (!mem->bus.is_iomem) {
 509                 ret = -EINVAL;
 510                 goto out_io_free;
 511         }
 512
 513         if (mem->bus.addr) {
 514                 dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
 515                 iter_io->needs_unmap = false;
 516         } else {
 517                 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
 518
 519                 iter_io->needs_unmap = true;
 520                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
 521                 if (mem->bus.caching == ttm_write_combined)
 522                         dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
 523                                                     ioremap_wc(mem->bus.offset,
 524                                                                bus_size));
 525                 else if (mem->bus.caching == ttm_cached)
 526                         dma_buf_map_set_vaddr(&iter_io->dmap,
 527                                               memremap(mem->bus.offset, bus_size,
 528                                                        MEMREMAP_WB |
 529                                                        MEMREMAP_WT |
 530                                                        MEMREMAP_WC));
 531
 532                 /* If uncached requested or if mapping cached or wc failed */
 533                 if (dma_buf_map_is_null(&iter_io->dmap))
 534                         dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
 535                                                     ioremap(mem->bus.offset,
 536                                                             bus_size));
 537
 538                 if (dma_buf_map_is_null(&iter_io->dmap)) {
 539                         ret = -ENOMEM;
 540                         goto out_io_free;
 541                 }
 542         }
 543
 544         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
 545         return &iter_io->base;
 546
 547 out_io_free:
 548         ttm_mem_io_free(bdev, mem);
 549 out_err:
 550         return ERR_PTR(ret);
 551 }
 552
 553 /**
 554  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
 555  * @iter_io: The iterator to initialize
 556  * @bdev: The TTM device
 557  * @mem: The ttm resource representing the iomap.
 558  *
 559  * This function is for internal TTM use only. It cleans up a memcpy kmap
 560  * iterator initialized by ttm_kmap_iter_linear_io_init.
 561  */
 562 void
 563 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
 564                              struct ttm_device *bdev,
 565                              struct ttm_resource *mem)
 566 {
 567         if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
 568                 if (iter_io->dmap.is_iomem)
 569                         iounmap(iter_io->dmap.vaddr_iomem);
 570                 else
 571                         memunmap(iter_io->dmap.vaddr);
 572         }
 573
 574         ttm_mem_io_free(bdev, mem);
 575 }