drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo_driver.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <linux/jiffies.h>
  37 #include <linux/slab.h>
  38 #include <linux/sched.h>
  39 #include <linux/mm.h>
  40 #include <linux/file.h>
  41 #include <linux/module.h>
  42 #include <linux/atomic.h>
  43 #include <linux/dma-resv.h>
  44
  45 #include "ttm_module.h"
  46
  47 /* default destructor */
  48 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  49 {
  50         kfree(bo);
  51 }
  52
  53 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  54                                         struct ttm_placement *placement)
  55 {
  56         struct drm_printer p = drm_debug_printer(TTM_PFX);
  57         struct ttm_resource_manager *man;
  58         int i, mem_type;
  59
  60         drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n",
  61                    bo, bo->mem.num_pages, bo->base.size >> 10,
  62                    bo->base.size >> 20);
  63         for (i = 0; i < placement->num_placement; i++) {
  64                 mem_type = placement->placement[i].mem_type;
  65                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  66                            i, placement->placement[i].flags, mem_type);
  67                 man = ttm_manager_type(bo->bdev, mem_type);
  68                 ttm_resource_manager_debug(man, &p);
  69         }
  70 }
  71
  72 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
  73 {
  74         struct ttm_device *bdev = bo->bdev;
  75
  76         list_del_init(&bo->swap);
  77         list_del_init(&bo->lru);
  78
  79         if (bdev->funcs->del_from_lru_notify)
  80                 bdev->funcs->del_from_lru_notify(bo);
  81 }
  82
  83 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
  84                                      struct ttm_buffer_object *bo)
  85 {
  86         if (!pos->first)
  87                 pos->first = bo;
  88         pos->last = bo;
  89 }
  90
  91 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
  92                              struct ttm_resource *mem,
  93                              struct ttm_lru_bulk_move *bulk)
  94 {
  95         struct ttm_device *bdev = bo->bdev;
  96         struct ttm_resource_manager *man;
  97
  98         if (!bo->deleted)
  99                 dma_resv_assert_held(bo->base.resv);
 100
 101         if (bo->pin_count) {
 102                 ttm_bo_del_from_lru(bo);
 103                 return;
 104         }
 105
 106         man = ttm_manager_type(bdev, mem->mem_type);
 107         list_move_tail(&bo->lru, &man->lru[bo->priority]);
 108         if (man->use_tt && bo->ttm &&
 109             !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
 110                                      TTM_PAGE_FLAG_SWAPPED))) {
 111                 struct list_head *swap;
 112
 113                 swap = &ttm_glob.swap_lru[bo->priority];
 114                 list_move_tail(&bo->swap, swap);
 115         } else {
 116                 list_del_init(&bo->swap);
 117         }
 118
 119         if (bdev->funcs->del_from_lru_notify)
 120                 bdev->funcs->del_from_lru_notify(bo);
 121
 122         if (bulk && !bo->pin_count) {
 123                 switch (bo->mem.mem_type) {
 124                 case TTM_PL_TT:
 125                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 126                         break;
 127
 128                 case TTM_PL_VRAM:
 129                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 130                         break;
 131                 }
 132                 if (bo->ttm && !(bo->ttm->page_flags &
 133                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 134                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 135         }
 136 }
 137 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 138
 139 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 140 {
 141         unsigned i;
 142
 143         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 144                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 145                 struct ttm_resource_manager *man;
 146
 147                 if (!pos->first)
 148                         continue;
 149
 150                 dma_resv_assert_held(pos->first->base.resv);
 151                 dma_resv_assert_held(pos->last->base.resv);
 152
 153                 man = ttm_manager_type(pos->first->bdev, TTM_PL_TT);
 154                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 155                                     &pos->last->lru);
 156         }
 157
 158         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 159                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 160                 struct ttm_resource_manager *man;
 161
 162                 if (!pos->first)
 163                         continue;
 164
 165                 dma_resv_assert_held(pos->first->base.resv);
 166                 dma_resv_assert_held(pos->last->base.resv);
 167
 168                 man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM);
 169                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 170                                     &pos->last->lru);
 171         }
 172
 173         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 174                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 175                 struct list_head *lru;
 176
 177                 if (!pos->first)
 178                         continue;
 179
 180                 dma_resv_assert_held(pos->first->base.resv);
 181                 dma_resv_assert_held(pos->last->base.resv);
 182
 183                 lru = &ttm_glob.swap_lru[i];
 184                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 185         }
 186 }
 187 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 188
 189 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 190                                   struct ttm_resource *mem, bool evict,
 191                                   struct ttm_operation_ctx *ctx,
 192                                   struct ttm_place *hop)
 193 {
 194         struct ttm_device *bdev = bo->bdev;
 195         struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type);
 196         struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type);
 197         int ret;
 198
 199         ttm_bo_unmap_virtual(bo);
 200
 201         /*
 202          * Create and bind a ttm if required.
 203          */
 204
 205         if (new_man->use_tt) {
 206                 /* Zero init the new TTM structure if the old location should
 207                  * have used one as well.
 208                  */
 209                 ret = ttm_tt_create(bo, old_man->use_tt);
 210                 if (ret)
 211                         goto out_err;
 212
 213                 if (mem->mem_type != TTM_PL_SYSTEM) {
 214                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 215                         if (ret)
 216                                 goto out_err;
 217                 }
 218         }
 219
 220         ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
 221         if (ret) {
 222                 if (ret == -EMULTIHOP)
 223                         return ret;
 224                 goto out_err;
 225         }
 226
 227         ctx->bytes_moved += bo->base.size;
 228         return 0;
 229
 230 out_err:
 231         new_man = ttm_manager_type(bdev, bo->mem.mem_type);
 232         if (!new_man->use_tt)
 233                 ttm_bo_tt_destroy(bo);
 234
 235         return ret;
 236 }
 237
 238 /*
 239  * Call bo::reserved.
 240  * Will release GPU memory type usage on destruction.
 241  * This is the place to put in driver specific hooks to release
 242  * driver private resources.
 243  * Will release the bo::reserved lock.
 244  */
 245
 246 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 247 {
 248         if (bo->bdev->funcs->delete_mem_notify)
 249                 bo->bdev->funcs->delete_mem_notify(bo);
 250
 251         ttm_bo_tt_destroy(bo);
 252         ttm_resource_free(bo, &bo->mem);
 253 }
 254
 255 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 256 {
 257         int r;
 258
 259         if (bo->base.resv == &bo->base._resv)
 260                 return 0;
 261
 262         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 263
 264         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 265         dma_resv_unlock(&bo->base._resv);
 266         if (r)
 267                 return r;
 268
 269         if (bo->type != ttm_bo_type_sg) {
 270                 /* This works because the BO is about to be destroyed and nobody
 271                  * reference it any more. The only tricky case is the trylock on
 272                  * the resv object while holding the lru_lock.
 273                  */
 274                 spin_lock(&ttm_glob.lru_lock);
 275                 bo->base.resv = &bo->base._resv;
 276                 spin_unlock(&ttm_glob.lru_lock);
 277         }
 278
 279         return r;
 280 }
 281
 282 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 283 {
 284         struct dma_resv *resv = &bo->base._resv;
 285         struct dma_resv_list *fobj;
 286         struct dma_fence *fence;
 287         int i;
 288
 289         rcu_read_lock();
 290         fobj = rcu_dereference(resv->fence);
 291         fence = rcu_dereference(resv->fence_excl);
 292         if (fence && !fence->ops->signaled)
 293                 dma_fence_enable_sw_signaling(fence);
 294
 295         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 296                 fence = rcu_dereference(fobj->shared[i]);
 297
 298                 if (!fence->ops->signaled)
 299                         dma_fence_enable_sw_signaling(fence);
 300         }
 301         rcu_read_unlock();
 302 }
 303
 304 /**
 305  * function ttm_bo_cleanup_refs
 306  * If bo idle, remove from lru lists, and unref.
 307  * If not idle, block if possible.
 308  *
 309  * Must be called with lru_lock and reservation held, this function
 310  * will drop the lru lock and optionally the reservation lock before returning.
 311  *
 312  * @bo:                    The buffer object to clean-up
 313  * @interruptible:         Any sleeps should occur interruptibly.
 314  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 315  * @unlock_resv:           Unlock the reservation lock as well.
 316  */
 317
 318 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 319                                bool interruptible, bool no_wait_gpu,
 320                                bool unlock_resv)
 321 {
 322         struct dma_resv *resv = &bo->base._resv;
 323         int ret;
 324
 325         if (dma_resv_test_signaled_rcu(resv, true))
 326                 ret = 0;
 327         else
 328                 ret = -EBUSY;
 329
 330         if (ret && !no_wait_gpu) {
 331                 long lret;
 332
 333                 if (unlock_resv)
 334                         dma_resv_unlock(bo->base.resv);
 335                 spin_unlock(&ttm_glob.lru_lock);
 336
 337                 lret = dma_resv_wait_timeout_rcu(resv, true, interruptible,
 338                                                  30 * HZ);
 339
 340                 if (lret < 0)
 341                         return lret;
 342                 else if (lret == 0)
 343                         return -EBUSY;
 344
 345                 spin_lock(&ttm_glob.lru_lock);
 346                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 347                         /*
 348                          * We raced, and lost, someone else holds the reservation now,
 349                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 350                          *
 351                          * Even if it's not the case, because we finished waiting any
 352                          * delayed destruction would succeed, so just return success
 353                          * here.
 354                          */
 355                         spin_unlock(&ttm_glob.lru_lock);
 356                         return 0;
 357                 }
 358                 ret = 0;
 359         }
 360
 361         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 362                 if (unlock_resv)
 363                         dma_resv_unlock(bo->base.resv);
 364                 spin_unlock(&ttm_glob.lru_lock);
 365                 return ret;
 366         }
 367
 368         ttm_bo_del_from_lru(bo);
 369         list_del_init(&bo->ddestroy);
 370         spin_unlock(&ttm_glob.lru_lock);
 371         ttm_bo_cleanup_memtype_use(bo);
 372
 373         if (unlock_resv)
 374                 dma_resv_unlock(bo->base.resv);
 375
 376         ttm_bo_put(bo);
 377
 378         return 0;
 379 }
 380
 381 /*
 382  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 383  * encountered buffers.
 384  */
 385 bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all)
 386 {
 387         struct ttm_global *glob = &ttm_glob;
 388         struct list_head removed;
 389         bool empty;
 390
 391         INIT_LIST_HEAD(&removed);
 392
 393         spin_lock(&glob->lru_lock);
 394         while (!list_empty(&bdev->ddestroy)) {
 395                 struct ttm_buffer_object *bo;
 396
 397                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 398                                       ddestroy);
 399                 list_move_tail(&bo->ddestroy, &removed);
 400                 if (!ttm_bo_get_unless_zero(bo))
 401                         continue;
 402
 403                 if (remove_all || bo->base.resv != &bo->base._resv) {
 404                         spin_unlock(&glob->lru_lock);
 405                         dma_resv_lock(bo->base.resv, NULL);
 406
 407                         spin_lock(&glob->lru_lock);
 408                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 409
 410                 } else if (dma_resv_trylock(bo->base.resv)) {
 411                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 412                 } else {
 413                         spin_unlock(&glob->lru_lock);
 414                 }
 415
 416                 ttm_bo_put(bo);
 417                 spin_lock(&glob->lru_lock);
 418         }
 419         list_splice_tail(&removed, &bdev->ddestroy);
 420         empty = list_empty(&bdev->ddestroy);
 421         spin_unlock(&glob->lru_lock);
 422
 423         return empty;
 424 }
 425
 426 static void ttm_bo_release(struct kref *kref)
 427 {
 428         struct ttm_buffer_object *bo =
 429             container_of(kref, struct ttm_buffer_object, kref);
 430         struct ttm_device *bdev = bo->bdev;
 431         int ret;
 432
 433         if (!bo->deleted) {
 434                 ret = ttm_bo_individualize_resv(bo);
 435                 if (ret) {
 436                         /* Last resort, if we fail to allocate memory for the
 437                          * fences block for the BO to become idle
 438                          */
 439                         dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
 440                                                   30 * HZ);
 441                 }
 442
 443                 if (bo->bdev->funcs->release_notify)
 444                         bo->bdev->funcs->release_notify(bo);
 445
 446                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 447                 ttm_mem_io_free(bdev, &bo->mem);
 448         }
 449
 450         if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
 451             !dma_resv_trylock(bo->base.resv)) {
 452                 /* The BO is not idle, resurrect it for delayed destroy */
 453                 ttm_bo_flush_all_fences(bo);
 454                 bo->deleted = true;
 455
 456                 spin_lock(&ttm_glob.lru_lock);
 457
 458                 /*
 459                  * Make pinned bos immediately available to
 460                  * shrinkers, now that they are queued for
 461                  * destruction.
 462                  *
 463                  * FIXME: QXL is triggering this. Can be removed when the
 464                  * driver is fixed.
 465                  */
 466                 if (WARN_ON_ONCE(bo->pin_count)) {
 467                         bo->pin_count = 0;
 468                         ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL);
 469                 }
 470
 471                 kref_init(&bo->kref);
 472                 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 473                 spin_unlock(&ttm_glob.lru_lock);
 474
 475                 schedule_delayed_work(&bdev->wq,
 476                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 477                 return;
 478         }
 479
 480         spin_lock(&ttm_glob.lru_lock);
 481         ttm_bo_del_from_lru(bo);
 482         list_del(&bo->ddestroy);
 483         spin_unlock(&ttm_glob.lru_lock);
 484
 485         ttm_bo_cleanup_memtype_use(bo);
 486         dma_resv_unlock(bo->base.resv);
 487
 488         atomic_dec(&ttm_glob.bo_count);
 489         dma_fence_put(bo->moving);
 490         if (!ttm_bo_uses_embedded_gem_object(bo))
 491                 dma_resv_fini(&bo->base._resv);
 492         bo->destroy(bo);
 493 }
 494
 495 void ttm_bo_put(struct ttm_buffer_object *bo)
 496 {
 497         kref_put(&bo->kref, ttm_bo_release);
 498 }
 499 EXPORT_SYMBOL(ttm_bo_put);
 500
 501 int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev)
 502 {
 503         return cancel_delayed_work_sync(&bdev->wq);
 504 }
 505 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 506
 507 void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched)
 508 {
 509         if (resched)
 510                 schedule_delayed_work(&bdev->wq,
 511                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 512 }
 513 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 514
 515 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 516                         struct ttm_operation_ctx *ctx)
 517 {
 518         struct ttm_device *bdev = bo->bdev;
 519         struct ttm_resource evict_mem;
 520         struct ttm_placement placement;
 521         struct ttm_place hop;
 522         int ret = 0;
 523
 524         memset(&hop, 0, sizeof(hop));
 525
 526         dma_resv_assert_held(bo->base.resv);
 527
 528         placement.num_placement = 0;
 529         placement.num_busy_placement = 0;
 530         bdev->funcs->evict_flags(bo, &placement);
 531
 532         if (!placement.num_placement && !placement.num_busy_placement) {
 533                 ttm_bo_wait(bo, false, false);
 534
 535                 ttm_bo_cleanup_memtype_use(bo);
 536                 return ttm_tt_create(bo, false);
 537         }
 538
 539         evict_mem = bo->mem;
 540         evict_mem.mm_node = NULL;
 541         evict_mem.bus.offset = 0;
 542         evict_mem.bus.addr = NULL;
 543
 544         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 545         if (ret) {
 546                 if (ret != -ERESTARTSYS) {
 547                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 548                                bo);
 549                         ttm_bo_mem_space_debug(bo, &placement);
 550                 }
 551                 goto out;
 552         }
 553
 554         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx, &hop);
 555         if (unlikely(ret)) {
 556                 WARN(ret == -EMULTIHOP, "Unexpected multihop in eviction - likely driver bug\n");
 557                 if (ret != -ERESTARTSYS)
 558                         pr_err("Buffer eviction failed\n");
 559                 ttm_resource_free(bo, &evict_mem);
 560         }
 561 out:
 562         return ret;
 563 }
 564
 565 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 566                               const struct ttm_place *place)
 567 {
 568         /* Don't evict this BO if it's outside of the
 569          * requested placement range
 570          */
 571         if (place->fpfn >= (bo->mem.start + bo->mem.num_pages) ||
 572             (place->lpfn && place->lpfn <= bo->mem.start))
 573                 return false;
 574
 575         return true;
 576 }
 577 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 578
 579 /*
 580  * Check the target bo is allowable to be evicted or swapout, including cases:
 581  *
 582  * a. if share same reservation object with ctx->resv, have assumption
 583  * reservation objects should already be locked, so not lock again and
 584  * return true directly when either the opreation allow_reserved_eviction
 585  * or the target bo already is in delayed free list;
 586  *
 587  * b. Otherwise, trylock it.
 588  */
 589 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 590                         struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
 591 {
 592         bool ret = false;
 593
 594         if (bo->base.resv == ctx->resv) {
 595                 dma_resv_assert_held(bo->base.resv);
 596                 if (ctx->allow_res_evict)
 597                         ret = true;
 598                 *locked = false;
 599                 if (busy)
 600                         *busy = false;
 601         } else {
 602                 ret = dma_resv_trylock(bo->base.resv);
 603                 *locked = ret;
 604                 if (busy)
 605                         *busy = !ret;
 606         }
 607
 608         return ret;
 609 }
 610
 611 /**
 612  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 613  *
 614  * @busy_bo: BO which couldn't be locked with trylock
 615  * @ctx: operation context
 616  * @ticket: acquire ticket
 617  *
 618  * Try to lock a busy buffer object to avoid failing eviction.
 619  */
 620 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 621                                    struct ttm_operation_ctx *ctx,
 622                                    struct ww_acquire_ctx *ticket)
 623 {
 624         int r;
 625
 626         if (!busy_bo || !ticket)
 627                 return -EBUSY;
 628
 629         if (ctx->interruptible)
 630                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 631                                                           ticket);
 632         else
 633                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 634
 635         /*
 636          * TODO: It would be better to keep the BO locked until allocation is at
 637          * least tried one more time, but that would mean a much larger rework
 638          * of TTM.
 639          */
 640         if (!r)
 641                 dma_resv_unlock(busy_bo->base.resv);
 642
 643         return r == -EDEADLK ? -EBUSY : r;
 644 }
 645
 646 int ttm_mem_evict_first(struct ttm_device *bdev,
 647                         struct ttm_resource_manager *man,
 648                         const struct ttm_place *place,
 649                         struct ttm_operation_ctx *ctx,
 650                         struct ww_acquire_ctx *ticket)
 651 {
 652         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 653         bool locked = false;
 654         unsigned i;
 655         int ret;
 656
 657         spin_lock(&ttm_glob.lru_lock);
 658         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 659                 list_for_each_entry(bo, &man->lru[i], lru) {
 660                         bool busy;
 661
 662                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
 663                                                             &busy)) {
 664                                 if (busy && !busy_bo && ticket !=
 665                                     dma_resv_locking_ctx(bo->base.resv))
 666                                         busy_bo = bo;
 667                                 continue;
 668                         }
 669
 670                         if (place && !bdev->funcs->eviction_valuable(bo,
 671                                                                       place)) {
 672                                 if (locked)
 673                                         dma_resv_unlock(bo->base.resv);
 674                                 continue;
 675                         }
 676                         if (!ttm_bo_get_unless_zero(bo)) {
 677                                 if (locked)
 678                                         dma_resv_unlock(bo->base.resv);
 679                                 continue;
 680                         }
 681                         break;
 682                 }
 683
 684                 /* If the inner loop terminated early, we have our candidate */
 685                 if (&bo->lru != &man->lru[i])
 686                         break;
 687
 688                 bo = NULL;
 689         }
 690
 691         if (!bo) {
 692                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 693                         busy_bo = NULL;
 694                 spin_unlock(&ttm_glob.lru_lock);
 695                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 696                 if (busy_bo)
 697                         ttm_bo_put(busy_bo);
 698                 return ret;
 699         }
 700
 701         if (bo->deleted) {
 702                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 703                                           ctx->no_wait_gpu, locked);
 704                 ttm_bo_put(bo);
 705                 return ret;
 706         }
 707
 708         spin_unlock(&ttm_glob.lru_lock);
 709
 710         ret = ttm_bo_evict(bo, ctx);
 711         if (locked)
 712                 ttm_bo_unreserve(bo);
 713
 714         ttm_bo_put(bo);
 715         return ret;
 716 }
 717
 718 /*
 719  * Add the last move fence to the BO and reserve a new shared slot.
 720  */
 721 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 722                                  struct ttm_resource_manager *man,
 723                                  struct ttm_resource *mem,
 724                                  bool no_wait_gpu)
 725 {
 726         struct dma_fence *fence;
 727         int ret;
 728
 729         spin_lock(&man->move_lock);
 730         fence = dma_fence_get(man->move);
 731         spin_unlock(&man->move_lock);
 732
 733         if (!fence)
 734                 return 0;
 735
 736         if (no_wait_gpu) {
 737                 dma_fence_put(fence);
 738                 return -EBUSY;
 739         }
 740
 741         dma_resv_add_shared_fence(bo->base.resv, fence);
 742
 743         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 744         if (unlikely(ret)) {
 745                 dma_fence_put(fence);
 746                 return ret;
 747         }
 748
 749         dma_fence_put(bo->moving);
 750         bo->moving = fence;
 751         return 0;
 752 }
 753
 754 /*
 755  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 756  * space, or we've evicted everything and there isn't enough space.
 757  */
 758 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 759                                   const struct ttm_place *place,
 760                                   struct ttm_resource *mem,
 761                                   struct ttm_operation_ctx *ctx)
 762 {
 763         struct ttm_device *bdev = bo->bdev;
 764         struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type);
 765         struct ww_acquire_ctx *ticket;
 766         int ret;
 767
 768         ticket = dma_resv_locking_ctx(bo->base.resv);
 769         do {
 770                 ret = ttm_resource_alloc(bo, place, mem);
 771                 if (likely(!ret))
 772                         break;
 773                 if (unlikely(ret != -ENOSPC))
 774                         return ret;
 775                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 776                                           ticket);
 777                 if (unlikely(ret != 0))
 778                         return ret;
 779         } while (1);
 780
 781         return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 782 }
 783
 784 /**
 785  * ttm_bo_mem_placement - check if placement is compatible
 786  * @bo: BO to find memory for
 787  * @place: where to search
 788  * @mem: the memory object to fill in
 789  *
 790  * Check if placement is compatible and fill in mem structure.
 791  * Returns -EBUSY if placement won't work or negative error code.
 792  * 0 when placement can be used.
 793  */
 794 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
 795                                 const struct ttm_place *place,
 796                                 struct ttm_resource *mem)
 797 {
 798         struct ttm_device *bdev = bo->bdev;
 799         struct ttm_resource_manager *man;
 800
 801         man = ttm_manager_type(bdev, place->mem_type);
 802         if (!man || !ttm_resource_manager_used(man))
 803                 return -EBUSY;
 804
 805         mem->mem_type = place->mem_type;
 806         mem->placement = place->flags;
 807
 808         spin_lock(&ttm_glob.lru_lock);
 809         ttm_bo_move_to_lru_tail(bo, mem, NULL);
 810         spin_unlock(&ttm_glob.lru_lock);
 811
 812         return 0;
 813 }
 814
 815 /*
 816  * Creates space for memory region @mem according to its type.
 817  *
 818  * This function first searches for free space in compatible memory types in
 819  * the priority order defined by the driver.  If free space isn't found, then
 820  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 821  * space.
 822  */
 823 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 824                         struct ttm_placement *placement,
 825                         struct ttm_resource *mem,
 826                         struct ttm_operation_ctx *ctx)
 827 {
 828         struct ttm_device *bdev = bo->bdev;
 829         bool type_found = false;
 830         int i, ret;
 831
 832         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 833         if (unlikely(ret))
 834                 return ret;
 835
 836         for (i = 0; i < placement->num_placement; ++i) {
 837                 const struct ttm_place *place = &placement->placement[i];
 838                 struct ttm_resource_manager *man;
 839
 840                 ret = ttm_bo_mem_placement(bo, place, mem);
 841                 if (ret)
 842                         continue;
 843
 844                 type_found = true;
 845                 ret = ttm_resource_alloc(bo, place, mem);
 846                 if (ret == -ENOSPC)
 847                         continue;
 848                 if (unlikely(ret))
 849                         goto error;
 850
 851                 man = ttm_manager_type(bdev, mem->mem_type);
 852                 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 853                 if (unlikely(ret)) {
 854                         ttm_resource_free(bo, mem);
 855                         if (ret == -EBUSY)
 856                                 continue;
 857
 858                         goto error;
 859                 }
 860                 return 0;
 861         }
 862
 863         for (i = 0; i < placement->num_busy_placement; ++i) {
 864                 const struct ttm_place *place = &placement->busy_placement[i];
 865
 866                 ret = ttm_bo_mem_placement(bo, place, mem);
 867                 if (ret)
 868                         continue;
 869
 870                 type_found = true;
 871                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 872                 if (likely(!ret))
 873                         return 0;
 874
 875                 if (ret && ret != -EBUSY)
 876                         goto error;
 877         }
 878
 879         ret = -ENOMEM;
 880         if (!type_found) {
 881                 pr_err(TTM_PFX "No compatible memory type found\n");
 882                 ret = -EINVAL;
 883         }
 884
 885 error:
 886         if (bo->mem.mem_type == TTM_PL_SYSTEM && !bo->pin_count)
 887                 ttm_bo_move_to_lru_tail_unlocked(bo);
 888
 889         return ret;
 890 }
 891 EXPORT_SYMBOL(ttm_bo_mem_space);
 892
 893 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 894                                      struct ttm_resource *mem,
 895                                      struct ttm_operation_ctx *ctx,
 896                                      struct ttm_place *hop)
 897 {
 898         struct ttm_placement hop_placement;
 899         int ret;
 900         struct ttm_resource hop_mem = *mem;
 901
 902         hop_mem.mm_node = NULL;
 903         hop_mem.mem_type = TTM_PL_SYSTEM;
 904         hop_mem.placement = 0;
 905
 906         hop_placement.num_placement = hop_placement.num_busy_placement = 1;
 907         hop_placement.placement = hop_placement.busy_placement = hop;
 908
 909         /* find space in the bounce domain */
 910         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 911         if (ret)
 912                 return ret;
 913         /* move to the bounce domain */
 914         ret = ttm_bo_handle_move_mem(bo, &hop_mem, false, ctx, NULL);
 915         if (ret) {
 916                 ttm_resource_free(bo, &hop_mem);
 917                 return ret;
 918         }
 919         return 0;
 920 }
 921
 922 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 923                               struct ttm_placement *placement,
 924                               struct ttm_operation_ctx *ctx)
 925 {
 926         int ret = 0;
 927         struct ttm_place hop;
 928         struct ttm_resource mem;
 929
 930         dma_resv_assert_held(bo->base.resv);
 931
 932         memset(&hop, 0, sizeof(hop));
 933
 934         mem.num_pages = PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT;
 935         mem.page_alignment = bo->mem.page_alignment;
 936         mem.bus.offset = 0;
 937         mem.bus.addr = NULL;
 938         mem.mm_node = NULL;
 939
 940         /*
 941          * Determine where to move the buffer.
 942          *
 943          * If driver determines move is going to need
 944          * an extra step then it will return -EMULTIHOP
 945          * and the buffer will be moved to the temporary
 946          * stop and the driver will be called to make
 947          * the second hop.
 948          */
 949         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 950         if (ret)
 951                 return ret;
 952 bounce:
 953         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx, &hop);
 954         if (ret == -EMULTIHOP) {
 955                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
 956                 if (ret)
 957                         goto out;
 958                 /* try and move to final place now. */
 959                 goto bounce;
 960         }
 961 out:
 962         if (ret)
 963                 ttm_resource_free(bo, &mem);
 964         return ret;
 965 }
 966
 967 static bool ttm_bo_places_compat(const struct ttm_place *places,
 968                                  unsigned num_placement,
 969                                  struct ttm_resource *mem,
 970                                  uint32_t *new_flags)
 971 {
 972         unsigned i;
 973
 974         for (i = 0; i < num_placement; i++) {
 975                 const struct ttm_place *heap = &places[i];
 976
 977                 if ((mem->start < heap->fpfn ||
 978                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
 979                         continue;
 980
 981                 *new_flags = heap->flags;
 982                 if ((mem->mem_type == heap->mem_type) &&
 983                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
 984                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
 985                         return true;
 986         }
 987         return false;
 988 }
 989
 990 bool ttm_bo_mem_compat(struct ttm_placement *placement,
 991                        struct ttm_resource *mem,
 992                        uint32_t *new_flags)
 993 {
 994         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
 995                                  mem, new_flags))
 996                 return true;
 997
 998         if ((placement->busy_placement != placement->placement ||
 999              placement->num_busy_placement > placement->num_placement) &&
1000             ttm_bo_places_compat(placement->busy_placement,
1001                                  placement->num_busy_placement,
1002                                  mem, new_flags))
1003                 return true;
1004
1005         return false;
1006 }
1007 EXPORT_SYMBOL(ttm_bo_mem_compat);
1008
1009 int ttm_bo_validate(struct ttm_buffer_object *bo,
1010                     struct ttm_placement *placement,
1011                     struct ttm_operation_ctx *ctx)
1012 {
1013         int ret;
1014         uint32_t new_flags;
1015
1016         dma_resv_assert_held(bo->base.resv);
1017
1018         /*
1019          * Remove the backing store if no placement is given.
1020          */
1021         if (!placement->num_placement && !placement->num_busy_placement) {
1022                 ret = ttm_bo_pipeline_gutting(bo);
1023                 if (ret)
1024                         return ret;
1025
1026                 return ttm_tt_create(bo, false);
1027         }
1028
1029         /*
1030          * Check whether we need to move buffer.
1031          */
1032         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1033                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1034                 if (ret)
1035                         return ret;
1036         }
1037         /*
1038          * We might need to add a TTM.
1039          */
1040         if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1041                 ret = ttm_tt_create(bo, true);
1042                 if (ret)
1043                         return ret;
1044         }
1045         return 0;
1046 }
1047 EXPORT_SYMBOL(ttm_bo_validate);
1048
1049 int ttm_bo_init_reserved(struct ttm_device *bdev,
1050                          struct ttm_buffer_object *bo,
1051                          size_t size,
1052                          enum ttm_bo_type type,
1053                          struct ttm_placement *placement,
1054                          uint32_t page_alignment,
1055                          struct ttm_operation_ctx *ctx,
1056                          struct sg_table *sg,
1057                          struct dma_resv *resv,
1058                          void (*destroy) (struct ttm_buffer_object *))
1059 {
1060         bool locked;
1061         int ret = 0;
1062
1063         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1064
1065         kref_init(&bo->kref);
1066         INIT_LIST_HEAD(&bo->lru);
1067         INIT_LIST_HEAD(&bo->ddestroy);
1068         INIT_LIST_HEAD(&bo->swap);
1069         bo->bdev = bdev;
1070         bo->type = type;
1071         bo->mem.mem_type = TTM_PL_SYSTEM;
1072         bo->mem.num_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1073         bo->mem.mm_node = NULL;
1074         bo->mem.page_alignment = page_alignment;
1075         bo->mem.bus.offset = 0;
1076         bo->mem.bus.addr = NULL;
1077         bo->moving = NULL;
1078         bo->mem.placement = 0;
1079         bo->pin_count = 0;
1080         bo->sg = sg;
1081         if (resv) {
1082                 bo->base.resv = resv;
1083                 dma_resv_assert_held(bo->base.resv);
1084         } else {
1085                 bo->base.resv = &bo->base._resv;
1086         }
1087         if (!ttm_bo_uses_embedded_gem_object(bo)) {
1088                 /*
1089                  * bo.base is not initialized, so we have to setup the
1090                  * struct elements we want use regardless.
1091                  */
1092                 bo->base.size = size;
1093                 dma_resv_init(&bo->base._resv);
1094                 drm_vma_node_reset(&bo->base.vma_node);
1095         }
1096         atomic_inc(&ttm_glob.bo_count);
1097
1098         /*
1099          * For ttm_bo_type_device buffers, allocate
1100          * address space from the device.
1101          */
1102         if (bo->type == ttm_bo_type_device ||
1103             bo->type == ttm_bo_type_sg)
1104                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1105                                          bo->mem.num_pages);
1106
1107         /* passed reservation objects should already be locked,
1108          * since otherwise lockdep will be angered in radeon.
1109          */
1110         if (!resv) {
1111                 locked = dma_resv_trylock(bo->base.resv);
1112                 WARN_ON(!locked);
1113         }
1114
1115         if (likely(!ret))
1116                 ret = ttm_bo_validate(bo, placement, ctx);
1117
1118         if (unlikely(ret)) {
1119                 if (!resv)
1120                         ttm_bo_unreserve(bo);
1121
1122                 ttm_bo_put(bo);
1123                 return ret;
1124         }
1125
1126         ttm_bo_move_to_lru_tail_unlocked(bo);
1127
1128         return ret;
1129 }
1130 EXPORT_SYMBOL(ttm_bo_init_reserved);
1131
1132 int ttm_bo_init(struct ttm_device *bdev,
1133                 struct ttm_buffer_object *bo,
1134                 size_t size,
1135                 enum ttm_bo_type type,
1136                 struct ttm_placement *placement,
1137                 uint32_t page_alignment,
1138                 bool interruptible,
1139                 struct sg_table *sg,
1140                 struct dma_resv *resv,
1141                 void (*destroy) (struct ttm_buffer_object *))
1142 {
1143         struct ttm_operation_ctx ctx = { interruptible, false };
1144         int ret;
1145
1146         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1147                                    page_alignment, &ctx, sg, resv, destroy);
1148         if (ret)
1149                 return ret;
1150
1151         if (!resv)
1152                 ttm_bo_unreserve(bo);
1153
1154         return 0;
1155 }
1156 EXPORT_SYMBOL(ttm_bo_init);
1157
1158 /*
1159  * buffer object vm functions.
1160  */
1161
1162 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1163 {
1164         struct ttm_device *bdev = bo->bdev;
1165
1166         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1167         ttm_mem_io_free(bdev, &bo->mem);
1168 }
1169 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1170
1171 int ttm_bo_wait(struct ttm_buffer_object *bo,
1172                 bool interruptible, bool no_wait)
1173 {
1174         long timeout = 15 * HZ;
1175
1176         if (no_wait) {
1177                 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1178                         return 0;
1179                 else
1180                         return -EBUSY;
1181         }
1182
1183         timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1184                                                       interruptible, timeout);
1185         if (timeout < 0)
1186                 return timeout;
1187
1188         if (timeout == 0)
1189                 return -EBUSY;
1190
1191         dma_resv_add_excl_fence(bo->base.resv, NULL);
1192         return 0;
1193 }
1194 EXPORT_SYMBOL(ttm_bo_wait);
1195
1196 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1197                    gfp_t gfp_flags)
1198 {
1199         struct ttm_global *glob = &ttm_glob;
1200         bool locked;
1201         int ret;
1202
1203         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, NULL))
1204                 return -EBUSY;
1205
1206         if (!ttm_bo_get_unless_zero(bo)) {
1207                 if (locked)
1208                         dma_resv_unlock(bo->base.resv);
1209                 return -EBUSY;
1210         }
1211
1212         if (bo->deleted) {
1213                 ttm_bo_cleanup_refs(bo, false, false, locked);
1214                 ttm_bo_put(bo);
1215                 return 0;
1216         }
1217
1218         ttm_bo_del_from_lru(bo);
1219         /* TODO: Cleanup the locking */
1220         spin_unlock(&glob->lru_lock);
1221
1222         /*
1223          * Move to system cached
1224          */
1225         if (bo->mem.mem_type != TTM_PL_SYSTEM) {
1226                 struct ttm_operation_ctx ctx = { false, false };
1227                 struct ttm_resource evict_mem;
1228                 struct ttm_place hop;
1229
1230                 memset(&hop, 0, sizeof(hop));
1231
1232                 evict_mem = bo->mem;
1233                 evict_mem.mm_node = NULL;
1234                 evict_mem.placement = 0;
1235                 evict_mem.mem_type = TTM_PL_SYSTEM;
1236
1237                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx, &hop);
1238                 if (unlikely(ret != 0)) {
1239                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1240                         goto out;
1241                 }
1242         }
1243
1244         /*
1245          * Make sure BO is idle.
1246          */
1247         ret = ttm_bo_wait(bo, false, false);
1248         if (unlikely(ret != 0))
1249                 goto out;
1250
1251         ttm_bo_unmap_virtual(bo);
1252
1253         /*
1254          * Swap out. Buffer will be swapped in again as soon as
1255          * anyone tries to access a ttm page.
1256          */
1257         if (bo->bdev->funcs->swap_notify)
1258                 bo->bdev->funcs->swap_notify(bo);
1259
1260         ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1261 out:
1262
1263         /*
1264          * Unreserve without putting on LRU to avoid swapping out an
1265          * already swapped buffer.
1266          */
1267         if (locked)
1268                 dma_resv_unlock(bo->base.resv);
1269         ttm_bo_put(bo);
1270         return ret;
1271 }
1272
1273 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1274 {
1275         if (bo->ttm == NULL)
1276                 return;
1277
1278         ttm_tt_destroy(bo->bdev, bo->ttm);
1279         bo->ttm = NULL;
1280 }