drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <drm/ttm/ttm_tt.h>
  37
  38 #include <linux/jiffies.h>
  39 #include <linux/slab.h>
  40 #include <linux/sched.h>
  41 #include <linux/mm.h>
  42 #include <linux/file.h>
  43 #include <linux/module.h>
  44 #include <linux/atomic.h>
  45 #include <linux/dma-resv.h>
  46
  47 #include "ttm_module.h"
  48
  49 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  50                                         struct ttm_placement *placement)
  51 {
  52         struct drm_printer p = drm_debug_printer(TTM_PFX);
  53         struct ttm_resource_manager *man;
  54         int i, mem_type;
  55
  56         for (i = 0; i < placement->num_placement; i++) {
  57                 mem_type = placement->placement[i].mem_type;
  58                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  59                            i, placement->placement[i].flags, mem_type);
  60                 man = ttm_manager_type(bo->bdev, mem_type);
  61                 ttm_resource_manager_debug(man, &p);
  62         }
  63 }
  64
  65 /**
  66  * ttm_bo_move_to_lru_tail
  67  *
  68  * @bo: The buffer object.
  69  *
  70  * Move this BO to the tail of all lru lists used to lookup and reserve an
  71  * object. This function must be called with struct ttm_global::lru_lock
  72  * held, and is used to make a BO less likely to be considered for eviction.
  73  */
  74 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
  75 {
  76         dma_resv_assert_held(bo->base.resv);
  77
  78         if (bo->resource)
  79                 ttm_resource_move_to_lru_tail(bo->resource);
  80 }
  81 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
  82
  83 /**
  84  * ttm_bo_set_bulk_move - update BOs bulk move object
  85  *
  86  * @bo: The buffer object.
  87  *
  88  * Update the BOs bulk move object, making sure that resources are added/removed
  89  * as well. A bulk move allows to move many resource on the LRU at once,
  90  * resulting in much less overhead of maintaining the LRU.
  91  * The only requirement is that the resources stay together on the LRU and are
  92  * never separated. This is enforces by setting the bulk_move structure on a BO.
  93  * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
  94  * their LRU list.
  95  */
  96 void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
  97                           struct ttm_lru_bulk_move *bulk)
  98 {
  99         dma_resv_assert_held(bo->base.resv);
 100
 101         if (bo->bulk_move == bulk)
 102                 return;
 103
 104         spin_lock(&bo->bdev->lru_lock);
 105         if (bo->resource)
 106                 ttm_resource_del_bulk_move(bo->resource, bo);
 107         bo->bulk_move = bulk;
 108         if (bo->resource)
 109                 ttm_resource_add_bulk_move(bo->resource, bo);
 110         spin_unlock(&bo->bdev->lru_lock);
 111 }
 112 EXPORT_SYMBOL(ttm_bo_set_bulk_move);
 113
 114 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 115                                   struct ttm_resource *mem, bool evict,
 116                                   struct ttm_operation_ctx *ctx,
 117                                   struct ttm_place *hop)
 118 {
 119         struct ttm_device *bdev = bo->bdev;
 120         bool old_use_tt, new_use_tt;
 121         int ret;
 122
 123         old_use_tt = bo->resource &&
 124                 ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
 125         new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 126
 127         ttm_bo_unmap_virtual(bo);
 128
 129         /*
 130          * Create and bind a ttm if required.
 131          */
 132
 133         if (new_use_tt) {
 134                 /* Zero init the new TTM structure if the old location should
 135                  * have used one as well.
 136                  */
 137                 ret = ttm_tt_create(bo, old_use_tt);
 138                 if (ret)
 139                         goto out_err;
 140
 141                 if (mem->mem_type != TTM_PL_SYSTEM) {
 142                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 143                         if (ret)
 144                                 goto out_err;
 145                 }
 146         }
 147
 148         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 149         if (ret)
 150                 goto out_err;
 151
 152         ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
 153         if (ret) {
 154                 if (ret == -EMULTIHOP)
 155                         return ret;
 156                 goto out_err;
 157         }
 158
 159         ctx->bytes_moved += bo->base.size;
 160         return 0;
 161
 162 out_err:
 163         if (!old_use_tt)
 164                 ttm_bo_tt_destroy(bo);
 165
 166         return ret;
 167 }
 168
 169 /*
 170  * Call bo::reserved.
 171  * Will release GPU memory type usage on destruction.
 172  * This is the place to put in driver specific hooks to release
 173  * driver private resources.
 174  * Will release the bo::reserved lock.
 175  */
 176
 177 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 178 {
 179         if (bo->bdev->funcs->delete_mem_notify)
 180                 bo->bdev->funcs->delete_mem_notify(bo);
 181
 182         ttm_bo_tt_destroy(bo);
 183         ttm_resource_free(bo, &bo->resource);
 184 }
 185
 186 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 187 {
 188         int r;
 189
 190         if (bo->base.resv == &bo->base._resv)
 191                 return 0;
 192
 193         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 194
 195         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 196         dma_resv_unlock(&bo->base._resv);
 197         if (r)
 198                 return r;
 199
 200         if (bo->type != ttm_bo_type_sg) {
 201                 /* This works because the BO is about to be destroyed and nobody
 202                  * reference it any more. The only tricky case is the trylock on
 203                  * the resv object while holding the lru_lock.
 204                  */
 205                 spin_lock(&bo->bdev->lru_lock);
 206                 bo->base.resv = &bo->base._resv;
 207                 spin_unlock(&bo->bdev->lru_lock);
 208         }
 209
 210         return r;
 211 }
 212
 213 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 214 {
 215         struct dma_resv *resv = &bo->base._resv;
 216         struct dma_resv_iter cursor;
 217         struct dma_fence *fence;
 218
 219         dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
 220         dma_resv_for_each_fence_unlocked(&cursor, fence) {
 221                 if (!fence->ops->signaled)
 222                         dma_fence_enable_sw_signaling(fence);
 223         }
 224         dma_resv_iter_end(&cursor);
 225 }
 226
 227 /**
 228  * ttm_bo_cleanup_refs
 229  * If bo idle, remove from lru lists, and unref.
 230  * If not idle, block if possible.
 231  *
 232  * Must be called with lru_lock and reservation held, this function
 233  * will drop the lru lock and optionally the reservation lock before returning.
 234  *
 235  * @bo:                    The buffer object to clean-up
 236  * @interruptible:         Any sleeps should occur interruptibly.
 237  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 238  * @unlock_resv:           Unlock the reservation lock as well.
 239  */
 240
 241 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 242                                bool interruptible, bool no_wait_gpu,
 243                                bool unlock_resv)
 244 {
 245         struct dma_resv *resv = &bo->base._resv;
 246         int ret;
 247
 248         if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
 249                 ret = 0;
 250         else
 251                 ret = -EBUSY;
 252
 253         if (ret && !no_wait_gpu) {
 254                 long lret;
 255
 256                 if (unlock_resv)
 257                         dma_resv_unlock(bo->base.resv);
 258                 spin_unlock(&bo->bdev->lru_lock);
 259
 260                 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
 261                                              interruptible,
 262                                              30 * HZ);
 263
 264                 if (lret < 0)
 265                         return lret;
 266                 else if (lret == 0)
 267                         return -EBUSY;
 268
 269                 spin_lock(&bo->bdev->lru_lock);
 270                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 271                         /*
 272                          * We raced, and lost, someone else holds the reservation now,
 273                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 274                          *
 275                          * Even if it's not the case, because we finished waiting any
 276                          * delayed destruction would succeed, so just return success
 277                          * here.
 278                          */
 279                         spin_unlock(&bo->bdev->lru_lock);
 280                         return 0;
 281                 }
 282                 ret = 0;
 283         }
 284
 285         if (ret) {
 286                 if (unlock_resv)
 287                         dma_resv_unlock(bo->base.resv);
 288                 spin_unlock(&bo->bdev->lru_lock);
 289                 return ret;
 290         }
 291
 292         spin_unlock(&bo->bdev->lru_lock);
 293         ttm_bo_cleanup_memtype_use(bo);
 294
 295         if (unlock_resv)
 296                 dma_resv_unlock(bo->base.resv);
 297
 298         ttm_bo_put(bo);
 299
 300         return 0;
 301 }
 302
 303 /*
 304  * Block for the dma_resv object to become idle, lock the buffer and clean up
 305  * the resource and tt object.
 306  */
 307 static void ttm_bo_delayed_delete(struct work_struct *work)
 308 {
 309         struct ttm_buffer_object *bo;
 310
 311         bo = container_of(work, typeof(*bo), delayed_delete);
 312
 313         dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
 314                               MAX_SCHEDULE_TIMEOUT);
 315         dma_resv_lock(bo->base.resv, NULL);
 316         ttm_bo_cleanup_memtype_use(bo);
 317         dma_resv_unlock(bo->base.resv);
 318         ttm_bo_put(bo);
 319 }
 320
 321 static void ttm_bo_release(struct kref *kref)
 322 {
 323         struct ttm_buffer_object *bo =
 324             container_of(kref, struct ttm_buffer_object, kref);
 325         struct ttm_device *bdev = bo->bdev;
 326         int ret;
 327
 328         WARN_ON_ONCE(bo->pin_count);
 329         WARN_ON_ONCE(bo->bulk_move);
 330
 331         if (!bo->deleted) {
 332                 ret = ttm_bo_individualize_resv(bo);
 333                 if (ret) {
 334                         /* Last resort, if we fail to allocate memory for the
 335                          * fences block for the BO to become idle
 336                          */
 337                         dma_resv_wait_timeout(bo->base.resv,
 338                                               DMA_RESV_USAGE_BOOKKEEP, false,
 339                                               30 * HZ);
 340                 }
 341
 342                 if (bo->bdev->funcs->release_notify)
 343                         bo->bdev->funcs->release_notify(bo);
 344
 345                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 346                 ttm_mem_io_free(bdev, bo->resource);
 347
 348                 if (!dma_resv_test_signaled(bo->base.resv,
 349                                             DMA_RESV_USAGE_BOOKKEEP) ||
 350                     !dma_resv_trylock(bo->base.resv)) {
 351                         /* The BO is not idle, resurrect it for delayed destroy */
 352                         ttm_bo_flush_all_fences(bo);
 353                         bo->deleted = true;
 354
 355                         spin_lock(&bo->bdev->lru_lock);
 356
 357                         /*
 358                          * Make pinned bos immediately available to
 359                          * shrinkers, now that they are queued for
 360                          * destruction.
 361                          *
 362                          * FIXME: QXL is triggering this. Can be removed when the
 363                          * driver is fixed.
 364                          */
 365                         if (bo->pin_count) {
 366                                 bo->pin_count = 0;
 367                                 ttm_resource_move_to_lru_tail(bo->resource);
 368                         }
 369
 370                         kref_init(&bo->kref);
 371                         spin_unlock(&bo->bdev->lru_lock);
 372
 373                         INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);
 374                         queue_work(bdev->wq, &bo->delayed_delete);
 375                         return;
 376                 }
 377
 378                 ttm_bo_cleanup_memtype_use(bo);
 379                 dma_resv_unlock(bo->base.resv);
 380         }
 381
 382         atomic_dec(&ttm_glob.bo_count);
 383         bo->destroy(bo);
 384 }
 385
 386 /**
 387  * ttm_bo_put
 388  *
 389  * @bo: The buffer object.
 390  *
 391  * Unreference a buffer object.
 392  */
 393 void ttm_bo_put(struct ttm_buffer_object *bo)
 394 {
 395         kref_put(&bo->kref, ttm_bo_release);
 396 }
 397 EXPORT_SYMBOL(ttm_bo_put);
 398
 399 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 400                                      struct ttm_resource **mem,
 401                                      struct ttm_operation_ctx *ctx,
 402                                      struct ttm_place *hop)
 403 {
 404         struct ttm_placement hop_placement;
 405         struct ttm_resource *hop_mem;
 406         int ret;
 407
 408         hop_placement.num_placement = hop_placement.num_busy_placement = 1;
 409         hop_placement.placement = hop_placement.busy_placement = hop;
 410
 411         /* find space in the bounce domain */
 412         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 413         if (ret)
 414                 return ret;
 415         /* move to the bounce domain */
 416         ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
 417         if (ret) {
 418                 ttm_resource_free(bo, &hop_mem);
 419                 return ret;
 420         }
 421         return 0;
 422 }
 423
 424 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 425                         struct ttm_operation_ctx *ctx)
 426 {
 427         struct ttm_device *bdev = bo->bdev;
 428         struct ttm_resource *evict_mem;
 429         struct ttm_placement placement;
 430         struct ttm_place hop;
 431         int ret = 0;
 432
 433         memset(&hop, 0, sizeof(hop));
 434
 435         dma_resv_assert_held(bo->base.resv);
 436
 437         placement.num_placement = 0;
 438         placement.num_busy_placement = 0;
 439         bdev->funcs->evict_flags(bo, &placement);
 440
 441         if (!placement.num_placement && !placement.num_busy_placement) {
 442                 ret = ttm_bo_wait_ctx(bo, ctx);
 443                 if (ret)
 444                         return ret;
 445
 446                 /*
 447                  * Since we've already synced, this frees backing store
 448                  * immediately.
 449                  */
 450                 return ttm_bo_pipeline_gutting(bo);
 451         }
 452
 453         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 454         if (ret) {
 455                 if (ret != -ERESTARTSYS) {
 456                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 457                                bo);
 458                         ttm_bo_mem_space_debug(bo, &placement);
 459                 }
 460                 goto out;
 461         }
 462
 463 bounce:
 464         ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
 465         if (ret == -EMULTIHOP) {
 466                 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
 467                 if (ret) {
 468                         pr_err("Buffer eviction failed\n");
 469                         ttm_resource_free(bo, &evict_mem);
 470                         goto out;
 471                 }
 472                 /* try and move to final place now. */
 473                 goto bounce;
 474         }
 475 out:
 476         return ret;
 477 }
 478
 479 /**
 480  * ttm_bo_eviction_valuable
 481  *
 482  * @bo: The buffer object to evict
 483  * @place: the placement we need to make room for
 484  *
 485  * Check if it is valuable to evict the BO to make room for the given placement.
 486  */
 487 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 488                               const struct ttm_place *place)
 489 {
 490         struct ttm_resource *res = bo->resource;
 491         struct ttm_device *bdev = bo->bdev;
 492
 493         dma_resv_assert_held(bo->base.resv);
 494         if (bo->resource->mem_type == TTM_PL_SYSTEM)
 495                 return true;
 496
 497         /* Don't evict this BO if it's outside of the
 498          * requested placement range
 499          */
 500         return ttm_resource_intersects(bdev, res, place, bo->base.size);
 501 }
 502 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 503
 504 /*
 505  * Check the target bo is allowable to be evicted or swapout, including cases:
 506  *
 507  * a. if share same reservation object with ctx->resv, have assumption
 508  * reservation objects should already be locked, so not lock again and
 509  * return true directly when either the opreation allow_reserved_eviction
 510  * or the target bo already is in delayed free list;
 511  *
 512  * b. Otherwise, trylock it.
 513  */
 514 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 515                                            struct ttm_operation_ctx *ctx,
 516                                            const struct ttm_place *place,
 517                                            bool *locked, bool *busy)
 518 {
 519         bool ret = false;
 520
 521         if (bo->base.resv == ctx->resv) {
 522                 dma_resv_assert_held(bo->base.resv);
 523                 if (ctx->allow_res_evict)
 524                         ret = true;
 525                 *locked = false;
 526                 if (busy)
 527                         *busy = false;
 528         } else {
 529                 ret = dma_resv_trylock(bo->base.resv);
 530                 *locked = ret;
 531                 if (busy)
 532                         *busy = !ret;
 533         }
 534
 535         if (ret && place && (bo->resource->mem_type != place->mem_type ||
 536                 !bo->bdev->funcs->eviction_valuable(bo, place))) {
 537                 ret = false;
 538                 if (*locked) {
 539                         dma_resv_unlock(bo->base.resv);
 540                         *locked = false;
 541                 }
 542         }
 543
 544         return ret;
 545 }
 546
 547 /**
 548  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 549  *
 550  * @busy_bo: BO which couldn't be locked with trylock
 551  * @ctx: operation context
 552  * @ticket: acquire ticket
 553  *
 554  * Try to lock a busy buffer object to avoid failing eviction.
 555  */
 556 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 557                                    struct ttm_operation_ctx *ctx,
 558                                    struct ww_acquire_ctx *ticket)
 559 {
 560         int r;
 561
 562         if (!busy_bo || !ticket)
 563                 return -EBUSY;
 564
 565         if (ctx->interruptible)
 566                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 567                                                           ticket);
 568         else
 569                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 570
 571         /*
 572          * TODO: It would be better to keep the BO locked until allocation is at
 573          * least tried one more time, but that would mean a much larger rework
 574          * of TTM.
 575          */
 576         if (!r)
 577                 dma_resv_unlock(busy_bo->base.resv);
 578
 579         return r == -EDEADLK ? -EBUSY : r;
 580 }
 581
 582 int ttm_mem_evict_first(struct ttm_device *bdev,
 583                         struct ttm_resource_manager *man,
 584                         const struct ttm_place *place,
 585                         struct ttm_operation_ctx *ctx,
 586                         struct ww_acquire_ctx *ticket)
 587 {
 588         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 589         struct ttm_resource_cursor cursor;
 590         struct ttm_resource *res;
 591         bool locked = false;
 592         int ret;
 593
 594         spin_lock(&bdev->lru_lock);
 595         ttm_resource_manager_for_each_res(man, &cursor, res) {
 596                 bool busy;
 597
 598                 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
 599                                                     &locked, &busy)) {
 600                         if (busy && !busy_bo && ticket !=
 601                             dma_resv_locking_ctx(res->bo->base.resv))
 602                                 busy_bo = res->bo;
 603                         continue;
 604                 }
 605
 606                 if (ttm_bo_get_unless_zero(res->bo)) {
 607                         bo = res->bo;
 608                         break;
 609                 }
 610                 if (locked)
 611                         dma_resv_unlock(res->bo->base.resv);
 612         }
 613
 614         if (!bo) {
 615                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 616                         busy_bo = NULL;
 617                 spin_unlock(&bdev->lru_lock);
 618                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 619                 if (busy_bo)
 620                         ttm_bo_put(busy_bo);
 621                 return ret;
 622         }
 623
 624         if (bo->deleted) {
 625                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 626                                           ctx->no_wait_gpu, locked);
 627                 ttm_bo_put(bo);
 628                 return ret;
 629         }
 630
 631         spin_unlock(&bdev->lru_lock);
 632
 633         ret = ttm_bo_evict(bo, ctx);
 634         if (locked)
 635                 ttm_bo_unreserve(bo);
 636         else
 637                 ttm_bo_move_to_lru_tail_unlocked(bo);
 638
 639         ttm_bo_put(bo);
 640         return ret;
 641 }
 642
 643 /**
 644  * ttm_bo_pin - Pin the buffer object.
 645  * @bo: The buffer object to pin
 646  *
 647  * Make sure the buffer is not evicted any more during memory pressure.
 648  * @bo must be unpinned again by calling ttm_bo_unpin().
 649  */
 650 void ttm_bo_pin(struct ttm_buffer_object *bo)
 651 {
 652         dma_resv_assert_held(bo->base.resv);
 653         WARN_ON_ONCE(!kref_read(&bo->kref));
 654         spin_lock(&bo->bdev->lru_lock);
 655         if (bo->resource)
 656                 ttm_resource_del_bulk_move(bo->resource, bo);
 657         ++bo->pin_count;
 658         spin_unlock(&bo->bdev->lru_lock);
 659 }
 660 EXPORT_SYMBOL(ttm_bo_pin);
 661
 662 /**
 663  * ttm_bo_unpin - Unpin the buffer object.
 664  * @bo: The buffer object to unpin
 665  *
 666  * Allows the buffer object to be evicted again during memory pressure.
 667  */
 668 void ttm_bo_unpin(struct ttm_buffer_object *bo)
 669 {
 670         dma_resv_assert_held(bo->base.resv);
 671         WARN_ON_ONCE(!kref_read(&bo->kref));
 672         if (WARN_ON_ONCE(!bo->pin_count))
 673                 return;
 674
 675         spin_lock(&bo->bdev->lru_lock);
 676         --bo->pin_count;
 677         if (bo->resource)
 678                 ttm_resource_add_bulk_move(bo->resource, bo);
 679         spin_unlock(&bo->bdev->lru_lock);
 680 }
 681 EXPORT_SYMBOL(ttm_bo_unpin);
 682
 683 /*
 684  * Add the last move fence to the BO as kernel dependency and reserve a new
 685  * fence slot.
 686  */
 687 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 688                                  struct ttm_resource_manager *man,
 689                                  struct ttm_resource *mem,
 690                                  bool no_wait_gpu)
 691 {
 692         struct dma_fence *fence;
 693         int ret;
 694
 695         spin_lock(&man->move_lock);
 696         fence = dma_fence_get(man->move);
 697         spin_unlock(&man->move_lock);
 698
 699         if (!fence)
 700                 return 0;
 701
 702         if (no_wait_gpu) {
 703                 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
 704                 dma_fence_put(fence);
 705                 return ret;
 706         }
 707
 708         dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
 709
 710         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 711         dma_fence_put(fence);
 712         return ret;
 713 }
 714
 715 /*
 716  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 717  * space, or we've evicted everything and there isn't enough space.
 718  */
 719 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 720                                   const struct ttm_place *place,
 721                                   struct ttm_resource **mem,
 722                                   struct ttm_operation_ctx *ctx)
 723 {
 724         struct ttm_device *bdev = bo->bdev;
 725         struct ttm_resource_manager *man;
 726         struct ww_acquire_ctx *ticket;
 727         int ret;
 728
 729         man = ttm_manager_type(bdev, place->mem_type);
 730         ticket = dma_resv_locking_ctx(bo->base.resv);
 731         do {
 732                 ret = ttm_resource_alloc(bo, place, mem);
 733                 if (likely(!ret))
 734                         break;
 735                 if (unlikely(ret != -ENOSPC))
 736                         return ret;
 737                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 738                                           ticket);
 739                 if (unlikely(ret != 0))
 740                         return ret;
 741         } while (1);
 742
 743         return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 744 }
 745
 746 /**
 747  * ttm_bo_mem_space
 748  *
 749  * @bo: Pointer to a struct ttm_buffer_object. the data of which
 750  * we want to allocate space for.
 751  * @proposed_placement: Proposed new placement for the buffer object.
 752  * @mem: A struct ttm_resource.
 753  * @ctx: if and how to sleep, lock buffers and alloc memory
 754  *
 755  * Allocate memory space for the buffer object pointed to by @bo, using
 756  * the placement flags in @placement, potentially evicting other idle buffer objects.
 757  * This function may sleep while waiting for space to become available.
 758  * Returns:
 759  * -EBUSY: No space available (only if no_wait == 1).
 760  * -ENOMEM: Could not allocate memory for the buffer object, either due to
 761  * fragmentation or concurrent allocators.
 762  * -ERESTARTSYS: An interruptible sleep was interrupted by a signal.
 763  */
 764 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 765                         struct ttm_placement *placement,
 766                         struct ttm_resource **mem,
 767                         struct ttm_operation_ctx *ctx)
 768 {
 769         struct ttm_device *bdev = bo->bdev;
 770         bool type_found = false;
 771         int i, ret;
 772
 773         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 774         if (unlikely(ret))
 775                 return ret;
 776
 777         for (i = 0; i < placement->num_placement; ++i) {
 778                 const struct ttm_place *place = &placement->placement[i];
 779                 struct ttm_resource_manager *man;
 780
 781                 man = ttm_manager_type(bdev, place->mem_type);
 782                 if (!man || !ttm_resource_manager_used(man))
 783                         continue;
 784
 785                 type_found = true;
 786                 ret = ttm_resource_alloc(bo, place, mem);
 787                 if (ret == -ENOSPC)
 788                         continue;
 789                 if (unlikely(ret))
 790                         goto error;
 791
 792                 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 793                 if (unlikely(ret)) {
 794                         ttm_resource_free(bo, mem);
 795                         if (ret == -EBUSY)
 796                                 continue;
 797
 798                         goto error;
 799                 }
 800                 return 0;
 801         }
 802
 803         for (i = 0; i < placement->num_busy_placement; ++i) {
 804                 const struct ttm_place *place = &placement->busy_placement[i];
 805                 struct ttm_resource_manager *man;
 806
 807                 man = ttm_manager_type(bdev, place->mem_type);
 808                 if (!man || !ttm_resource_manager_used(man))
 809                         continue;
 810
 811                 type_found = true;
 812                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 813                 if (likely(!ret))
 814                         return 0;
 815
 816                 if (ret && ret != -EBUSY)
 817                         goto error;
 818         }
 819
 820         ret = -ENOMEM;
 821         if (!type_found) {
 822                 pr_err(TTM_PFX "No compatible memory type found\n");
 823                 ret = -EINVAL;
 824         }
 825
 826 error:
 827         return ret;
 828 }
 829 EXPORT_SYMBOL(ttm_bo_mem_space);
 830
 831 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 832                               struct ttm_placement *placement,
 833                               struct ttm_operation_ctx *ctx)
 834 {
 835         struct ttm_resource *mem;
 836         struct ttm_place hop;
 837         int ret;
 838
 839         dma_resv_assert_held(bo->base.resv);
 840
 841         /*
 842          * Determine where to move the buffer.
 843          *
 844          * If driver determines move is going to need
 845          * an extra step then it will return -EMULTIHOP
 846          * and the buffer will be moved to the temporary
 847          * stop and the driver will be called to make
 848          * the second hop.
 849          */
 850         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 851         if (ret)
 852                 return ret;
 853 bounce:
 854         ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
 855         if (ret == -EMULTIHOP) {
 856                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
 857                 if (ret)
 858                         goto out;
 859                 /* try and move to final place now. */
 860                 goto bounce;
 861         }
 862 out:
 863         if (ret)
 864                 ttm_resource_free(bo, &mem);
 865         return ret;
 866 }
 867
 868 /**
 869  * ttm_bo_validate
 870  *
 871  * @bo: The buffer object.
 872  * @placement: Proposed placement for the buffer object.
 873  * @ctx: validation parameters.
 874  *
 875  * Changes placement and caching policy of the buffer object
 876  * according proposed placement.
 877  * Returns
 878  * -EINVAL on invalid proposed placement.
 879  * -ENOMEM on out-of-memory condition.
 880  * -EBUSY if no_wait is true and buffer busy.
 881  * -ERESTARTSYS if interrupted by a signal.
 882  */
 883 int ttm_bo_validate(struct ttm_buffer_object *bo,
 884                     struct ttm_placement *placement,
 885                     struct ttm_operation_ctx *ctx)
 886 {
 887         int ret;
 888
 889         dma_resv_assert_held(bo->base.resv);
 890
 891         /*
 892          * Remove the backing store if no placement is given.
 893          */
 894         if (!placement->num_placement && !placement->num_busy_placement)
 895                 return ttm_bo_pipeline_gutting(bo);
 896
 897         /*
 898          * Check whether we need to move buffer.
 899          */
 900         if (!bo->resource || !ttm_resource_compat(bo->resource, placement)) {
 901                 ret = ttm_bo_move_buffer(bo, placement, ctx);
 902                 if (ret)
 903                         return ret;
 904         }
 905         /*
 906          * We might need to add a TTM.
 907          */
 908         if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) {
 909                 ret = ttm_tt_create(bo, true);
 910                 if (ret)
 911                         return ret;
 912         }
 913         return 0;
 914 }
 915 EXPORT_SYMBOL(ttm_bo_validate);
 916
 917 /**
 918  * ttm_bo_init_reserved
 919  *
 920  * @bdev: Pointer to a ttm_device struct.
 921  * @bo: Pointer to a ttm_buffer_object to be initialized.
 922  * @type: Requested type of buffer object.
 923  * @placement: Initial placement for buffer object.
 924  * @alignment: Data alignment in pages.
 925  * @ctx: TTM operation context for memory allocation.
 926  * @sg: Scatter-gather table.
 927  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
 928  * @destroy: Destroy function. Use NULL for kfree().
 929  *
 930  * This function initializes a pre-allocated struct ttm_buffer_object.
 931  * As this object may be part of a larger structure, this function,
 932  * together with the @destroy function, enables driver-specific objects
 933  * derived from a ttm_buffer_object.
 934  *
 935  * On successful return, the caller owns an object kref to @bo. The kref and
 936  * list_kref are usually set to 1, but note that in some situations, other
 937  * tasks may already be holding references to @bo as well.
 938  * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
 939  * and it is the caller's responsibility to call ttm_bo_unreserve.
 940  *
 941  * If a failure occurs, the function will call the @destroy function. Thus,
 942  * after a failure, dereferencing @bo is illegal and will likely cause memory
 943  * corruption.
 944  *
 945  * Returns
 946  * -ENOMEM: Out of memory.
 947  * -EINVAL: Invalid placement flags.
 948  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
 949  */
 950 int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 951                          enum ttm_bo_type type, struct ttm_placement *placement,
 952                          uint32_t alignment, struct ttm_operation_ctx *ctx,
 953                          struct sg_table *sg, struct dma_resv *resv,
 954                          void (*destroy) (struct ttm_buffer_object *))
 955 {
 956         static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
 957         int ret;
 958
 959         kref_init(&bo->kref);
 960         bo->bdev = bdev;
 961         bo->type = type;
 962         bo->page_alignment = alignment;
 963         bo->destroy = destroy;
 964         bo->pin_count = 0;
 965         bo->sg = sg;
 966         bo->bulk_move = NULL;
 967         if (resv)
 968                 bo->base.resv = resv;
 969         else
 970                 bo->base.resv = &bo->base._resv;
 971         atomic_inc(&ttm_glob.bo_count);
 972
 973         ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
 974         if (unlikely(ret)) {
 975                 ttm_bo_put(bo);
 976                 return ret;
 977         }
 978
 979         /*
 980          * For ttm_bo_type_device buffers, allocate
 981          * address space from the device.
 982          */
 983         if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
 984                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
 985                                          PFN_UP(bo->base.size));
 986                 if (ret)
 987                         goto err_put;
 988         }
 989
 990         /* passed reservation objects should already be locked,
 991          * since otherwise lockdep will be angered in radeon.
 992          */
 993         if (!resv)
 994                 WARN_ON(!dma_resv_trylock(bo->base.resv));
 995         else
 996                 dma_resv_assert_held(resv);
 997
 998         ret = ttm_bo_validate(bo, placement, ctx);
 999         if (unlikely(ret))
1000                 goto err_unlock;
1001
1002         return 0;
1003
1004 err_unlock:
1005         if (!resv)
1006                 dma_resv_unlock(bo->base.resv);
1007
1008 err_put:
1009         ttm_bo_put(bo);
1010         return ret;
1011 }
1012 EXPORT_SYMBOL(ttm_bo_init_reserved);
1013
1014 /**
1015  * ttm_bo_init_validate
1016  *
1017  * @bdev: Pointer to a ttm_device struct.
1018  * @bo: Pointer to a ttm_buffer_object to be initialized.
1019  * @type: Requested type of buffer object.
1020  * @placement: Initial placement for buffer object.
1021  * @alignment: Data alignment in pages.
1022  * @interruptible: If needing to sleep to wait for GPU resources,
1023  * sleep interruptible.
1024  * pinned in physical memory. If this behaviour is not desired, this member
1025  * holds a pointer to a persistent shmem object. Typically, this would
1026  * point to the shmem object backing a GEM object if TTM is used to back a
1027  * GEM user interface.
1028  * @sg: Scatter-gather table.
1029  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
1030  * @destroy: Destroy function. Use NULL for kfree().
1031  *
1032  * This function initializes a pre-allocated struct ttm_buffer_object.
1033  * As this object may be part of a larger structure, this function,
1034  * together with the @destroy function,
1035  * enables driver-specific objects derived from a ttm_buffer_object.
1036  *
1037  * On successful return, the caller owns an object kref to @bo. The kref and
1038  * list_kref are usually set to 1, but note that in some situations, other
1039  * tasks may already be holding references to @bo as well.
1040  *
1041  * If a failure occurs, the function will call the @destroy function, Thus,
1042  * after a failure, dereferencing @bo is illegal and will likely cause memory
1043  * corruption.
1044  *
1045  * Returns
1046  * -ENOMEM: Out of memory.
1047  * -EINVAL: Invalid placement flags.
1048  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
1049  */
1050 int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
1051                          enum ttm_bo_type type, struct ttm_placement *placement,
1052                          uint32_t alignment, bool interruptible,
1053                          struct sg_table *sg, struct dma_resv *resv,
1054                          void (*destroy) (struct ttm_buffer_object *))
1055 {
1056         struct ttm_operation_ctx ctx = { interruptible, false };
1057         int ret;
1058
1059         ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
1060                                    sg, resv, destroy);
1061         if (ret)
1062                 return ret;
1063
1064         if (!resv)
1065                 ttm_bo_unreserve(bo);
1066
1067         return 0;
1068 }
1069 EXPORT_SYMBOL(ttm_bo_init_validate);
1070
1071 /*
1072  * buffer object vm functions.
1073  */
1074
1075 /**
1076  * ttm_bo_unmap_virtual
1077  *
1078  * @bo: tear down the virtual mappings for this BO
1079  */
1080 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1081 {
1082         struct ttm_device *bdev = bo->bdev;
1083
1084         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1085         ttm_mem_io_free(bdev, bo->resource);
1086 }
1087 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1088
1089 /**
1090  * ttm_bo_wait_ctx - wait for buffer idle.
1091  *
1092  * @bo:  The buffer object.
1093  * @ctx: defines how to wait
1094  *
1095  * Waits for the buffer to be idle. Used timeout depends on the context.
1096  * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or
1097  * zero on success.
1098  */
1099 int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
1100 {
1101         long ret;
1102
1103         if (ctx->no_wait_gpu) {
1104                 if (dma_resv_test_signaled(bo->base.resv,
1105                                            DMA_RESV_USAGE_BOOKKEEP))
1106                         return 0;
1107                 else
1108                         return -EBUSY;
1109         }
1110
1111         ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1112                                     ctx->interruptible, 15 * HZ);
1113         if (unlikely(ret < 0))
1114                 return ret;
1115         if (unlikely(ret == 0))
1116                 return -EBUSY;
1117         return 0;
1118 }
1119 EXPORT_SYMBOL(ttm_bo_wait_ctx);
1120
1121 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1122                    gfp_t gfp_flags)
1123 {
1124         struct ttm_place place;
1125         bool locked;
1126         long ret;
1127
1128         /*
1129          * While the bo may already reside in SYSTEM placement, set
1130          * SYSTEM as new placement to cover also the move further below.
1131          * The driver may use the fact that we're moving from SYSTEM
1132          * as an indication that we're about to swap out.
1133          */
1134         memset(&place, 0, sizeof(place));
1135         place.mem_type = bo->resource->mem_type;
1136         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1137                 return -EBUSY;
1138
1139         if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1140             bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1141             bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1142             !ttm_bo_get_unless_zero(bo)) {
1143                 if (locked)
1144                         dma_resv_unlock(bo->base.resv);
1145                 return -EBUSY;
1146         }
1147
1148         if (bo->deleted) {
1149                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1150                 ttm_bo_put(bo);
1151                 return ret == -EBUSY ? -ENOSPC : ret;
1152         }
1153
1154         /* TODO: Cleanup the locking */
1155         spin_unlock(&bo->bdev->lru_lock);
1156
1157         /*
1158          * Move to system cached
1159          */
1160         if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1161                 struct ttm_operation_ctx ctx = { false, false };
1162                 struct ttm_resource *evict_mem;
1163                 struct ttm_place hop;
1164
1165                 memset(&hop, 0, sizeof(hop));
1166                 place.mem_type = TTM_PL_SYSTEM;
1167                 ret = ttm_resource_alloc(bo, &place, &evict_mem);
1168                 if (unlikely(ret))
1169                         goto out;
1170
1171                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop);
1172                 if (unlikely(ret != 0)) {
1173                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1174                         goto out;
1175                 }
1176         }
1177
1178         /*
1179          * Make sure BO is idle.
1180          */
1181         ret = ttm_bo_wait_ctx(bo, ctx);
1182         if (unlikely(ret != 0))
1183                 goto out;
1184
1185         ttm_bo_unmap_virtual(bo);
1186
1187         /*
1188          * Swap out. Buffer will be swapped in again as soon as
1189          * anyone tries to access a ttm page.
1190          */
1191         if (bo->bdev->funcs->swap_notify)
1192                 bo->bdev->funcs->swap_notify(bo);
1193
1194         if (ttm_tt_is_populated(bo->ttm))
1195                 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1196 out:
1197
1198         /*
1199          * Unreserve without putting on LRU to avoid swapping out an
1200          * already swapped buffer.
1201          */
1202         if (locked)
1203                 dma_resv_unlock(bo->base.resv);
1204         ttm_bo_put(bo);
1205         return ret == -EBUSY ? -ENOSPC : ret;
1206 }
1207
1208 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1209 {
1210         if (bo->ttm == NULL)
1211                 return;
1212
1213         ttm_tt_unpopulate(bo->bdev, bo->ttm);
1214         ttm_tt_destroy(bo->bdev, bo->ttm);
1215         bo->ttm = NULL;
1216 }