drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <drm/ttm/ttm_tt.h>
  37
  38 #include <linux/jiffies.h>
  39 #include <linux/slab.h>
  40 #include <linux/sched.h>
  41 #include <linux/mm.h>
  42 #include <linux/file.h>
  43 #include <linux/module.h>
  44 #include <linux/atomic.h>
  45 #include <linux/dma-resv.h>
  46
  47 #include "ttm_module.h"
  48
  49 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  50                                         struct ttm_placement *placement)
  51 {
  52         struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX);
  53         struct ttm_resource_manager *man;
  54         int i, mem_type;
  55
  56         for (i = 0; i < placement->num_placement; i++) {
  57                 mem_type = placement->placement[i].mem_type;
  58                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  59                            i, placement->placement[i].flags, mem_type);
  60                 man = ttm_manager_type(bo->bdev, mem_type);
  61                 ttm_resource_manager_debug(man, &p);
  62         }
  63 }
  64
  65 /**
  66  * ttm_bo_move_to_lru_tail
  67  *
  68  * @bo: The buffer object.
  69  *
  70  * Move this BO to the tail of all lru lists used to lookup and reserve an
  71  * object. This function must be called with struct ttm_global::lru_lock
  72  * held, and is used to make a BO less likely to be considered for eviction.
  73  */
  74 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
  75 {
  76         dma_resv_assert_held(bo->base.resv);
  77
  78         if (bo->resource)
  79                 ttm_resource_move_to_lru_tail(bo->resource);
  80 }
  81 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
  82
  83 /**
  84  * ttm_bo_set_bulk_move - update BOs bulk move object
  85  *
  86  * @bo: The buffer object.
  87  * @bulk: bulk move structure
  88  *
  89  * Update the BOs bulk move object, making sure that resources are added/removed
  90  * as well. A bulk move allows to move many resource on the LRU at once,
  91  * resulting in much less overhead of maintaining the LRU.
  92  * The only requirement is that the resources stay together on the LRU and are
  93  * never separated. This is enforces by setting the bulk_move structure on a BO.
  94  * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
  95  * their LRU list.
  96  */
  97 void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
  98                           struct ttm_lru_bulk_move *bulk)
  99 {
 100         dma_resv_assert_held(bo->base.resv);
 101
 102         if (bo->bulk_move == bulk)
 103                 return;
 104
 105         spin_lock(&bo->bdev->lru_lock);
 106         if (bo->resource)
 107                 ttm_resource_del_bulk_move(bo->resource, bo);
 108         bo->bulk_move = bulk;
 109         if (bo->resource)
 110                 ttm_resource_add_bulk_move(bo->resource, bo);
 111         spin_unlock(&bo->bdev->lru_lock);
 112 }
 113 EXPORT_SYMBOL(ttm_bo_set_bulk_move);
 114
 115 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 116                                   struct ttm_resource *mem, bool evict,
 117                                   struct ttm_operation_ctx *ctx,
 118                                   struct ttm_place *hop)
 119 {
 120         struct ttm_device *bdev = bo->bdev;
 121         bool old_use_tt, new_use_tt;
 122         int ret;
 123
 124         old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
 125         new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 126
 127         ttm_bo_unmap_virtual(bo);
 128
 129         /*
 130          * Create and bind a ttm if required.
 131          */
 132
 133         if (new_use_tt) {
 134                 /* Zero init the new TTM structure if the old location should
 135                  * have used one as well.
 136                  */
 137                 ret = ttm_tt_create(bo, old_use_tt);
 138                 if (ret)
 139                         goto out_err;
 140
 141                 if (mem->mem_type != TTM_PL_SYSTEM) {
 142                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 143                         if (ret)
 144                                 goto out_err;
 145                 }
 146         }
 147
 148         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 149         if (ret)
 150                 goto out_err;
 151
 152         ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
 153         if (ret) {
 154                 if (ret == -EMULTIHOP)
 155                         return ret;
 156                 goto out_err;
 157         }
 158
 159         ctx->bytes_moved += bo->base.size;
 160         return 0;
 161
 162 out_err:
 163         if (!old_use_tt)
 164                 ttm_bo_tt_destroy(bo);
 165
 166         return ret;
 167 }
 168
 169 /*
 170  * Call bo::reserved.
 171  * Will release GPU memory type usage on destruction.
 172  * This is the place to put in driver specific hooks to release
 173  * driver private resources.
 174  * Will release the bo::reserved lock.
 175  */
 176
 177 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 178 {
 179         if (bo->bdev->funcs->delete_mem_notify)
 180                 bo->bdev->funcs->delete_mem_notify(bo);
 181
 182         ttm_bo_tt_destroy(bo);
 183         ttm_resource_free(bo, &bo->resource);
 184 }
 185
 186 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 187 {
 188         int r;
 189
 190         if (bo->base.resv == &bo->base._resv)
 191                 return 0;
 192
 193         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 194
 195         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 196         dma_resv_unlock(&bo->base._resv);
 197         if (r)
 198                 return r;
 199
 200         if (bo->type != ttm_bo_type_sg) {
 201                 /* This works because the BO is about to be destroyed and nobody
 202                  * reference it any more. The only tricky case is the trylock on
 203                  * the resv object while holding the lru_lock.
 204                  */
 205                 spin_lock(&bo->bdev->lru_lock);
 206                 bo->base.resv = &bo->base._resv;
 207                 spin_unlock(&bo->bdev->lru_lock);
 208         }
 209
 210         return r;
 211 }
 212
 213 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 214 {
 215         struct dma_resv *resv = &bo->base._resv;
 216         struct dma_resv_iter cursor;
 217         struct dma_fence *fence;
 218
 219         dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
 220         dma_resv_for_each_fence_unlocked(&cursor, fence) {
 221                 if (!fence->ops->signaled)
 222                         dma_fence_enable_sw_signaling(fence);
 223         }
 224         dma_resv_iter_end(&cursor);
 225 }
 226
 227 /**
 228  * ttm_bo_cleanup_refs
 229  * If bo idle, remove from lru lists, and unref.
 230  * If not idle, block if possible.
 231  *
 232  * Must be called with lru_lock and reservation held, this function
 233  * will drop the lru lock and optionally the reservation lock before returning.
 234  *
 235  * @bo:                    The buffer object to clean-up
 236  * @interruptible:         Any sleeps should occur interruptibly.
 237  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 238  * @unlock_resv:           Unlock the reservation lock as well.
 239  */
 240
 241 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 242                                bool interruptible, bool no_wait_gpu,
 243                                bool unlock_resv)
 244 {
 245         struct dma_resv *resv = &bo->base._resv;
 246         int ret;
 247
 248         if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
 249                 ret = 0;
 250         else
 251                 ret = -EBUSY;
 252
 253         if (ret && !no_wait_gpu) {
 254                 long lret;
 255
 256                 if (unlock_resv)
 257                         dma_resv_unlock(bo->base.resv);
 258                 spin_unlock(&bo->bdev->lru_lock);
 259
 260                 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
 261                                              interruptible,
 262                                              30 * HZ);
 263
 264                 if (lret < 0)
 265                         return lret;
 266                 else if (lret == 0)
 267                         return -EBUSY;
 268
 269                 spin_lock(&bo->bdev->lru_lock);
 270                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 271                         /*
 272                          * We raced, and lost, someone else holds the reservation now,
 273                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 274                          *
 275                          * Even if it's not the case, because we finished waiting any
 276                          * delayed destruction would succeed, so just return success
 277                          * here.
 278                          */
 279                         spin_unlock(&bo->bdev->lru_lock);
 280                         return 0;
 281                 }
 282                 ret = 0;
 283         }
 284
 285         if (ret) {
 286                 if (unlock_resv)
 287                         dma_resv_unlock(bo->base.resv);
 288                 spin_unlock(&bo->bdev->lru_lock);
 289                 return ret;
 290         }
 291
 292         spin_unlock(&bo->bdev->lru_lock);
 293         ttm_bo_cleanup_memtype_use(bo);
 294
 295         if (unlock_resv)
 296                 dma_resv_unlock(bo->base.resv);
 297
 298         return 0;
 299 }
 300
 301 /*
 302  * Block for the dma_resv object to become idle, lock the buffer and clean up
 303  * the resource and tt object.
 304  */
 305 static void ttm_bo_delayed_delete(struct work_struct *work)
 306 {
 307         struct ttm_buffer_object *bo;
 308
 309         bo = container_of(work, typeof(*bo), delayed_delete);
 310
 311         dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
 312                               MAX_SCHEDULE_TIMEOUT);
 313         dma_resv_lock(bo->base.resv, NULL);
 314         ttm_bo_cleanup_memtype_use(bo);
 315         dma_resv_unlock(bo->base.resv);
 316         ttm_bo_put(bo);
 317 }
 318
 319 static void ttm_bo_release(struct kref *kref)
 320 {
 321         struct ttm_buffer_object *bo =
 322             container_of(kref, struct ttm_buffer_object, kref);
 323         struct ttm_device *bdev = bo->bdev;
 324         int ret;
 325
 326         WARN_ON_ONCE(bo->pin_count);
 327         WARN_ON_ONCE(bo->bulk_move);
 328
 329         if (!bo->deleted) {
 330                 ret = ttm_bo_individualize_resv(bo);
 331                 if (ret) {
 332                         /* Last resort, if we fail to allocate memory for the
 333                          * fences block for the BO to become idle
 334                          */
 335                         dma_resv_wait_timeout(bo->base.resv,
 336                                               DMA_RESV_USAGE_BOOKKEEP, false,
 337                                               30 * HZ);
 338                 }
 339
 340                 if (bo->bdev->funcs->release_notify)
 341                         bo->bdev->funcs->release_notify(bo);
 342
 343                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 344                 ttm_mem_io_free(bdev, bo->resource);
 345
 346                 if (!dma_resv_test_signaled(bo->base.resv,
 347                                             DMA_RESV_USAGE_BOOKKEEP) ||
 348                     (want_init_on_free() && (bo->ttm != NULL)) ||
 349                     !dma_resv_trylock(bo->base.resv)) {
 350                         /* The BO is not idle, resurrect it for delayed destroy */
 351                         ttm_bo_flush_all_fences(bo);
 352                         bo->deleted = true;
 353
 354                         spin_lock(&bo->bdev->lru_lock);
 355
 356                         /*
 357                          * Make pinned bos immediately available to
 358                          * shrinkers, now that they are queued for
 359                          * destruction.
 360                          *
 361                          * FIXME: QXL is triggering this. Can be removed when the
 362                          * driver is fixed.
 363                          */
 364                         if (bo->pin_count) {
 365                                 bo->pin_count = 0;
 366                                 ttm_resource_move_to_lru_tail(bo->resource);
 367                         }
 368
 369                         kref_init(&bo->kref);
 370                         spin_unlock(&bo->bdev->lru_lock);
 371
 372                         INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);
 373
 374                         /* Schedule the worker on the closest NUMA node. This
 375                          * improves performance since system memory might be
 376                          * cleared on free and that is best done on a CPU core
 377                          * close to it.
 378                          */
 379                         queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete);
 380                         return;
 381                 }
 382
 383                 ttm_bo_cleanup_memtype_use(bo);
 384                 dma_resv_unlock(bo->base.resv);
 385         }
 386
 387         atomic_dec(&ttm_glob.bo_count);
 388         bo->destroy(bo);
 389 }
 390
 391 /**
 392  * ttm_bo_put
 393  *
 394  * @bo: The buffer object.
 395  *
 396  * Unreference a buffer object.
 397  */
 398 void ttm_bo_put(struct ttm_buffer_object *bo)
 399 {
 400         kref_put(&bo->kref, ttm_bo_release);
 401 }
 402 EXPORT_SYMBOL(ttm_bo_put);
 403
 404 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 405                                      struct ttm_resource **mem,
 406                                      struct ttm_operation_ctx *ctx,
 407                                      struct ttm_place *hop)
 408 {
 409         struct ttm_placement hop_placement;
 410         struct ttm_resource *hop_mem;
 411         int ret;
 412
 413         hop_placement.num_placement = 1;
 414         hop_placement.placement = hop;
 415
 416         /* find space in the bounce domain */
 417         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 418         if (ret)
 419                 return ret;
 420         /* move to the bounce domain */
 421         ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
 422         if (ret) {
 423                 ttm_resource_free(bo, &hop_mem);
 424                 return ret;
 425         }
 426         return 0;
 427 }
 428
 429 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 430                         struct ttm_operation_ctx *ctx)
 431 {
 432         struct ttm_device *bdev = bo->bdev;
 433         struct ttm_resource *evict_mem;
 434         struct ttm_placement placement;
 435         struct ttm_place hop;
 436         int ret = 0;
 437
 438         memset(&hop, 0, sizeof(hop));
 439
 440         dma_resv_assert_held(bo->base.resv);
 441
 442         placement.num_placement = 0;
 443         bdev->funcs->evict_flags(bo, &placement);
 444
 445         if (!placement.num_placement) {
 446                 ret = ttm_bo_wait_ctx(bo, ctx);
 447                 if (ret)
 448                         return ret;
 449
 450                 /*
 451                  * Since we've already synced, this frees backing store
 452                  * immediately.
 453                  */
 454                 return ttm_bo_pipeline_gutting(bo);
 455         }
 456
 457         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 458         if (ret) {
 459                 if (ret != -ERESTARTSYS) {
 460                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 461                                bo);
 462                         ttm_bo_mem_space_debug(bo, &placement);
 463                 }
 464                 goto out;
 465         }
 466
 467         do {
 468                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
 469                 if (ret != -EMULTIHOP)
 470                         break;
 471
 472                 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
 473         } while (!ret);
 474
 475         if (ret) {
 476                 ttm_resource_free(bo, &evict_mem);
 477                 if (ret != -ERESTARTSYS && ret != -EINTR)
 478                         pr_err("Buffer eviction failed\n");
 479         }
 480 out:
 481         return ret;
 482 }
 483
 484 /**
 485  * ttm_bo_eviction_valuable
 486  *
 487  * @bo: The buffer object to evict
 488  * @place: the placement we need to make room for
 489  *
 490  * Check if it is valuable to evict the BO to make room for the given placement.
 491  */
 492 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 493                               const struct ttm_place *place)
 494 {
 495         struct ttm_resource *res = bo->resource;
 496         struct ttm_device *bdev = bo->bdev;
 497
 498         dma_resv_assert_held(bo->base.resv);
 499         if (bo->resource->mem_type == TTM_PL_SYSTEM)
 500                 return true;
 501
 502         /* Don't evict this BO if it's outside of the
 503          * requested placement range
 504          */
 505         return ttm_resource_intersects(bdev, res, place, bo->base.size);
 506 }
 507 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 508
 509 /*
 510  * Check the target bo is allowable to be evicted or swapout, including cases:
 511  *
 512  * a. if share same reservation object with ctx->resv, have assumption
 513  * reservation objects should already be locked, so not lock again and
 514  * return true directly when either the opreation allow_reserved_eviction
 515  * or the target bo already is in delayed free list;
 516  *
 517  * b. Otherwise, trylock it.
 518  */
 519 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 520                                            struct ttm_operation_ctx *ctx,
 521                                            const struct ttm_place *place,
 522                                            bool *locked, bool *busy)
 523 {
 524         bool ret = false;
 525
 526         if (bo->pin_count) {
 527                 *locked = false;
 528                 if (busy)
 529                         *busy = false;
 530                 return false;
 531         }
 532
 533         if (bo->base.resv == ctx->resv) {
 534                 dma_resv_assert_held(bo->base.resv);
 535                 if (ctx->allow_res_evict)
 536                         ret = true;
 537                 *locked = false;
 538                 if (busy)
 539                         *busy = false;
 540         } else {
 541                 ret = dma_resv_trylock(bo->base.resv);
 542                 *locked = ret;
 543                 if (busy)
 544                         *busy = !ret;
 545         }
 546
 547         if (ret && place && (bo->resource->mem_type != place->mem_type ||
 548                 !bo->bdev->funcs->eviction_valuable(bo, place))) {
 549                 ret = false;
 550                 if (*locked) {
 551                         dma_resv_unlock(bo->base.resv);
 552                         *locked = false;
 553                 }
 554         }
 555
 556         return ret;
 557 }
 558
 559 /**
 560  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 561  *
 562  * @busy_bo: BO which couldn't be locked with trylock
 563  * @ctx: operation context
 564  * @ticket: acquire ticket
 565  *
 566  * Try to lock a busy buffer object to avoid failing eviction.
 567  */
 568 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 569                                    struct ttm_operation_ctx *ctx,
 570                                    struct ww_acquire_ctx *ticket)
 571 {
 572         int r;
 573
 574         if (!busy_bo || !ticket)
 575                 return -EBUSY;
 576
 577         if (ctx->interruptible)
 578                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 579                                                           ticket);
 580         else
 581                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 582
 583         /*
 584          * TODO: It would be better to keep the BO locked until allocation is at
 585          * least tried one more time, but that would mean a much larger rework
 586          * of TTM.
 587          */
 588         if (!r)
 589                 dma_resv_unlock(busy_bo->base.resv);
 590
 591         return r == -EDEADLK ? -EBUSY : r;
 592 }
 593
 594 int ttm_mem_evict_first(struct ttm_device *bdev,
 595                         struct ttm_resource_manager *man,
 596                         const struct ttm_place *place,
 597                         struct ttm_operation_ctx *ctx,
 598                         struct ww_acquire_ctx *ticket)
 599 {
 600         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 601         struct ttm_resource_cursor cursor;
 602         struct ttm_resource *res;
 603         bool locked = false;
 604         int ret;
 605
 606         spin_lock(&bdev->lru_lock);
 607         ttm_resource_manager_for_each_res(man, &cursor, res) {
 608                 bool busy;
 609
 610                 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
 611                                                     &locked, &busy)) {
 612                         if (busy && !busy_bo && ticket !=
 613                             dma_resv_locking_ctx(res->bo->base.resv))
 614                                 busy_bo = res->bo;
 615                         continue;
 616                 }
 617
 618                 if (ttm_bo_get_unless_zero(res->bo)) {
 619                         bo = res->bo;
 620                         break;
 621                 }
 622                 if (locked)
 623                         dma_resv_unlock(res->bo->base.resv);
 624         }
 625
 626         if (!bo) {
 627                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 628                         busy_bo = NULL;
 629                 spin_unlock(&bdev->lru_lock);
 630                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 631                 if (busy_bo)
 632                         ttm_bo_put(busy_bo);
 633                 return ret;
 634         }
 635
 636         if (bo->deleted) {
 637                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 638                                           ctx->no_wait_gpu, locked);
 639                 ttm_bo_put(bo);
 640                 return ret;
 641         }
 642
 643         spin_unlock(&bdev->lru_lock);
 644
 645         ret = ttm_bo_evict(bo, ctx);
 646         if (locked)
 647                 ttm_bo_unreserve(bo);
 648         else
 649                 ttm_bo_move_to_lru_tail_unlocked(bo);
 650
 651         ttm_bo_put(bo);
 652         return ret;
 653 }
 654
 655 /**
 656  * ttm_bo_pin - Pin the buffer object.
 657  * @bo: The buffer object to pin
 658  *
 659  * Make sure the buffer is not evicted any more during memory pressure.
 660  * @bo must be unpinned again by calling ttm_bo_unpin().
 661  */
 662 void ttm_bo_pin(struct ttm_buffer_object *bo)
 663 {
 664         dma_resv_assert_held(bo->base.resv);
 665         WARN_ON_ONCE(!kref_read(&bo->kref));
 666         spin_lock(&bo->bdev->lru_lock);
 667         if (bo->resource)
 668                 ttm_resource_del_bulk_move(bo->resource, bo);
 669         ++bo->pin_count;
 670         spin_unlock(&bo->bdev->lru_lock);
 671 }
 672 EXPORT_SYMBOL(ttm_bo_pin);
 673
 674 /**
 675  * ttm_bo_unpin - Unpin the buffer object.
 676  * @bo: The buffer object to unpin
 677  *
 678  * Allows the buffer object to be evicted again during memory pressure.
 679  */
 680 void ttm_bo_unpin(struct ttm_buffer_object *bo)
 681 {
 682         dma_resv_assert_held(bo->base.resv);
 683         WARN_ON_ONCE(!kref_read(&bo->kref));
 684         if (WARN_ON_ONCE(!bo->pin_count))
 685                 return;
 686
 687         spin_lock(&bo->bdev->lru_lock);
 688         --bo->pin_count;
 689         if (bo->resource)
 690                 ttm_resource_add_bulk_move(bo->resource, bo);
 691         spin_unlock(&bo->bdev->lru_lock);
 692 }
 693 EXPORT_SYMBOL(ttm_bo_unpin);
 694
 695 /*
 696  * Add the last move fence to the BO as kernel dependency and reserve a new
 697  * fence slot.
 698  */
 699 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 700                                  struct ttm_resource_manager *man,
 701                                  struct ttm_resource *mem,
 702                                  bool no_wait_gpu)
 703 {
 704         struct dma_fence *fence;
 705         int ret;
 706
 707         spin_lock(&man->move_lock);
 708         fence = dma_fence_get(man->move);
 709         spin_unlock(&man->move_lock);
 710
 711         if (!fence)
 712                 return 0;
 713
 714         if (no_wait_gpu) {
 715                 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
 716                 dma_fence_put(fence);
 717                 return ret;
 718         }
 719
 720         dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
 721
 722         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 723         dma_fence_put(fence);
 724         return ret;
 725 }
 726
 727 /*
 728  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 729  * space, or we've evicted everything and there isn't enough space.
 730  */
 731 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 732                                   const struct ttm_place *place,
 733                                   struct ttm_resource **mem,
 734                                   struct ttm_operation_ctx *ctx)
 735 {
 736         struct ttm_device *bdev = bo->bdev;
 737         struct ttm_resource_manager *man;
 738         struct ww_acquire_ctx *ticket;
 739         int ret;
 740
 741         man = ttm_manager_type(bdev, place->mem_type);
 742         ticket = dma_resv_locking_ctx(bo->base.resv);
 743         do {
 744                 ret = ttm_resource_alloc(bo, place, mem);
 745                 if (likely(!ret))
 746                         break;
 747                 if (unlikely(ret != -ENOSPC))
 748                         return ret;
 749                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 750                                           ticket);
 751                 if (unlikely(ret != 0))
 752                         return ret;
 753         } while (1);
 754
 755         return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 756 }
 757
 758 /**
 759  * ttm_bo_mem_space
 760  *
 761  * @bo: Pointer to a struct ttm_buffer_object. the data of which
 762  * we want to allocate space for.
 763  * @placement: Proposed new placement for the buffer object.
 764  * @mem: A struct ttm_resource.
 765  * @ctx: if and how to sleep, lock buffers and alloc memory
 766  *
 767  * Allocate memory space for the buffer object pointed to by @bo, using
 768  * the placement flags in @placement, potentially evicting other idle buffer objects.
 769  * This function may sleep while waiting for space to become available.
 770  * Returns:
 771  * -EBUSY: No space available (only if no_wait == 1).
 772  * -ENOSPC: Could not allocate space for the buffer object, either due to
 773  * fragmentation or concurrent allocators.
 774  * -ERESTARTSYS: An interruptible sleep was interrupted by a signal.
 775  */
 776 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 777                         struct ttm_placement *placement,
 778                         struct ttm_resource **mem,
 779                         struct ttm_operation_ctx *ctx)
 780 {
 781         struct ttm_device *bdev = bo->bdev;
 782         bool type_found = false;
 783         int i, ret;
 784
 785         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 786         if (unlikely(ret))
 787                 return ret;
 788
 789         for (i = 0; i < placement->num_placement; ++i) {
 790                 const struct ttm_place *place = &placement->placement[i];
 791                 struct ttm_resource_manager *man;
 792
 793                 if (place->flags & TTM_PL_FLAG_FALLBACK)
 794                         continue;
 795
 796                 man = ttm_manager_type(bdev, place->mem_type);
 797                 if (!man || !ttm_resource_manager_used(man))
 798                         continue;
 799
 800                 type_found = true;
 801                 ret = ttm_resource_alloc(bo, place, mem);
 802                 if (ret == -ENOSPC)
 803                         continue;
 804                 if (unlikely(ret))
 805                         goto error;
 806
 807                 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 808                 if (unlikely(ret)) {
 809                         ttm_resource_free(bo, mem);
 810                         if (ret == -EBUSY)
 811                                 continue;
 812
 813                         goto error;
 814                 }
 815                 return 0;
 816         }
 817
 818         for (i = 0; i < placement->num_placement; ++i) {
 819                 const struct ttm_place *place = &placement->placement[i];
 820                 struct ttm_resource_manager *man;
 821
 822                 if (place->flags & TTM_PL_FLAG_DESIRED)
 823                         continue;
 824
 825                 man = ttm_manager_type(bdev, place->mem_type);
 826                 if (!man || !ttm_resource_manager_used(man))
 827                         continue;
 828
 829                 type_found = true;
 830                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 831                 if (likely(!ret))
 832                         return 0;
 833
 834                 if (ret && ret != -EBUSY)
 835                         goto error;
 836         }
 837
 838         ret = -ENOSPC;
 839         if (!type_found) {
 840                 pr_err(TTM_PFX "No compatible memory type found\n");
 841                 ret = -EINVAL;
 842         }
 843
 844 error:
 845         return ret;
 846 }
 847 EXPORT_SYMBOL(ttm_bo_mem_space);
 848
 849 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 850                               struct ttm_placement *placement,
 851                               struct ttm_operation_ctx *ctx)
 852 {
 853         struct ttm_resource *mem;
 854         struct ttm_place hop;
 855         int ret;
 856
 857         dma_resv_assert_held(bo->base.resv);
 858
 859         /*
 860          * Determine where to move the buffer.
 861          *
 862          * If driver determines move is going to need
 863          * an extra step then it will return -EMULTIHOP
 864          * and the buffer will be moved to the temporary
 865          * stop and the driver will be called to make
 866          * the second hop.
 867          */
 868         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 869         if (ret)
 870                 return ret;
 871 bounce:
 872         ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
 873         if (ret == -EMULTIHOP) {
 874                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
 875                 if (ret)
 876                         goto out;
 877                 /* try and move to final place now. */
 878                 goto bounce;
 879         }
 880 out:
 881         if (ret)
 882                 ttm_resource_free(bo, &mem);
 883         return ret;
 884 }
 885
 886 /**
 887  * ttm_bo_validate
 888  *
 889  * @bo: The buffer object.
 890  * @placement: Proposed placement for the buffer object.
 891  * @ctx: validation parameters.
 892  *
 893  * Changes placement and caching policy of the buffer object
 894  * according proposed placement.
 895  * Returns
 896  * -EINVAL on invalid proposed placement.
 897  * -ENOMEM on out-of-memory condition.
 898  * -EBUSY if no_wait is true and buffer busy.
 899  * -ERESTARTSYS if interrupted by a signal.
 900  */
 901 int ttm_bo_validate(struct ttm_buffer_object *bo,
 902                     struct ttm_placement *placement,
 903                     struct ttm_operation_ctx *ctx)
 904 {
 905         int ret;
 906
 907         dma_resv_assert_held(bo->base.resv);
 908
 909         /*
 910          * Remove the backing store if no placement is given.
 911          */
 912         if (!placement->num_placement)
 913                 return ttm_bo_pipeline_gutting(bo);
 914
 915         /* Check whether we need to move buffer. */
 916         if (bo->resource && ttm_resource_compatible(bo->resource, placement))
 917                 return 0;
 918
 919         /* Moving of pinned BOs is forbidden */
 920         if (bo->pin_count)
 921                 return -EINVAL;
 922
 923         ret = ttm_bo_move_buffer(bo, placement, ctx);
 924         /* For backward compatibility with userspace */
 925         if (ret == -ENOSPC)
 926                 return -ENOMEM;
 927         if (ret)
 928                 return ret;
 929
 930         /*
 931          * We might need to add a TTM.
 932          */
 933         if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) {
 934                 ret = ttm_tt_create(bo, true);
 935                 if (ret)
 936                         return ret;
 937         }
 938         return 0;
 939 }
 940 EXPORT_SYMBOL(ttm_bo_validate);
 941
 942 /**
 943  * ttm_bo_init_reserved
 944  *
 945  * @bdev: Pointer to a ttm_device struct.
 946  * @bo: Pointer to a ttm_buffer_object to be initialized.
 947  * @type: Requested type of buffer object.
 948  * @placement: Initial placement for buffer object.
 949  * @alignment: Data alignment in pages.
 950  * @ctx: TTM operation context for memory allocation.
 951  * @sg: Scatter-gather table.
 952  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
 953  * @destroy: Destroy function. Use NULL for kfree().
 954  *
 955  * This function initializes a pre-allocated struct ttm_buffer_object.
 956  * As this object may be part of a larger structure, this function,
 957  * together with the @destroy function, enables driver-specific objects
 958  * derived from a ttm_buffer_object.
 959  *
 960  * On successful return, the caller owns an object kref to @bo. The kref and
 961  * list_kref are usually set to 1, but note that in some situations, other
 962  * tasks may already be holding references to @bo as well.
 963  * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
 964  * and it is the caller's responsibility to call ttm_bo_unreserve.
 965  *
 966  * If a failure occurs, the function will call the @destroy function. Thus,
 967  * after a failure, dereferencing @bo is illegal and will likely cause memory
 968  * corruption.
 969  *
 970  * Returns
 971  * -ENOMEM: Out of memory.
 972  * -EINVAL: Invalid placement flags.
 973  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
 974  */
 975 int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 976                          enum ttm_bo_type type, struct ttm_placement *placement,
 977                          uint32_t alignment, struct ttm_operation_ctx *ctx,
 978                          struct sg_table *sg, struct dma_resv *resv,
 979                          void (*destroy) (struct ttm_buffer_object *))
 980 {
 981         int ret;
 982
 983         kref_init(&bo->kref);
 984         bo->bdev = bdev;
 985         bo->type = type;
 986         bo->page_alignment = alignment;
 987         bo->destroy = destroy;
 988         bo->pin_count = 0;
 989         bo->sg = sg;
 990         bo->bulk_move = NULL;
 991         if (resv)
 992                 bo->base.resv = resv;
 993         else
 994                 bo->base.resv = &bo->base._resv;
 995         atomic_inc(&ttm_glob.bo_count);
 996
 997         /*
 998          * For ttm_bo_type_device buffers, allocate
 999          * address space from the device.
1000          */
1001         if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
1002                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1003                                          PFN_UP(bo->base.size));
1004                 if (ret)
1005                         goto err_put;
1006         }
1007
1008         /* passed reservation objects should already be locked,
1009          * since otherwise lockdep will be angered in radeon.
1010          */
1011         if (!resv)
1012                 WARN_ON(!dma_resv_trylock(bo->base.resv));
1013         else
1014                 dma_resv_assert_held(resv);
1015
1016         ret = ttm_bo_validate(bo, placement, ctx);
1017         if (unlikely(ret))
1018                 goto err_unlock;
1019
1020         return 0;
1021
1022 err_unlock:
1023         if (!resv)
1024                 dma_resv_unlock(bo->base.resv);
1025
1026 err_put:
1027         ttm_bo_put(bo);
1028         return ret;
1029 }
1030 EXPORT_SYMBOL(ttm_bo_init_reserved);
1031
1032 /**
1033  * ttm_bo_init_validate
1034  *
1035  * @bdev: Pointer to a ttm_device struct.
1036  * @bo: Pointer to a ttm_buffer_object to be initialized.
1037  * @type: Requested type of buffer object.
1038  * @placement: Initial placement for buffer object.
1039  * @alignment: Data alignment in pages.
1040  * @interruptible: If needing to sleep to wait for GPU resources,
1041  * sleep interruptible.
1042  * pinned in physical memory. If this behaviour is not desired, this member
1043  * holds a pointer to a persistent shmem object. Typically, this would
1044  * point to the shmem object backing a GEM object if TTM is used to back a
1045  * GEM user interface.
1046  * @sg: Scatter-gather table.
1047  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
1048  * @destroy: Destroy function. Use NULL for kfree().
1049  *
1050  * This function initializes a pre-allocated struct ttm_buffer_object.
1051  * As this object may be part of a larger structure, this function,
1052  * together with the @destroy function,
1053  * enables driver-specific objects derived from a ttm_buffer_object.
1054  *
1055  * On successful return, the caller owns an object kref to @bo. The kref and
1056  * list_kref are usually set to 1, but note that in some situations, other
1057  * tasks may already be holding references to @bo as well.
1058  *
1059  * If a failure occurs, the function will call the @destroy function, Thus,
1060  * after a failure, dereferencing @bo is illegal and will likely cause memory
1061  * corruption.
1062  *
1063  * Returns
1064  * -ENOMEM: Out of memory.
1065  * -EINVAL: Invalid placement flags.
1066  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
1067  */
1068 int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
1069                          enum ttm_bo_type type, struct ttm_placement *placement,
1070                          uint32_t alignment, bool interruptible,
1071                          struct sg_table *sg, struct dma_resv *resv,
1072                          void (*destroy) (struct ttm_buffer_object *))
1073 {
1074         struct ttm_operation_ctx ctx = { interruptible, false };
1075         int ret;
1076
1077         ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
1078                                    sg, resv, destroy);
1079         if (ret)
1080                 return ret;
1081
1082         if (!resv)
1083                 ttm_bo_unreserve(bo);
1084
1085         return 0;
1086 }
1087 EXPORT_SYMBOL(ttm_bo_init_validate);
1088
1089 /*
1090  * buffer object vm functions.
1091  */
1092
1093 /**
1094  * ttm_bo_unmap_virtual
1095  *
1096  * @bo: tear down the virtual mappings for this BO
1097  */
1098 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1099 {
1100         struct ttm_device *bdev = bo->bdev;
1101
1102         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1103         ttm_mem_io_free(bdev, bo->resource);
1104 }
1105 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1106
1107 /**
1108  * ttm_bo_wait_ctx - wait for buffer idle.
1109  *
1110  * @bo:  The buffer object.
1111  * @ctx: defines how to wait
1112  *
1113  * Waits for the buffer to be idle. Used timeout depends on the context.
1114  * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or
1115  * zero on success.
1116  */
1117 int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
1118 {
1119         long ret;
1120
1121         if (ctx->no_wait_gpu) {
1122                 if (dma_resv_test_signaled(bo->base.resv,
1123                                            DMA_RESV_USAGE_BOOKKEEP))
1124                         return 0;
1125                 else
1126                         return -EBUSY;
1127         }
1128
1129         ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1130                                     ctx->interruptible, 15 * HZ);
1131         if (unlikely(ret < 0))
1132                 return ret;
1133         if (unlikely(ret == 0))
1134                 return -EBUSY;
1135         return 0;
1136 }
1137 EXPORT_SYMBOL(ttm_bo_wait_ctx);
1138
1139 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1140                    gfp_t gfp_flags)
1141 {
1142         struct ttm_place place;
1143         bool locked;
1144         long ret;
1145
1146         /*
1147          * While the bo may already reside in SYSTEM placement, set
1148          * SYSTEM as new placement to cover also the move further below.
1149          * The driver may use the fact that we're moving from SYSTEM
1150          * as an indication that we're about to swap out.
1151          */
1152         memset(&place, 0, sizeof(place));
1153         place.mem_type = bo->resource->mem_type;
1154         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1155                 return -EBUSY;
1156
1157         if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1158             bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1159             bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1160             !ttm_bo_get_unless_zero(bo)) {
1161                 if (locked)
1162                         dma_resv_unlock(bo->base.resv);
1163                 return -EBUSY;
1164         }
1165
1166         if (bo->deleted) {
1167                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1168                 ttm_bo_put(bo);
1169                 return ret == -EBUSY ? -ENOSPC : ret;
1170         }
1171
1172         /* TODO: Cleanup the locking */
1173         spin_unlock(&bo->bdev->lru_lock);
1174
1175         /*
1176          * Move to system cached
1177          */
1178         if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1179                 struct ttm_resource *evict_mem;
1180                 struct ttm_place hop;
1181
1182                 memset(&hop, 0, sizeof(hop));
1183                 place.mem_type = TTM_PL_SYSTEM;
1184                 ret = ttm_resource_alloc(bo, &place, &evict_mem);
1185                 if (unlikely(ret))
1186                         goto out;
1187
1188                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
1189                 if (unlikely(ret != 0)) {
1190                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1191                         ttm_resource_free(bo, &evict_mem);
1192                         goto out;
1193                 }
1194         }
1195
1196         /*
1197          * Make sure BO is idle.
1198          */
1199         ret = ttm_bo_wait_ctx(bo, ctx);
1200         if (unlikely(ret != 0))
1201                 goto out;
1202
1203         ttm_bo_unmap_virtual(bo);
1204
1205         /*
1206          * Swap out. Buffer will be swapped in again as soon as
1207          * anyone tries to access a ttm page.
1208          */
1209         if (bo->bdev->funcs->swap_notify)
1210                 bo->bdev->funcs->swap_notify(bo);
1211
1212         if (ttm_tt_is_populated(bo->ttm))
1213                 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1214 out:
1215
1216         /*
1217          * Unreserve without putting on LRU to avoid swapping out an
1218          * already swapped buffer.
1219          */
1220         if (locked)
1221                 dma_resv_unlock(bo->base.resv);
1222         ttm_bo_put(bo);
1223         return ret == -EBUSY ? -ENOSPC : ret;
1224 }
1225
1226 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1227 {
1228         if (bo->ttm == NULL)
1229                 return;
1230
1231         ttm_tt_unpopulate(bo->bdev, bo->ttm);
1232         ttm_tt_destroy(bo->bdev, bo->ttm);
1233         bo->ttm = NULL;
1234 }