drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_module.h>
  35 #include <drm/ttm/ttm_bo_driver.h>
  36 #include <drm/ttm/ttm_placement.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40 #include <linux/mm.h>
  41 #include <linux/file.h>
  42 #include <linux/module.h>
  43 #include <linux/atomic.h>
  44 #include <linux/dma-resv.h>
  45
  46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  47
  48 /**
  49  * ttm_global_mutex - protecting the global BO state
  50  */
  51 DEFINE_MUTEX(ttm_global_mutex);
  52 unsigned ttm_bo_glob_use_count;
  53 struct ttm_bo_global ttm_bo_glob;
  54 EXPORT_SYMBOL(ttm_bo_glob);
  55
  56 static struct attribute ttm_bo_count = {
  57         .name = "bo_count",
  58         .mode = S_IRUGO
  59 };
  60
  61 /* default destructor */
  62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  63 {
  64         kfree(bo);
  65 }
  66
  67 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  68                                         struct ttm_placement *placement)
  69 {
  70         struct drm_printer p = drm_debug_printer(TTM_PFX);
  71         struct ttm_resource_manager *man;
  72         int i, mem_type;
  73
  74         drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
  75                    bo, bo->mem.num_pages, bo->mem.size >> 10,
  76                    bo->mem.size >> 20);
  77         for (i = 0; i < placement->num_placement; i++) {
  78                 mem_type = placement->placement[i].mem_type;
  79                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  80                            i, placement->placement[i].flags, mem_type);
  81                 man = ttm_manager_type(bo->bdev, mem_type);
  82                 ttm_resource_manager_debug(man, &p);
  83         }
  84 }
  85
  86 static ssize_t ttm_bo_global_show(struct kobject *kobj,
  87                                   struct attribute *attr,
  88                                   char *buffer)
  89 {
  90         struct ttm_bo_global *glob =
  91                 container_of(kobj, struct ttm_bo_global, kobj);
  92
  93         return snprintf(buffer, PAGE_SIZE, "%d\n",
  94                                 atomic_read(&glob->bo_count));
  95 }
  96
  97 static struct attribute *ttm_bo_global_attrs[] = {
  98         &ttm_bo_count,
  99         NULL
 100 };
 101
 102 static const struct sysfs_ops ttm_bo_global_ops = {
 103         .show = &ttm_bo_global_show
 104 };
 105
 106 static struct kobj_type ttm_bo_glob_kobj_type  = {
 107         .release = &ttm_bo_global_kobj_release,
 108         .sysfs_ops = &ttm_bo_global_ops,
 109         .default_attrs = ttm_bo_global_attrs
 110 };
 111
 112 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
 113                                   struct ttm_resource *mem)
 114 {
 115         struct ttm_bo_device *bdev = bo->bdev;
 116         struct ttm_resource_manager *man;
 117
 118         if (!list_empty(&bo->lru))
 119                 return;
 120
 121         if (mem->placement & TTM_PL_FLAG_NO_EVICT)
 122                 return;
 123
 124         man = ttm_manager_type(bdev, mem->mem_type);
 125         list_add_tail(&bo->lru, &man->lru[bo->priority]);
 126
 127         if (man->use_tt && bo->ttm &&
 128             !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
 129                                      TTM_PAGE_FLAG_SWAPPED))) {
 130                 list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]);
 131         }
 132 }
 133
 134 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 135 {
 136         struct ttm_bo_device *bdev = bo->bdev;
 137         bool notify = false;
 138
 139         if (!list_empty(&bo->swap)) {
 140                 list_del_init(&bo->swap);
 141                 notify = true;
 142         }
 143         if (!list_empty(&bo->lru)) {
 144                 list_del_init(&bo->lru);
 145                 notify = true;
 146         }
 147
 148         if (notify && bdev->driver->del_from_lru_notify)
 149                 bdev->driver->del_from_lru_notify(bo);
 150 }
 151
 152 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
 153                                      struct ttm_buffer_object *bo)
 154 {
 155         if (!pos->first)
 156                 pos->first = bo;
 157         pos->last = bo;
 158 }
 159
 160 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
 161                              struct ttm_lru_bulk_move *bulk)
 162 {
 163         dma_resv_assert_held(bo->base.resv);
 164
 165         ttm_bo_del_from_lru(bo);
 166         ttm_bo_add_mem_to_lru(bo, &bo->mem);
 167
 168         if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 169                 switch (bo->mem.mem_type) {
 170                 case TTM_PL_TT:
 171                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 172                         break;
 173
 174                 case TTM_PL_VRAM:
 175                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 176                         break;
 177                 }
 178                 if (bo->ttm && !(bo->ttm->page_flags &
 179                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 180                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 181         }
 182 }
 183 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 184
 185 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 186 {
 187         unsigned i;
 188
 189         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 190                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 191                 struct ttm_resource_manager *man;
 192
 193                 if (!pos->first)
 194                         continue;
 195
 196                 dma_resv_assert_held(pos->first->base.resv);
 197                 dma_resv_assert_held(pos->last->base.resv);
 198
 199                 man = ttm_manager_type(pos->first->bdev, TTM_PL_TT);
 200                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 201                                     &pos->last->lru);
 202         }
 203
 204         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 205                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 206                 struct ttm_resource_manager *man;
 207
 208                 if (!pos->first)
 209                         continue;
 210
 211                 dma_resv_assert_held(pos->first->base.resv);
 212                 dma_resv_assert_held(pos->last->base.resv);
 213
 214                 man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM);
 215                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 216                                     &pos->last->lru);
 217         }
 218
 219         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 220                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 221                 struct list_head *lru;
 222
 223                 if (!pos->first)
 224                         continue;
 225
 226                 dma_resv_assert_held(pos->first->base.resv);
 227                 dma_resv_assert_held(pos->last->base.resv);
 228
 229                 lru = &ttm_bo_glob.swap_lru[i];
 230                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 231         }
 232 }
 233 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 234
 235 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 236                                   struct ttm_resource *mem, bool evict,
 237                                   struct ttm_operation_ctx *ctx)
 238 {
 239         struct ttm_bo_device *bdev = bo->bdev;
 240         struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type);
 241         struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type);
 242         int ret;
 243
 244         ttm_bo_unmap_virtual(bo);
 245
 246         /*
 247          * Create and bind a ttm if required.
 248          */
 249
 250         if (new_man->use_tt) {
 251                 /* Zero init the new TTM structure if the old location should
 252                  * have used one as well.
 253                  */
 254                 ret = ttm_tt_create(bo, old_man->use_tt);
 255                 if (ret)
 256                         goto out_err;
 257
 258                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 259                 if (ret)
 260                         goto out_err;
 261
 262                 if (mem->mem_type != TTM_PL_SYSTEM) {
 263                         ret = ttm_tt_bind(bdev, bo->ttm, mem, ctx);
 264                         if (ret)
 265                                 goto out_err;
 266                 }
 267
 268                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 269                         if (bdev->driver->move_notify)
 270                                 bdev->driver->move_notify(bo, evict, mem);
 271                         bo->mem = *mem;
 272                         goto moved;
 273                 }
 274         }
 275
 276         if (bdev->driver->move_notify)
 277                 bdev->driver->move_notify(bo, evict, mem);
 278
 279         if (old_man->use_tt && new_man->use_tt)
 280                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 281         else if (bdev->driver->move)
 282                 ret = bdev->driver->move(bo, evict, ctx, mem);
 283         else
 284                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 285
 286         if (ret) {
 287                 if (bdev->driver->move_notify) {
 288                         swap(*mem, bo->mem);
 289                         bdev->driver->move_notify(bo, false, mem);
 290                         swap(*mem, bo->mem);
 291                 }
 292
 293                 goto out_err;
 294         }
 295
 296 moved:
 297         bo->evicted = false;
 298
 299         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 300         return 0;
 301
 302 out_err:
 303         new_man = ttm_manager_type(bdev, bo->mem.mem_type);
 304         if (!new_man->use_tt) {
 305                 ttm_tt_destroy(bdev, bo->ttm);
 306                 bo->ttm = NULL;
 307         }
 308
 309         return ret;
 310 }
 311
 312 /**
 313  * Call bo::reserved.
 314  * Will release GPU memory type usage on destruction.
 315  * This is the place to put in driver specific hooks to release
 316  * driver private resources.
 317  * Will release the bo::reserved lock.
 318  */
 319
 320 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 321 {
 322         if (bo->bdev->driver->move_notify)
 323                 bo->bdev->driver->move_notify(bo, false, NULL);
 324
 325         ttm_tt_destroy(bo->bdev, bo->ttm);
 326         bo->ttm = NULL;
 327         ttm_resource_free(bo, &bo->mem);
 328 }
 329
 330 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 331 {
 332         int r;
 333
 334         if (bo->base.resv == &bo->base._resv)
 335                 return 0;
 336
 337         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 338
 339         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 340         dma_resv_unlock(&bo->base._resv);
 341         if (r)
 342                 return r;
 343
 344         if (bo->type != ttm_bo_type_sg) {
 345                 /* This works because the BO is about to be destroyed and nobody
 346                  * reference it any more. The only tricky case is the trylock on
 347                  * the resv object while holding the lru_lock.
 348                  */
 349                 spin_lock(&ttm_bo_glob.lru_lock);
 350                 bo->base.resv = &bo->base._resv;
 351                 spin_unlock(&ttm_bo_glob.lru_lock);
 352         }
 353
 354         return r;
 355 }
 356
 357 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 358 {
 359         struct dma_resv *resv = &bo->base._resv;
 360         struct dma_resv_list *fobj;
 361         struct dma_fence *fence;
 362         int i;
 363
 364         rcu_read_lock();
 365         fobj = rcu_dereference(resv->fence);
 366         fence = rcu_dereference(resv->fence_excl);
 367         if (fence && !fence->ops->signaled)
 368                 dma_fence_enable_sw_signaling(fence);
 369
 370         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 371                 fence = rcu_dereference(fobj->shared[i]);
 372
 373                 if (!fence->ops->signaled)
 374                         dma_fence_enable_sw_signaling(fence);
 375         }
 376         rcu_read_unlock();
 377 }
 378
 379 /**
 380  * function ttm_bo_cleanup_refs
 381  * If bo idle, remove from lru lists, and unref.
 382  * If not idle, block if possible.
 383  *
 384  * Must be called with lru_lock and reservation held, this function
 385  * will drop the lru lock and optionally the reservation lock before returning.
 386  *
 387  * @interruptible         Any sleeps should occur interruptibly.
 388  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 389  * @unlock_resv           Unlock the reservation lock as well.
 390  */
 391
 392 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 393                                bool interruptible, bool no_wait_gpu,
 394                                bool unlock_resv)
 395 {
 396         struct dma_resv *resv = &bo->base._resv;
 397         int ret;
 398
 399         if (dma_resv_test_signaled_rcu(resv, true))
 400                 ret = 0;
 401         else
 402                 ret = -EBUSY;
 403
 404         if (ret && !no_wait_gpu) {
 405                 long lret;
 406
 407                 if (unlock_resv)
 408                         dma_resv_unlock(bo->base.resv);
 409                 spin_unlock(&ttm_bo_glob.lru_lock);
 410
 411                 lret = dma_resv_wait_timeout_rcu(resv, true, interruptible,
 412                                                  30 * HZ);
 413
 414                 if (lret < 0)
 415                         return lret;
 416                 else if (lret == 0)
 417                         return -EBUSY;
 418
 419                 spin_lock(&ttm_bo_glob.lru_lock);
 420                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 421                         /*
 422                          * We raced, and lost, someone else holds the reservation now,
 423                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 424                          *
 425                          * Even if it's not the case, because we finished waiting any
 426                          * delayed destruction would succeed, so just return success
 427                          * here.
 428                          */
 429                         spin_unlock(&ttm_bo_glob.lru_lock);
 430                         return 0;
 431                 }
 432                 ret = 0;
 433         }
 434
 435         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 436                 if (unlock_resv)
 437                         dma_resv_unlock(bo->base.resv);
 438                 spin_unlock(&ttm_bo_glob.lru_lock);
 439                 return ret;
 440         }
 441
 442         ttm_bo_del_from_lru(bo);
 443         list_del_init(&bo->ddestroy);
 444         spin_unlock(&ttm_bo_glob.lru_lock);
 445         ttm_bo_cleanup_memtype_use(bo);
 446
 447         if (unlock_resv)
 448                 dma_resv_unlock(bo->base.resv);
 449
 450         ttm_bo_put(bo);
 451
 452         return 0;
 453 }
 454
 455 /**
 456  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 457  * encountered buffers.
 458  */
 459 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 460 {
 461         struct ttm_bo_global *glob = &ttm_bo_glob;
 462         struct list_head removed;
 463         bool empty;
 464
 465         INIT_LIST_HEAD(&removed);
 466
 467         spin_lock(&glob->lru_lock);
 468         while (!list_empty(&bdev->ddestroy)) {
 469                 struct ttm_buffer_object *bo;
 470
 471                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 472                                       ddestroy);
 473                 list_move_tail(&bo->ddestroy, &removed);
 474                 if (!ttm_bo_get_unless_zero(bo))
 475                         continue;
 476
 477                 if (remove_all || bo->base.resv != &bo->base._resv) {
 478                         spin_unlock(&glob->lru_lock);
 479                         dma_resv_lock(bo->base.resv, NULL);
 480
 481                         spin_lock(&glob->lru_lock);
 482                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 483
 484                 } else if (dma_resv_trylock(bo->base.resv)) {
 485                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 486                 } else {
 487                         spin_unlock(&glob->lru_lock);
 488                 }
 489
 490                 ttm_bo_put(bo);
 491                 spin_lock(&glob->lru_lock);
 492         }
 493         list_splice_tail(&removed, &bdev->ddestroy);
 494         empty = list_empty(&bdev->ddestroy);
 495         spin_unlock(&glob->lru_lock);
 496
 497         return empty;
 498 }
 499
 500 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 501 {
 502         struct ttm_bo_device *bdev =
 503             container_of(work, struct ttm_bo_device, wq.work);
 504
 505         if (!ttm_bo_delayed_delete(bdev, false))
 506                 schedule_delayed_work(&bdev->wq,
 507                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 508 }
 509
 510 static void ttm_bo_release(struct kref *kref)
 511 {
 512         struct ttm_buffer_object *bo =
 513             container_of(kref, struct ttm_buffer_object, kref);
 514         struct ttm_bo_device *bdev = bo->bdev;
 515         size_t acc_size = bo->acc_size;
 516         int ret;
 517
 518         if (!bo->deleted) {
 519                 ret = ttm_bo_individualize_resv(bo);
 520                 if (ret) {
 521                         /* Last resort, if we fail to allocate memory for the
 522                          * fences block for the BO to become idle
 523                          */
 524                         dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
 525                                                   30 * HZ);
 526                 }
 527
 528                 if (bo->bdev->driver->release_notify)
 529                         bo->bdev->driver->release_notify(bo);
 530
 531                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 532                 ttm_mem_io_free(bdev, &bo->mem);
 533         }
 534
 535         if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
 536             !dma_resv_trylock(bo->base.resv)) {
 537                 /* The BO is not idle, resurrect it for delayed destroy */
 538                 ttm_bo_flush_all_fences(bo);
 539                 bo->deleted = true;
 540
 541                 spin_lock(&ttm_bo_glob.lru_lock);
 542
 543                 /*
 544                  * Make NO_EVICT bos immediately available to
 545                  * shrinkers, now that they are queued for
 546                  * destruction.
 547                  */
 548                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 549                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 550                         ttm_bo_del_from_lru(bo);
 551                         ttm_bo_add_mem_to_lru(bo, &bo->mem);
 552                 }
 553
 554                 kref_init(&bo->kref);
 555                 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 556                 spin_unlock(&ttm_bo_glob.lru_lock);
 557
 558                 schedule_delayed_work(&bdev->wq,
 559                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 560                 return;
 561         }
 562
 563         spin_lock(&ttm_bo_glob.lru_lock);
 564         ttm_bo_del_from_lru(bo);
 565         list_del(&bo->ddestroy);
 566         spin_unlock(&ttm_bo_glob.lru_lock);
 567
 568         ttm_bo_cleanup_memtype_use(bo);
 569         dma_resv_unlock(bo->base.resv);
 570
 571         atomic_dec(&ttm_bo_glob.bo_count);
 572         dma_fence_put(bo->moving);
 573         if (!ttm_bo_uses_embedded_gem_object(bo))
 574                 dma_resv_fini(&bo->base._resv);
 575         bo->destroy(bo);
 576         ttm_mem_global_free(&ttm_mem_glob, acc_size);
 577 }
 578
 579 void ttm_bo_put(struct ttm_buffer_object *bo)
 580 {
 581         kref_put(&bo->kref, ttm_bo_release);
 582 }
 583 EXPORT_SYMBOL(ttm_bo_put);
 584
 585 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 586 {
 587         return cancel_delayed_work_sync(&bdev->wq);
 588 }
 589 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 590
 591 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 592 {
 593         if (resched)
 594                 schedule_delayed_work(&bdev->wq,
 595                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 596 }
 597 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 598
 599 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 600                         struct ttm_operation_ctx *ctx)
 601 {
 602         struct ttm_bo_device *bdev = bo->bdev;
 603         struct ttm_resource evict_mem;
 604         struct ttm_placement placement;
 605         int ret = 0;
 606
 607         dma_resv_assert_held(bo->base.resv);
 608
 609         placement.num_placement = 0;
 610         placement.num_busy_placement = 0;
 611         bdev->driver->evict_flags(bo, &placement);
 612
 613         if (!placement.num_placement && !placement.num_busy_placement) {
 614                 ttm_bo_wait(bo, false, false);
 615
 616                 ttm_bo_cleanup_memtype_use(bo);
 617                 return ttm_tt_create(bo, false);
 618         }
 619
 620         evict_mem = bo->mem;
 621         evict_mem.mm_node = NULL;
 622         evict_mem.bus.offset = 0;
 623         evict_mem.bus.addr = NULL;
 624
 625         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 626         if (ret) {
 627                 if (ret != -ERESTARTSYS) {
 628                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 629                                bo);
 630                         ttm_bo_mem_space_debug(bo, &placement);
 631                 }
 632                 goto out;
 633         }
 634
 635         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 636         if (unlikely(ret)) {
 637                 if (ret != -ERESTARTSYS)
 638                         pr_err("Buffer eviction failed\n");
 639                 ttm_resource_free(bo, &evict_mem);
 640                 goto out;
 641         }
 642         bo->evicted = true;
 643 out:
 644         return ret;
 645 }
 646
 647 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 648                               const struct ttm_place *place)
 649 {
 650         /* Don't evict this BO if it's outside of the
 651          * requested placement range
 652          */
 653         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 654             (place->lpfn && place->lpfn <= bo->mem.start))
 655                 return false;
 656
 657         return true;
 658 }
 659 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 660
 661 /**
 662  * Check the target bo is allowable to be evicted or swapout, including cases:
 663  *
 664  * a. if share same reservation object with ctx->resv, have assumption
 665  * reservation objects should already be locked, so not lock again and
 666  * return true directly when either the opreation allow_reserved_eviction
 667  * or the target bo already is in delayed free list;
 668  *
 669  * b. Otherwise, trylock it.
 670  */
 671 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 672                         struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
 673 {
 674         bool ret = false;
 675
 676         if (bo->base.resv == ctx->resv) {
 677                 dma_resv_assert_held(bo->base.resv);
 678                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT)
 679                         ret = true;
 680                 *locked = false;
 681                 if (busy)
 682                         *busy = false;
 683         } else {
 684                 ret = dma_resv_trylock(bo->base.resv);
 685                 *locked = ret;
 686                 if (busy)
 687                         *busy = !ret;
 688         }
 689
 690         return ret;
 691 }
 692
 693 /**
 694  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 695  *
 696  * @busy_bo: BO which couldn't be locked with trylock
 697  * @ctx: operation context
 698  * @ticket: acquire ticket
 699  *
 700  * Try to lock a busy buffer object to avoid failing eviction.
 701  */
 702 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 703                                    struct ttm_operation_ctx *ctx,
 704                                    struct ww_acquire_ctx *ticket)
 705 {
 706         int r;
 707
 708         if (!busy_bo || !ticket)
 709                 return -EBUSY;
 710
 711         if (ctx->interruptible)
 712                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 713                                                           ticket);
 714         else
 715                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 716
 717         /*
 718          * TODO: It would be better to keep the BO locked until allocation is at
 719          * least tried one more time, but that would mean a much larger rework
 720          * of TTM.
 721          */
 722         if (!r)
 723                 dma_resv_unlock(busy_bo->base.resv);
 724
 725         return r == -EDEADLK ? -EBUSY : r;
 726 }
 727
 728 int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 729                         struct ttm_resource_manager *man,
 730                         const struct ttm_place *place,
 731                         struct ttm_operation_ctx *ctx,
 732                         struct ww_acquire_ctx *ticket)
 733 {
 734         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 735         bool locked = false;
 736         unsigned i;
 737         int ret;
 738
 739         spin_lock(&ttm_bo_glob.lru_lock);
 740         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 741                 list_for_each_entry(bo, &man->lru[i], lru) {
 742                         bool busy;
 743
 744                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
 745                                                             &busy)) {
 746                                 if (busy && !busy_bo && ticket !=
 747                                     dma_resv_locking_ctx(bo->base.resv))
 748                                         busy_bo = bo;
 749                                 continue;
 750                         }
 751
 752                         if (place && !bdev->driver->eviction_valuable(bo,
 753                                                                       place)) {
 754                                 if (locked)
 755                                         dma_resv_unlock(bo->base.resv);
 756                                 continue;
 757                         }
 758                         if (!ttm_bo_get_unless_zero(bo)) {
 759                                 if (locked)
 760                                         dma_resv_unlock(bo->base.resv);
 761                                 continue;
 762                         }
 763                         break;
 764                 }
 765
 766                 /* If the inner loop terminated early, we have our candidate */
 767                 if (&bo->lru != &man->lru[i])
 768                         break;
 769
 770                 bo = NULL;
 771         }
 772
 773         if (!bo) {
 774                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 775                         busy_bo = NULL;
 776                 spin_unlock(&ttm_bo_glob.lru_lock);
 777                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 778                 if (busy_bo)
 779                         ttm_bo_put(busy_bo);
 780                 return ret;
 781         }
 782
 783         if (bo->deleted) {
 784                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 785                                           ctx->no_wait_gpu, locked);
 786                 ttm_bo_put(bo);
 787                 return ret;
 788         }
 789
 790         spin_unlock(&ttm_bo_glob.lru_lock);
 791
 792         ret = ttm_bo_evict(bo, ctx);
 793         if (locked)
 794                 ttm_bo_unreserve(bo);
 795
 796         ttm_bo_put(bo);
 797         return ret;
 798 }
 799
 800 /**
 801  * Add the last move fence to the BO and reserve a new shared slot.
 802  */
 803 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 804                                  struct ttm_resource_manager *man,
 805                                  struct ttm_resource *mem,
 806                                  bool no_wait_gpu)
 807 {
 808         struct dma_fence *fence;
 809         int ret;
 810
 811         spin_lock(&man->move_lock);
 812         fence = dma_fence_get(man->move);
 813         spin_unlock(&man->move_lock);
 814
 815         if (!fence)
 816                 return 0;
 817
 818         if (no_wait_gpu) {
 819                 dma_fence_put(fence);
 820                 return -EBUSY;
 821         }
 822
 823         dma_resv_add_shared_fence(bo->base.resv, fence);
 824
 825         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 826         if (unlikely(ret)) {
 827                 dma_fence_put(fence);
 828                 return ret;
 829         }
 830
 831         dma_fence_put(bo->moving);
 832         bo->moving = fence;
 833         return 0;
 834 }
 835
 836 /**
 837  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 838  * space, or we've evicted everything and there isn't enough space.
 839  */
 840 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 841                                   const struct ttm_place *place,
 842                                   struct ttm_resource *mem,
 843                                   struct ttm_operation_ctx *ctx)
 844 {
 845         struct ttm_bo_device *bdev = bo->bdev;
 846         struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type);
 847         struct ww_acquire_ctx *ticket;
 848         int ret;
 849
 850         ticket = dma_resv_locking_ctx(bo->base.resv);
 851         do {
 852                 ret = ttm_resource_alloc(bo, place, mem);
 853                 if (likely(!ret))
 854                         break;
 855                 if (unlikely(ret != -ENOSPC))
 856                         return ret;
 857                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 858                                           ticket);
 859                 if (unlikely(ret != 0))
 860                         return ret;
 861         } while (1);
 862
 863         return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 864 }
 865
 866 static uint32_t ttm_bo_select_caching(struct ttm_resource_manager *man,
 867                                       uint32_t cur_placement,
 868                                       uint32_t proposed_placement)
 869 {
 870         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 871         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 872
 873         /**
 874          * Keep current caching if possible.
 875          */
 876
 877         if ((cur_placement & caching) != 0)
 878                 result |= (cur_placement & caching);
 879         else if ((man->default_caching & caching) != 0)
 880                 result |= man->default_caching;
 881         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 882                 result |= TTM_PL_FLAG_CACHED;
 883         else if ((TTM_PL_FLAG_WC & caching) != 0)
 884                 result |= TTM_PL_FLAG_WC;
 885         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 886                 result |= TTM_PL_FLAG_UNCACHED;
 887
 888         return result;
 889 }
 890
 891 /**
 892  * ttm_bo_mem_placement - check if placement is compatible
 893  * @bo: BO to find memory for
 894  * @place: where to search
 895  * @mem: the memory object to fill in
 896  * @ctx: operation context
 897  *
 898  * Check if placement is compatible and fill in mem structure.
 899  * Returns -EBUSY if placement won't work or negative error code.
 900  * 0 when placement can be used.
 901  */
 902 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
 903                                 const struct ttm_place *place,
 904                                 struct ttm_resource *mem,
 905                                 struct ttm_operation_ctx *ctx)
 906 {
 907         struct ttm_bo_device *bdev = bo->bdev;
 908         struct ttm_resource_manager *man;
 909         uint32_t cur_flags = 0;
 910
 911         man = ttm_manager_type(bdev, place->mem_type);
 912         if (!man || !ttm_resource_manager_used(man))
 913                 return -EBUSY;
 914
 915         if ((place->flags & man->available_caching) == 0)
 916                 return -EBUSY;
 917
 918         cur_flags = place->flags & man->available_caching;
 919         cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags);
 920         cur_flags |= place->flags & ~TTM_PL_MASK_CACHING;
 921
 922         mem->mem_type = place->mem_type;
 923         mem->placement = cur_flags;
 924
 925         spin_lock(&ttm_bo_glob.lru_lock);
 926         ttm_bo_del_from_lru(bo);
 927         ttm_bo_add_mem_to_lru(bo, mem);
 928         spin_unlock(&ttm_bo_glob.lru_lock);
 929
 930         return 0;
 931 }
 932
 933 /**
 934  * Creates space for memory region @mem according to its type.
 935  *
 936  * This function first searches for free space in compatible memory types in
 937  * the priority order defined by the driver.  If free space isn't found, then
 938  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 939  * space.
 940  */
 941 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 942                         struct ttm_placement *placement,
 943                         struct ttm_resource *mem,
 944                         struct ttm_operation_ctx *ctx)
 945 {
 946         struct ttm_bo_device *bdev = bo->bdev;
 947         bool type_found = false;
 948         int i, ret;
 949
 950         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 951         if (unlikely(ret))
 952                 return ret;
 953
 954         for (i = 0; i < placement->num_placement; ++i) {
 955                 const struct ttm_place *place = &placement->placement[i];
 956                 struct ttm_resource_manager *man;
 957
 958                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
 959                 if (ret)
 960                         continue;
 961
 962                 type_found = true;
 963                 ret = ttm_resource_alloc(bo, place, mem);
 964                 if (ret == -ENOSPC)
 965                         continue;
 966                 if (unlikely(ret))
 967                         goto error;
 968
 969                 man = ttm_manager_type(bdev, mem->mem_type);
 970                 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 971                 if (unlikely(ret)) {
 972                         ttm_resource_free(bo, mem);
 973                         if (ret == -EBUSY)
 974                                 continue;
 975
 976                         goto error;
 977                 }
 978                 return 0;
 979         }
 980
 981         for (i = 0; i < placement->num_busy_placement; ++i) {
 982                 const struct ttm_place *place = &placement->busy_placement[i];
 983
 984                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
 985                 if (ret)
 986                         continue;
 987
 988                 type_found = true;
 989                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 990                 if (likely(!ret))
 991                         return 0;
 992
 993                 if (ret && ret != -EBUSY)
 994                         goto error;
 995         }
 996
 997         ret = -ENOMEM;
 998         if (!type_found) {
 999                 pr_err(TTM_PFX "No compatible memory type found\n");
1000                 ret = -EINVAL;
1001         }
1002
1003 error:
1004         if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) {
1005                 ttm_bo_move_to_lru_tail_unlocked(bo);
1006         }
1007
1008         return ret;
1009 }
1010 EXPORT_SYMBOL(ttm_bo_mem_space);
1011
1012 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1013                               struct ttm_placement *placement,
1014                               struct ttm_operation_ctx *ctx)
1015 {
1016         int ret = 0;
1017         struct ttm_resource mem;
1018
1019         dma_resv_assert_held(bo->base.resv);
1020
1021         mem.num_pages = bo->num_pages;
1022         mem.size = mem.num_pages << PAGE_SHIFT;
1023         mem.page_alignment = bo->mem.page_alignment;
1024         mem.bus.offset = 0;
1025         mem.bus.addr = NULL;
1026         mem.mm_node = NULL;
1027
1028         /*
1029          * Determine where to move the buffer.
1030          */
1031         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1032         if (ret)
1033                 goto out_unlock;
1034         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1035 out_unlock:
1036         if (ret)
1037                 ttm_resource_free(bo, &mem);
1038         return ret;
1039 }
1040
1041 static bool ttm_bo_places_compat(const struct ttm_place *places,
1042                                  unsigned num_placement,
1043                                  struct ttm_resource *mem,
1044                                  uint32_t *new_flags)
1045 {
1046         unsigned i;
1047
1048         for (i = 0; i < num_placement; i++) {
1049                 const struct ttm_place *heap = &places[i];
1050
1051                 if ((mem->start < heap->fpfn ||
1052                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1053                         continue;
1054
1055                 *new_flags = heap->flags;
1056                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1057                     (mem->mem_type == heap->mem_type) &&
1058                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1059                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1060                         return true;
1061         }
1062         return false;
1063 }
1064
1065 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1066                        struct ttm_resource *mem,
1067                        uint32_t *new_flags)
1068 {
1069         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1070                                  mem, new_flags))
1071                 return true;
1072
1073         if ((placement->busy_placement != placement->placement ||
1074              placement->num_busy_placement > placement->num_placement) &&
1075             ttm_bo_places_compat(placement->busy_placement,
1076                                  placement->num_busy_placement,
1077                                  mem, new_flags))
1078                 return true;
1079
1080         return false;
1081 }
1082 EXPORT_SYMBOL(ttm_bo_mem_compat);
1083
1084 int ttm_bo_validate(struct ttm_buffer_object *bo,
1085                     struct ttm_placement *placement,
1086                     struct ttm_operation_ctx *ctx)
1087 {
1088         int ret;
1089         uint32_t new_flags;
1090
1091         dma_resv_assert_held(bo->base.resv);
1092
1093         /*
1094          * Remove the backing store if no placement is given.
1095          */
1096         if (!placement->num_placement && !placement->num_busy_placement) {
1097                 ret = ttm_bo_pipeline_gutting(bo);
1098                 if (ret)
1099                         return ret;
1100
1101                 return ttm_tt_create(bo, false);
1102         }
1103
1104         /*
1105          * Check whether we need to move buffer.
1106          */
1107         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1108                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1109                 if (ret)
1110                         return ret;
1111         } else {
1112                 bo->mem.placement &= TTM_PL_MASK_CACHING;
1113                 bo->mem.placement |= new_flags & ~TTM_PL_MASK_CACHING;
1114         }
1115         /*
1116          * We might need to add a TTM.
1117          */
1118         if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1119                 ret = ttm_tt_create(bo, true);
1120                 if (ret)
1121                         return ret;
1122         }
1123         return 0;
1124 }
1125 EXPORT_SYMBOL(ttm_bo_validate);
1126
1127 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1128                          struct ttm_buffer_object *bo,
1129                          unsigned long size,
1130                          enum ttm_bo_type type,
1131                          struct ttm_placement *placement,
1132                          uint32_t page_alignment,
1133                          struct ttm_operation_ctx *ctx,
1134                          size_t acc_size,
1135                          struct sg_table *sg,
1136                          struct dma_resv *resv,
1137                          void (*destroy) (struct ttm_buffer_object *))
1138 {
1139         struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1140         int ret = 0;
1141         unsigned long num_pages;
1142         bool locked;
1143
1144         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1145         if (ret) {
1146                 pr_err("Out of kernel memory\n");
1147                 if (destroy)
1148                         (*destroy)(bo);
1149                 else
1150                         kfree(bo);
1151                 return -ENOMEM;
1152         }
1153
1154         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1155         if (num_pages == 0) {
1156                 pr_err("Illegal buffer object size\n");
1157                 if (destroy)
1158                         (*destroy)(bo);
1159                 else
1160                         kfree(bo);
1161                 ttm_mem_global_free(mem_glob, acc_size);
1162                 return -EINVAL;
1163         }
1164         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1165
1166         kref_init(&bo->kref);
1167         INIT_LIST_HEAD(&bo->lru);
1168         INIT_LIST_HEAD(&bo->ddestroy);
1169         INIT_LIST_HEAD(&bo->swap);
1170         bo->bdev = bdev;
1171         bo->type = type;
1172         bo->num_pages = num_pages;
1173         bo->mem.size = num_pages << PAGE_SHIFT;
1174         bo->mem.mem_type = TTM_PL_SYSTEM;
1175         bo->mem.num_pages = bo->num_pages;
1176         bo->mem.mm_node = NULL;
1177         bo->mem.page_alignment = page_alignment;
1178         bo->mem.bus.offset = 0;
1179         bo->mem.bus.addr = NULL;
1180         bo->moving = NULL;
1181         bo->mem.placement = TTM_PL_FLAG_CACHED;
1182         bo->acc_size = acc_size;
1183         bo->sg = sg;
1184         if (resv) {
1185                 bo->base.resv = resv;
1186                 dma_resv_assert_held(bo->base.resv);
1187         } else {
1188                 bo->base.resv = &bo->base._resv;
1189         }
1190         if (!ttm_bo_uses_embedded_gem_object(bo)) {
1191                 /*
1192                  * bo.gem is not initialized, so we have to setup the
1193                  * struct elements we want use regardless.
1194                  */
1195                 dma_resv_init(&bo->base._resv);
1196                 drm_vma_node_reset(&bo->base.vma_node);
1197         }
1198         atomic_inc(&ttm_bo_glob.bo_count);
1199
1200         /*
1201          * For ttm_bo_type_device buffers, allocate
1202          * address space from the device.
1203          */
1204         if (bo->type == ttm_bo_type_device ||
1205             bo->type == ttm_bo_type_sg)
1206                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1207                                          bo->mem.num_pages);
1208
1209         /* passed reservation objects should already be locked,
1210          * since otherwise lockdep will be angered in radeon.
1211          */
1212         if (!resv) {
1213                 locked = dma_resv_trylock(bo->base.resv);
1214                 WARN_ON(!locked);
1215         }
1216
1217         if (likely(!ret))
1218                 ret = ttm_bo_validate(bo, placement, ctx);
1219
1220         if (unlikely(ret)) {
1221                 if (!resv)
1222                         ttm_bo_unreserve(bo);
1223
1224                 ttm_bo_put(bo);
1225                 return ret;
1226         }
1227
1228         ttm_bo_move_to_lru_tail_unlocked(bo);
1229
1230         return ret;
1231 }
1232 EXPORT_SYMBOL(ttm_bo_init_reserved);
1233
1234 int ttm_bo_init(struct ttm_bo_device *bdev,
1235                 struct ttm_buffer_object *bo,
1236                 unsigned long size,
1237                 enum ttm_bo_type type,
1238                 struct ttm_placement *placement,
1239                 uint32_t page_alignment,
1240                 bool interruptible,
1241                 size_t acc_size,
1242                 struct sg_table *sg,
1243                 struct dma_resv *resv,
1244                 void (*destroy) (struct ttm_buffer_object *))
1245 {
1246         struct ttm_operation_ctx ctx = { interruptible, false };
1247         int ret;
1248
1249         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1250                                    page_alignment, &ctx, acc_size,
1251                                    sg, resv, destroy);
1252         if (ret)
1253                 return ret;
1254
1255         if (!resv)
1256                 ttm_bo_unreserve(bo);
1257
1258         return 0;
1259 }
1260 EXPORT_SYMBOL(ttm_bo_init);
1261
1262 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1263                        unsigned long bo_size,
1264                        unsigned struct_size)
1265 {
1266         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1267         size_t size = 0;
1268
1269         size += ttm_round_pot(struct_size);
1270         size += ttm_round_pot(npages * sizeof(void *));
1271         size += ttm_round_pot(sizeof(struct ttm_tt));
1272         return size;
1273 }
1274 EXPORT_SYMBOL(ttm_bo_acc_size);
1275
1276 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1277                            unsigned long bo_size,
1278                            unsigned struct_size)
1279 {
1280         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1281         size_t size = 0;
1282
1283         size += ttm_round_pot(struct_size);
1284         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1285         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1286         return size;
1287 }
1288 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1289
1290 int ttm_bo_create(struct ttm_bo_device *bdev,
1291                         unsigned long size,
1292                         enum ttm_bo_type type,
1293                         struct ttm_placement *placement,
1294                         uint32_t page_alignment,
1295                         bool interruptible,
1296                         struct ttm_buffer_object **p_bo)
1297 {
1298         struct ttm_buffer_object *bo;
1299         size_t acc_size;
1300         int ret;
1301
1302         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1303         if (unlikely(bo == NULL))
1304                 return -ENOMEM;
1305
1306         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1307         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1308                           interruptible, acc_size,
1309                           NULL, NULL, NULL);
1310         if (likely(ret == 0))
1311                 *p_bo = bo;
1312
1313         return ret;
1314 }
1315 EXPORT_SYMBOL(ttm_bo_create);
1316
1317 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1318 {
1319         struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type);
1320
1321         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1322                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1323                 return -EINVAL;
1324         }
1325
1326         if (!man) {
1327                 pr_err("Memory type %u has not been initialized\n", mem_type);
1328                 return 0;
1329         }
1330
1331         return ttm_resource_manager_force_list_clean(bdev, man);
1332 }
1333 EXPORT_SYMBOL(ttm_bo_evict_mm);
1334
1335 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1336 {
1337         struct ttm_bo_global *glob =
1338                 container_of(kobj, struct ttm_bo_global, kobj);
1339
1340         __free_page(glob->dummy_read_page);
1341 }
1342
1343 static void ttm_bo_global_release(void)
1344 {
1345         struct ttm_bo_global *glob = &ttm_bo_glob;
1346
1347         mutex_lock(&ttm_global_mutex);
1348         if (--ttm_bo_glob_use_count > 0)
1349                 goto out;
1350
1351         kobject_del(&glob->kobj);
1352         kobject_put(&glob->kobj);
1353         ttm_mem_global_release(&ttm_mem_glob);
1354         memset(glob, 0, sizeof(*glob));
1355 out:
1356         mutex_unlock(&ttm_global_mutex);
1357 }
1358
1359 static int ttm_bo_global_init(void)
1360 {
1361         struct ttm_bo_global *glob = &ttm_bo_glob;
1362         int ret = 0;
1363         unsigned i;
1364
1365         mutex_lock(&ttm_global_mutex);
1366         if (++ttm_bo_glob_use_count > 1)
1367                 goto out;
1368
1369         ret = ttm_mem_global_init(&ttm_mem_glob);
1370         if (ret)
1371                 goto out;
1372
1373         spin_lock_init(&glob->lru_lock);
1374         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1375
1376         if (unlikely(glob->dummy_read_page == NULL)) {
1377                 ret = -ENOMEM;
1378                 goto out;
1379         }
1380
1381         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1382                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1383         INIT_LIST_HEAD(&glob->device_list);
1384         atomic_set(&glob->bo_count, 0);
1385
1386         ret = kobject_init_and_add(
1387                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1388         if (unlikely(ret != 0))
1389                 kobject_put(&glob->kobj);
1390 out:
1391         mutex_unlock(&ttm_global_mutex);
1392         return ret;
1393 }
1394
1395 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1396 {
1397         struct ttm_bo_global *glob = &ttm_bo_glob;
1398         int ret = 0;
1399         unsigned i;
1400         struct ttm_resource_manager *man;
1401
1402         man = ttm_manager_type(bdev, TTM_PL_SYSTEM);
1403         ttm_resource_manager_set_used(man, false);
1404         ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL);
1405
1406         mutex_lock(&ttm_global_mutex);
1407         list_del(&bdev->device_list);
1408         mutex_unlock(&ttm_global_mutex);
1409
1410         cancel_delayed_work_sync(&bdev->wq);
1411
1412         if (ttm_bo_delayed_delete(bdev, true))
1413                 pr_debug("Delayed destroy list was clean\n");
1414
1415         spin_lock(&glob->lru_lock);
1416         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1417                 if (list_empty(&man->lru[0]))
1418                         pr_debug("Swap list %d was clean\n", i);
1419         spin_unlock(&glob->lru_lock);
1420
1421         if (!ret)
1422                 ttm_bo_global_release();
1423
1424         return ret;
1425 }
1426 EXPORT_SYMBOL(ttm_bo_device_release);
1427
1428 static void ttm_bo_init_sysman(struct ttm_bo_device *bdev)
1429 {
1430         struct ttm_resource_manager *man = &bdev->sysman;
1431
1432         /*
1433          * Initialize the system memory buffer type.
1434          * Other types need to be driver / IOCTL initialized.
1435          */
1436         man->use_tt = true;
1437         man->available_caching = TTM_PL_MASK_CACHING;
1438         man->default_caching = TTM_PL_FLAG_CACHED;
1439
1440         ttm_resource_manager_init(man, 0);
1441         ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man);
1442         ttm_resource_manager_set_used(man, true);
1443 }
1444
1445 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1446                        struct ttm_bo_driver *driver,
1447                        struct address_space *mapping,
1448                        struct drm_vma_offset_manager *vma_manager,
1449                        bool need_dma32)
1450 {
1451         struct ttm_bo_global *glob = &ttm_bo_glob;
1452         int ret;
1453
1454         if (WARN_ON(vma_manager == NULL))
1455                 return -EINVAL;
1456
1457         ret = ttm_bo_global_init();
1458         if (ret)
1459                 return ret;
1460
1461         bdev->driver = driver;
1462
1463         ttm_bo_init_sysman(bdev);
1464
1465         bdev->vma_manager = vma_manager;
1466         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1467         INIT_LIST_HEAD(&bdev->ddestroy);
1468         bdev->dev_mapping = mapping;
1469         bdev->need_dma32 = need_dma32;
1470         mutex_lock(&ttm_global_mutex);
1471         list_add_tail(&bdev->device_list, &glob->device_list);
1472         mutex_unlock(&ttm_global_mutex);
1473
1474         return 0;
1475 }
1476 EXPORT_SYMBOL(ttm_bo_device_init);
1477
1478 /*
1479  * buffer object vm functions.
1480  */
1481
1482 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1483 {
1484         struct ttm_bo_device *bdev = bo->bdev;
1485
1486         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1487         ttm_mem_io_free(bdev, &bo->mem);
1488 }
1489 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1490
1491 int ttm_bo_wait(struct ttm_buffer_object *bo,
1492                 bool interruptible, bool no_wait)
1493 {
1494         long timeout = 15 * HZ;
1495
1496         if (no_wait) {
1497                 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1498                         return 0;
1499                 else
1500                         return -EBUSY;
1501         }
1502
1503         timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1504                                                       interruptible, timeout);
1505         if (timeout < 0)
1506                 return timeout;
1507
1508         if (timeout == 0)
1509                 return -EBUSY;
1510
1511         dma_resv_add_excl_fence(bo->base.resv, NULL);
1512         return 0;
1513 }
1514 EXPORT_SYMBOL(ttm_bo_wait);
1515
1516 /**
1517  * A buffer object shrink method that tries to swap out the first
1518  * buffer object on the bo_global::swap_lru list.
1519  */
1520 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1521 {
1522         struct ttm_buffer_object *bo;
1523         int ret = -EBUSY;
1524         bool locked;
1525         unsigned i;
1526
1527         spin_lock(&glob->lru_lock);
1528         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1529                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1530                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1531                                                             NULL))
1532                                 continue;
1533
1534                         if (!ttm_bo_get_unless_zero(bo)) {
1535                                 if (locked)
1536                                         dma_resv_unlock(bo->base.resv);
1537                                 continue;
1538                         }
1539
1540                         ret = 0;
1541                         break;
1542                 }
1543                 if (!ret)
1544                         break;
1545         }
1546
1547         if (ret) {
1548                 spin_unlock(&glob->lru_lock);
1549                 return ret;
1550         }
1551
1552         if (bo->deleted) {
1553                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1554                 ttm_bo_put(bo);
1555                 return ret;
1556         }
1557
1558         ttm_bo_del_from_lru(bo);
1559         spin_unlock(&glob->lru_lock);
1560
1561         /**
1562          * Move to system cached
1563          */
1564
1565         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1566             bo->ttm->caching_state != tt_cached) {
1567                 struct ttm_operation_ctx ctx = { false, false };
1568                 struct ttm_resource evict_mem;
1569
1570                 evict_mem = bo->mem;
1571                 evict_mem.mm_node = NULL;
1572                 evict_mem.placement = TTM_PL_FLAG_CACHED;
1573                 evict_mem.mem_type = TTM_PL_SYSTEM;
1574
1575                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1576                 if (unlikely(ret != 0))
1577                         goto out;
1578         }
1579
1580         /**
1581          * Make sure BO is idle.
1582          */
1583
1584         ret = ttm_bo_wait(bo, false, false);
1585         if (unlikely(ret != 0))
1586                 goto out;
1587
1588         ttm_bo_unmap_virtual(bo);
1589
1590         /**
1591          * Swap out. Buffer will be swapped in again as soon as
1592          * anyone tries to access a ttm page.
1593          */
1594
1595         if (bo->bdev->driver->swap_notify)
1596                 bo->bdev->driver->swap_notify(bo);
1597
1598         ret = ttm_tt_swapout(bo->bdev, bo->ttm, bo->persistent_swap_storage);
1599 out:
1600
1601         /**
1602          *
1603          * Unreserve without putting on LRU to avoid swapping out an
1604          * already swapped buffer.
1605          */
1606         if (locked)
1607                 dma_resv_unlock(bo->base.resv);
1608         ttm_bo_put(bo);
1609         return ret;
1610 }
1611 EXPORT_SYMBOL(ttm_bo_swapout);
1612
1613 void ttm_bo_swapout_all(void)
1614 {
1615         struct ttm_operation_ctx ctx = {
1616                 .interruptible = false,
1617                 .no_wait_gpu = false
1618         };
1619
1620         while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0);
1621 }
1622 EXPORT_SYMBOL(ttm_bo_swapout_all);