drivers/gpu/drm/ttm/ttm_bo.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #define pr_fmt(fmt) "[TTM] " fmt
  32
  33 #include <drm/ttm/ttm_module.h>
  34 #include <drm/ttm/ttm_bo_driver.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <linux/jiffies.h>
  37 #include <linux/slab.h>
  38 #include <linux/sched.h>
  39 #include <linux/mm.h>
  40 #include <linux/file.h>
  41 #include <linux/module.h>
  42 #include <linux/atomic.h>
  43 #include <linux/reservation.h>
  44
  45 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  46
  47 static struct attribute ttm_bo_count = {
  48         .name = "bo_count",
  49         .mode = S_IRUGO
  50 };
  51
  52 /* default destructor */
  53 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  54 {
  55         kfree(bo);
  56 }
  57
  58 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
  59                                           uint32_t *mem_type)
  60 {
  61         int pos;
  62
  63         pos = ffs(place->flags & TTM_PL_MASK_MEM);
  64         if (unlikely(!pos))
  65                 return -EINVAL;
  66
  67         *mem_type = pos - 1;
  68         return 0;
  69 }
  70
  71 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
  72 {
  73         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  74         struct drm_printer p = drm_debug_printer(TTM_PFX);
  75
  76         pr_err("    has_type: %d\n", man->has_type);
  77         pr_err("    use_type: %d\n", man->use_type);
  78         pr_err("    flags: 0x%08X\n", man->flags);
  79         pr_err("    gpu_offset: 0x%08llX\n", man->gpu_offset);
  80         pr_err("    size: %llu\n", man->size);
  81         pr_err("    available_caching: 0x%08X\n", man->available_caching);
  82         pr_err("    default_caching: 0x%08X\n", man->default_caching);
  83         if (mem_type != TTM_PL_SYSTEM)
  84                 (*man->func->debug)(man, &p);
  85 }
  86
  87 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  88                                         struct ttm_placement *placement)
  89 {
  90         int i, ret, mem_type;
  91
  92         pr_err("No space for %p (%lu pages, %luK, %luM)\n",
  93                bo, bo->mem.num_pages, bo->mem.size >> 10,
  94                bo->mem.size >> 20);
  95         for (i = 0; i < placement->num_placement; i++) {
  96                 ret = ttm_mem_type_from_place(&placement->placement[i],
  97                                                 &mem_type);
  98                 if (ret)
  99                         return;
 100                 pr_err("  placement[%d]=0x%08X (%d)\n",
 101                        i, placement->placement[i].flags, mem_type);
 102                 ttm_mem_type_debug(bo->bdev, mem_type);
 103         }
 104 }
 105
 106 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 107                                   struct attribute *attr,
 108                                   char *buffer)
 109 {
 110         struct ttm_bo_global *glob =
 111                 container_of(kobj, struct ttm_bo_global, kobj);
 112
 113         return snprintf(buffer, PAGE_SIZE, "%d\n",
 114                                 atomic_read(&glob->bo_count));
 115 }
 116
 117 static struct attribute *ttm_bo_global_attrs[] = {
 118         &ttm_bo_count,
 119         NULL
 120 };
 121
 122 static const struct sysfs_ops ttm_bo_global_ops = {
 123         .show = &ttm_bo_global_show
 124 };
 125
 126 static struct kobj_type ttm_bo_glob_kobj_type  = {
 127         .release = &ttm_bo_global_kobj_release,
 128         .sysfs_ops = &ttm_bo_global_ops,
 129         .default_attrs = ttm_bo_global_attrs
 130 };
 131
 132
 133 static inline uint32_t ttm_bo_type_flags(unsigned type)
 134 {
 135         return 1 << (type);
 136 }
 137
 138 static void ttm_bo_release_list(struct kref *list_kref)
 139 {
 140         struct ttm_buffer_object *bo =
 141             container_of(list_kref, struct ttm_buffer_object, list_kref);
 142         struct ttm_bo_device *bdev = bo->bdev;
 143         size_t acc_size = bo->acc_size;
 144
 145         BUG_ON(kref_read(&bo->list_kref));
 146         BUG_ON(kref_read(&bo->kref));
 147         BUG_ON(atomic_read(&bo->cpu_writers));
 148         BUG_ON(bo->mem.mm_node != NULL);
 149         BUG_ON(!list_empty(&bo->lru));
 150         BUG_ON(!list_empty(&bo->ddestroy));
 151         ttm_tt_destroy(bo->ttm);
 152         atomic_dec(&bo->bdev->glob->bo_count);
 153         dma_fence_put(bo->moving);
 154         reservation_object_fini(&bo->ttm_resv);
 155         mutex_destroy(&bo->wu_mutex);
 156         bo->destroy(bo);
 157         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
 158 }
 159
 160 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 161 {
 162         struct ttm_bo_device *bdev = bo->bdev;
 163         struct ttm_mem_type_manager *man;
 164
 165         reservation_object_assert_held(bo->resv);
 166
 167         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 168                 BUG_ON(!list_empty(&bo->lru));
 169
 170                 man = &bdev->man[bo->mem.mem_type];
 171                 list_add_tail(&bo->lru, &man->lru[bo->priority]);
 172                 kref_get(&bo->list_kref);
 173
 174                 if (bo->ttm && !(bo->ttm->page_flags &
 175                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) {
 176                         list_add_tail(&bo->swap,
 177                                       &bdev->glob->swap_lru[bo->priority]);
 178                         kref_get(&bo->list_kref);
 179                 }
 180         }
 181 }
 182 EXPORT_SYMBOL(ttm_bo_add_to_lru);
 183
 184 static void ttm_bo_ref_bug(struct kref *list_kref)
 185 {
 186         BUG();
 187 }
 188
 189 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 190 {
 191         if (!list_empty(&bo->swap)) {
 192                 list_del_init(&bo->swap);
 193                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 194         }
 195         if (!list_empty(&bo->lru)) {
 196                 list_del_init(&bo->lru);
 197                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 198         }
 199
 200         /*
 201          * TODO: Add a driver hook to delete from
 202          * driver-specific LRU's here.
 203          */
 204 }
 205
 206 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
 207 {
 208         struct ttm_bo_global *glob = bo->bdev->glob;
 209
 210         spin_lock(&glob->lru_lock);
 211         ttm_bo_del_from_lru(bo);
 212         spin_unlock(&glob->lru_lock);
 213 }
 214 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 215
 216 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
 217 {
 218         reservation_object_assert_held(bo->resv);
 219
 220         ttm_bo_del_from_lru(bo);
 221         ttm_bo_add_to_lru(bo);
 222 }
 223 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 224
 225 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 226                                   struct ttm_mem_reg *mem, bool evict,
 227                                   struct ttm_operation_ctx *ctx)
 228 {
 229         struct ttm_bo_device *bdev = bo->bdev;
 230         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 231         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 232         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 233         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 234         int ret = 0;
 235
 236         if (old_is_pci || new_is_pci ||
 237             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 238                 ret = ttm_mem_io_lock(old_man, true);
 239                 if (unlikely(ret != 0))
 240                         goto out_err;
 241                 ttm_bo_unmap_virtual_locked(bo);
 242                 ttm_mem_io_unlock(old_man);
 243         }
 244
 245         /*
 246          * Create and bind a ttm if required.
 247          */
 248
 249         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 250                 if (bo->ttm == NULL) {
 251                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 252                         ret = ttm_tt_create(bo, zero);
 253                         if (ret)
 254                                 goto out_err;
 255                 }
 256
 257                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 258                 if (ret)
 259                         goto out_err;
 260
 261                 if (mem->mem_type != TTM_PL_SYSTEM) {
 262                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
 263                         if (ret)
 264                                 goto out_err;
 265                 }
 266
 267                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 268                         if (bdev->driver->move_notify)
 269                                 bdev->driver->move_notify(bo, evict, mem);
 270                         bo->mem = *mem;
 271                         mem->mm_node = NULL;
 272                         goto moved;
 273                 }
 274         }
 275
 276         if (bdev->driver->move_notify)
 277                 bdev->driver->move_notify(bo, evict, mem);
 278
 279         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 280             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 281                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 282         else if (bdev->driver->move)
 283                 ret = bdev->driver->move(bo, evict, ctx, mem);
 284         else
 285                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 286
 287         if (ret) {
 288                 if (bdev->driver->move_notify) {
 289                         struct ttm_mem_reg tmp_mem = *mem;
 290                         *mem = bo->mem;
 291                         bo->mem = tmp_mem;
 292                         bdev->driver->move_notify(bo, false, mem);
 293                         bo->mem = *mem;
 294                         *mem = tmp_mem;
 295                 }
 296
 297                 goto out_err;
 298         }
 299
 300 moved:
 301         if (bo->evicted) {
 302                 if (bdev->driver->invalidate_caches) {
 303                         ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
 304                         if (ret)
 305                                 pr_err("Can not flush read caches\n");
 306                 }
 307                 bo->evicted = false;
 308         }
 309
 310         if (bo->mem.mm_node)
 311                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 312                     bdev->man[bo->mem.mem_type].gpu_offset;
 313         else
 314                 bo->offset = 0;
 315
 316         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 317         return 0;
 318
 319 out_err:
 320         new_man = &bdev->man[bo->mem.mem_type];
 321         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 322                 ttm_tt_destroy(bo->ttm);
 323                 bo->ttm = NULL;
 324         }
 325
 326         return ret;
 327 }
 328
 329 /**
 330  * Call bo::reserved.
 331  * Will release GPU memory type usage on destruction.
 332  * This is the place to put in driver specific hooks to release
 333  * driver private resources.
 334  * Will release the bo::reserved lock.
 335  */
 336
 337 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 338 {
 339         if (bo->bdev->driver->move_notify)
 340                 bo->bdev->driver->move_notify(bo, false, NULL);
 341
 342         ttm_tt_destroy(bo->ttm);
 343         bo->ttm = NULL;
 344         ttm_bo_mem_put(bo, &bo->mem);
 345 }
 346
 347 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 348 {
 349         int r;
 350
 351         if (bo->resv == &bo->ttm_resv)
 352                 return 0;
 353
 354         BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
 355
 356         r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
 357         if (r)
 358                 reservation_object_unlock(&bo->ttm_resv);
 359
 360         return r;
 361 }
 362
 363 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 364 {
 365         struct reservation_object_list *fobj;
 366         struct dma_fence *fence;
 367         int i;
 368
 369         fobj = reservation_object_get_list(&bo->ttm_resv);
 370         fence = reservation_object_get_excl(&bo->ttm_resv);
 371         if (fence && !fence->ops->signaled)
 372                 dma_fence_enable_sw_signaling(fence);
 373
 374         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 375                 fence = rcu_dereference_protected(fobj->shared[i],
 376                                         reservation_object_held(bo->resv));
 377
 378                 if (!fence->ops->signaled)
 379                         dma_fence_enable_sw_signaling(fence);
 380         }
 381 }
 382
 383 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 384 {
 385         struct ttm_bo_device *bdev = bo->bdev;
 386         struct ttm_bo_global *glob = bdev->glob;
 387         int ret;
 388
 389         ret = ttm_bo_individualize_resv(bo);
 390         if (ret) {
 391                 /* Last resort, if we fail to allocate memory for the
 392                  * fences block for the BO to become idle
 393                  */
 394                 reservation_object_wait_timeout_rcu(bo->resv, true, false,
 395                                                     30 * HZ);
 396                 spin_lock(&glob->lru_lock);
 397                 goto error;
 398         }
 399
 400         spin_lock(&glob->lru_lock);
 401         ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
 402         if (!ret) {
 403                 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
 404                         ttm_bo_del_from_lru(bo);
 405                         spin_unlock(&glob->lru_lock);
 406                         if (bo->resv != &bo->ttm_resv)
 407                                 reservation_object_unlock(&bo->ttm_resv);
 408
 409                         ttm_bo_cleanup_memtype_use(bo);
 410                         reservation_object_unlock(bo->resv);
 411                         return;
 412                 }
 413
 414                 ttm_bo_flush_all_fences(bo);
 415
 416                 /*
 417                  * Make NO_EVICT bos immediately available to
 418                  * shrinkers, now that they are queued for
 419                  * destruction.
 420                  */
 421                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 422                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 423                         ttm_bo_add_to_lru(bo);
 424                 }
 425
 426                 reservation_object_unlock(bo->resv);
 427         }
 428         if (bo->resv != &bo->ttm_resv)
 429                 reservation_object_unlock(&bo->ttm_resv);
 430
 431 error:
 432         kref_get(&bo->list_kref);
 433         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 434         spin_unlock(&glob->lru_lock);
 435
 436         schedule_delayed_work(&bdev->wq,
 437                               ((HZ / 100) < 1) ? 1 : HZ / 100);
 438 }
 439
 440 /**
 441  * function ttm_bo_cleanup_refs
 442  * If bo idle, remove from delayed- and lru lists, and unref.
 443  * If not idle, do nothing.
 444  *
 445  * Must be called with lru_lock and reservation held, this function
 446  * will drop the lru lock and optionally the reservation lock before returning.
 447  *
 448  * @interruptible         Any sleeps should occur interruptibly.
 449  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 450  * @unlock_resv           Unlock the reservation lock as well.
 451  */
 452
 453 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 454                                bool interruptible, bool no_wait_gpu,
 455                                bool unlock_resv)
 456 {
 457         struct ttm_bo_global *glob = bo->bdev->glob;
 458         struct reservation_object *resv;
 459         int ret;
 460
 461         if (unlikely(list_empty(&bo->ddestroy)))
 462                 resv = bo->resv;
 463         else
 464                 resv = &bo->ttm_resv;
 465
 466         if (reservation_object_test_signaled_rcu(resv, true))
 467                 ret = 0;
 468         else
 469                 ret = -EBUSY;
 470
 471         if (ret && !no_wait_gpu) {
 472                 long lret;
 473
 474                 if (unlock_resv)
 475                         reservation_object_unlock(bo->resv);
 476                 spin_unlock(&glob->lru_lock);
 477
 478                 lret = reservation_object_wait_timeout_rcu(resv, true,
 479                                                            interruptible,
 480                                                            30 * HZ);
 481
 482                 if (lret < 0)
 483                         return lret;
 484                 else if (lret == 0)
 485                         return -EBUSY;
 486
 487                 spin_lock(&glob->lru_lock);
 488                 if (unlock_resv && !reservation_object_trylock(bo->resv)) {
 489                         /*
 490                          * We raced, and lost, someone else holds the reservation now,
 491                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 492                          *
 493                          * Even if it's not the case, because we finished waiting any
 494                          * delayed destruction would succeed, so just return success
 495                          * here.
 496                          */
 497                         spin_unlock(&glob->lru_lock);
 498                         return 0;
 499                 }
 500                 ret = 0;
 501         }
 502
 503         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 504                 if (unlock_resv)
 505                         reservation_object_unlock(bo->resv);
 506                 spin_unlock(&glob->lru_lock);
 507                 return ret;
 508         }
 509
 510         ttm_bo_del_from_lru(bo);
 511         list_del_init(&bo->ddestroy);
 512         kref_put(&bo->list_kref, ttm_bo_ref_bug);
 513
 514         spin_unlock(&glob->lru_lock);
 515         ttm_bo_cleanup_memtype_use(bo);
 516
 517         if (unlock_resv)
 518                 reservation_object_unlock(bo->resv);
 519
 520         return 0;
 521 }
 522
 523 /**
 524  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 525  * encountered buffers.
 526  */
 527 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 528 {
 529         struct ttm_bo_global *glob = bdev->glob;
 530         struct list_head removed;
 531         bool empty;
 532
 533         INIT_LIST_HEAD(&removed);
 534
 535         spin_lock(&glob->lru_lock);
 536         while (!list_empty(&bdev->ddestroy)) {
 537                 struct ttm_buffer_object *bo;
 538
 539                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 540                                       ddestroy);
 541                 kref_get(&bo->list_kref);
 542                 list_move_tail(&bo->ddestroy, &removed);
 543
 544                 if (remove_all || bo->resv != &bo->ttm_resv) {
 545                         spin_unlock(&glob->lru_lock);
 546                         reservation_object_lock(bo->resv, NULL);
 547
 548                         spin_lock(&glob->lru_lock);
 549                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 550
 551                 } else if (reservation_object_trylock(bo->resv)) {
 552                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 553                 } else {
 554                         spin_unlock(&glob->lru_lock);
 555                 }
 556
 557                 kref_put(&bo->list_kref, ttm_bo_release_list);
 558                 spin_lock(&glob->lru_lock);
 559         }
 560         list_splice_tail(&removed, &bdev->ddestroy);
 561         empty = list_empty(&bdev->ddestroy);
 562         spin_unlock(&glob->lru_lock);
 563
 564         return empty;
 565 }
 566
 567 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 568 {
 569         struct ttm_bo_device *bdev =
 570             container_of(work, struct ttm_bo_device, wq.work);
 571
 572         if (!ttm_bo_delayed_delete(bdev, false))
 573                 schedule_delayed_work(&bdev->wq,
 574                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 575 }
 576
 577 static void ttm_bo_release(struct kref *kref)
 578 {
 579         struct ttm_buffer_object *bo =
 580             container_of(kref, struct ttm_buffer_object, kref);
 581         struct ttm_bo_device *bdev = bo->bdev;
 582         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 583
 584         drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
 585         ttm_mem_io_lock(man, false);
 586         ttm_mem_io_free_vm(bo);
 587         ttm_mem_io_unlock(man);
 588         ttm_bo_cleanup_refs_or_queue(bo);
 589         kref_put(&bo->list_kref, ttm_bo_release_list);
 590 }
 591
 592 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 593 {
 594         struct ttm_buffer_object *bo = *p_bo;
 595
 596         *p_bo = NULL;
 597         kref_put(&bo->kref, ttm_bo_release);
 598 }
 599 EXPORT_SYMBOL(ttm_bo_unref);
 600
 601 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 602 {
 603         return cancel_delayed_work_sync(&bdev->wq);
 604 }
 605 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 606
 607 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 608 {
 609         if (resched)
 610                 schedule_delayed_work(&bdev->wq,
 611                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 612 }
 613 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 614
 615 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 616                         struct ttm_operation_ctx *ctx)
 617 {
 618         struct ttm_bo_device *bdev = bo->bdev;
 619         struct ttm_mem_reg evict_mem;
 620         struct ttm_placement placement;
 621         int ret = 0;
 622
 623         reservation_object_assert_held(bo->resv);
 624
 625         placement.num_placement = 0;
 626         placement.num_busy_placement = 0;
 627         bdev->driver->evict_flags(bo, &placement);
 628
 629         if (!placement.num_placement && !placement.num_busy_placement) {
 630                 ret = ttm_bo_pipeline_gutting(bo);
 631                 if (ret)
 632                         return ret;
 633
 634                 return ttm_tt_create(bo, false);
 635         }
 636
 637         evict_mem = bo->mem;
 638         evict_mem.mm_node = NULL;
 639         evict_mem.bus.io_reserved_vm = false;
 640         evict_mem.bus.io_reserved_count = 0;
 641
 642         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 643         if (ret) {
 644                 if (ret != -ERESTARTSYS) {
 645                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 646                                bo);
 647                         ttm_bo_mem_space_debug(bo, &placement);
 648                 }
 649                 goto out;
 650         }
 651
 652         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 653         if (unlikely(ret)) {
 654                 if (ret != -ERESTARTSYS)
 655                         pr_err("Buffer eviction failed\n");
 656                 ttm_bo_mem_put(bo, &evict_mem);
 657                 goto out;
 658         }
 659         bo->evicted = true;
 660 out:
 661         return ret;
 662 }
 663
 664 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 665                               const struct ttm_place *place)
 666 {
 667         /* Don't evict this BO if it's outside of the
 668          * requested placement range
 669          */
 670         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 671             (place->lpfn && place->lpfn <= bo->mem.start))
 672                 return false;
 673
 674         return true;
 675 }
 676 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 677
 678 /**
 679  * Check the target bo is allowable to be evicted or swapout, including cases:
 680  *
 681  * a. if share same reservation object with ctx->resv, have assumption
 682  * reservation objects should already be locked, so not lock again and
 683  * return true directly when either the opreation allow_reserved_eviction
 684  * or the target bo already is in delayed free list;
 685  *
 686  * b. Otherwise, trylock it.
 687  */
 688 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 689                         struct ttm_operation_ctx *ctx, bool *locked)
 690 {
 691         bool ret = false;
 692
 693         *locked = false;
 694         if (bo->resv == ctx->resv) {
 695                 reservation_object_assert_held(bo->resv);
 696                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
 697                     || !list_empty(&bo->ddestroy))
 698                         ret = true;
 699         } else {
 700                 *locked = reservation_object_trylock(bo->resv);
 701                 ret = *locked;
 702         }
 703
 704         return ret;
 705 }
 706
 707 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 708                                uint32_t mem_type,
 709                                const struct ttm_place *place,
 710                                struct ttm_operation_ctx *ctx)
 711 {
 712         struct ttm_bo_global *glob = bdev->glob;
 713         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 714         struct ttm_buffer_object *bo = NULL;
 715         bool locked = false;
 716         unsigned i;
 717         int ret;
 718
 719         spin_lock(&glob->lru_lock);
 720         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 721                 list_for_each_entry(bo, &man->lru[i], lru) {
 722                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked))
 723                                 continue;
 724
 725                         if (place && !bdev->driver->eviction_valuable(bo,
 726                                                                       place)) {
 727                                 if (locked)
 728                                         reservation_object_unlock(bo->resv);
 729                                 continue;
 730                         }
 731                         break;
 732                 }
 733
 734                 /* If the inner loop terminated early, we have our candidate */
 735                 if (&bo->lru != &man->lru[i])
 736                         break;
 737
 738                 bo = NULL;
 739         }
 740
 741         if (!bo) {
 742                 spin_unlock(&glob->lru_lock);
 743                 return -EBUSY;
 744         }
 745
 746         kref_get(&bo->list_kref);
 747
 748         if (!list_empty(&bo->ddestroy)) {
 749                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 750                                           ctx->no_wait_gpu, locked);
 751                 kref_put(&bo->list_kref, ttm_bo_release_list);
 752                 return ret;
 753         }
 754
 755         ttm_bo_del_from_lru(bo);
 756         spin_unlock(&glob->lru_lock);
 757
 758         ret = ttm_bo_evict(bo, ctx);
 759         if (locked) {
 760                 ttm_bo_unreserve(bo);
 761         } else {
 762                 spin_lock(&glob->lru_lock);
 763                 ttm_bo_add_to_lru(bo);
 764                 spin_unlock(&glob->lru_lock);
 765         }
 766
 767         kref_put(&bo->list_kref, ttm_bo_release_list);
 768         return ret;
 769 }
 770
 771 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 772 {
 773         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 774
 775         if (mem->mm_node)
 776                 (*man->func->put_node)(man, mem);
 777 }
 778 EXPORT_SYMBOL(ttm_bo_mem_put);
 779
 780 /**
 781  * Add the last move fence to the BO and reserve a new shared slot.
 782  */
 783 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 784                                  struct ttm_mem_type_manager *man,
 785                                  struct ttm_mem_reg *mem)
 786 {
 787         struct dma_fence *fence;
 788         int ret;
 789
 790         spin_lock(&man->move_lock);
 791         fence = dma_fence_get(man->move);
 792         spin_unlock(&man->move_lock);
 793
 794         if (fence) {
 795                 reservation_object_add_shared_fence(bo->resv, fence);
 796
 797                 ret = reservation_object_reserve_shared(bo->resv);
 798                 if (unlikely(ret))
 799                         return ret;
 800
 801                 dma_fence_put(bo->moving);
 802                 bo->moving = fence;
 803         }
 804
 805         return 0;
 806 }
 807
 808 /**
 809  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 810  * space, or we've evicted everything and there isn't enough space.
 811  */
 812 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 813                                         uint32_t mem_type,
 814                                         const struct ttm_place *place,
 815                                         struct ttm_mem_reg *mem,
 816                                         struct ttm_operation_ctx *ctx)
 817 {
 818         struct ttm_bo_device *bdev = bo->bdev;
 819         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 820         int ret;
 821
 822         do {
 823                 ret = (*man->func->get_node)(man, bo, place, mem);
 824                 if (unlikely(ret != 0))
 825                         return ret;
 826                 if (mem->mm_node)
 827                         break;
 828                 ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);
 829                 if (unlikely(ret != 0))
 830                         return ret;
 831         } while (1);
 832         mem->mem_type = mem_type;
 833         return ttm_bo_add_move_fence(bo, man, mem);
 834 }
 835
 836 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 837                                       uint32_t cur_placement,
 838                                       uint32_t proposed_placement)
 839 {
 840         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 841         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 842
 843         /**
 844          * Keep current caching if possible.
 845          */
 846
 847         if ((cur_placement & caching) != 0)
 848                 result |= (cur_placement & caching);
 849         else if ((man->default_caching & caching) != 0)
 850                 result |= man->default_caching;
 851         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 852                 result |= TTM_PL_FLAG_CACHED;
 853         else if ((TTM_PL_FLAG_WC & caching) != 0)
 854                 result |= TTM_PL_FLAG_WC;
 855         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 856                 result |= TTM_PL_FLAG_UNCACHED;
 857
 858         return result;
 859 }
 860
 861 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 862                                  uint32_t mem_type,
 863                                  const struct ttm_place *place,
 864                                  uint32_t *masked_placement)
 865 {
 866         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 867
 868         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 869                 return false;
 870
 871         if ((place->flags & man->available_caching) == 0)
 872                 return false;
 873
 874         cur_flags |= (place->flags & man->available_caching);
 875
 876         *masked_placement = cur_flags;
 877         return true;
 878 }
 879
 880 /**
 881  * Creates space for memory region @mem according to its type.
 882  *
 883  * This function first searches for free space in compatible memory types in
 884  * the priority order defined by the driver.  If free space isn't found, then
 885  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 886  * space.
 887  */
 888 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 889                         struct ttm_placement *placement,
 890                         struct ttm_mem_reg *mem,
 891                         struct ttm_operation_ctx *ctx)
 892 {
 893         struct ttm_bo_device *bdev = bo->bdev;
 894         struct ttm_mem_type_manager *man;
 895         uint32_t mem_type = TTM_PL_SYSTEM;
 896         uint32_t cur_flags = 0;
 897         bool type_found = false;
 898         bool type_ok = false;
 899         bool has_erestartsys = false;
 900         int i, ret;
 901
 902         ret = reservation_object_reserve_shared(bo->resv);
 903         if (unlikely(ret))
 904                 return ret;
 905
 906         mem->mm_node = NULL;
 907         for (i = 0; i < placement->num_placement; ++i) {
 908                 const struct ttm_place *place = &placement->placement[i];
 909
 910                 ret = ttm_mem_type_from_place(place, &mem_type);
 911                 if (ret)
 912                         return ret;
 913                 man = &bdev->man[mem_type];
 914                 if (!man->has_type || !man->use_type)
 915                         continue;
 916
 917                 type_ok = ttm_bo_mt_compatible(man, mem_type, place,
 918                                                 &cur_flags);
 919
 920                 if (!type_ok)
 921                         continue;
 922
 923                 type_found = true;
 924                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 925                                                   cur_flags);
 926                 /*
 927                  * Use the access and other non-mapping-related flag bits from
 928                  * the memory placement flags to the current flags
 929                  */
 930                 ttm_flag_masked(&cur_flags, place->flags,
 931                                 ~TTM_PL_MASK_MEMTYPE);
 932
 933                 if (mem_type == TTM_PL_SYSTEM)
 934                         break;
 935
 936                 ret = (*man->func->get_node)(man, bo, place, mem);
 937                 if (unlikely(ret))
 938                         return ret;
 939
 940                 if (mem->mm_node) {
 941                         ret = ttm_bo_add_move_fence(bo, man, mem);
 942                         if (unlikely(ret)) {
 943                                 (*man->func->put_node)(man, mem);
 944                                 return ret;
 945                         }
 946                         break;
 947                 }
 948         }
 949
 950         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
 951                 mem->mem_type = mem_type;
 952                 mem->placement = cur_flags;
 953                 return 0;
 954         }
 955
 956         for (i = 0; i < placement->num_busy_placement; ++i) {
 957                 const struct ttm_place *place = &placement->busy_placement[i];
 958
 959                 ret = ttm_mem_type_from_place(place, &mem_type);
 960                 if (ret)
 961                         return ret;
 962                 man = &bdev->man[mem_type];
 963                 if (!man->has_type || !man->use_type)
 964                         continue;
 965                 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
 966                         continue;
 967
 968                 type_found = true;
 969                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 970                                                   cur_flags);
 971                 /*
 972                  * Use the access and other non-mapping-related flag bits from
 973                  * the memory placement flags to the current flags
 974                  */
 975                 ttm_flag_masked(&cur_flags, place->flags,
 976                                 ~TTM_PL_MASK_MEMTYPE);
 977
 978                 if (mem_type == TTM_PL_SYSTEM) {
 979                         mem->mem_type = mem_type;
 980                         mem->placement = cur_flags;
 981                         mem->mm_node = NULL;
 982                         return 0;
 983                 }
 984
 985                 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx);
 986                 if (ret == 0 && mem->mm_node) {
 987                         mem->placement = cur_flags;
 988                         return 0;
 989                 }
 990                 if (ret == -ERESTARTSYS)
 991                         has_erestartsys = true;
 992         }
 993
 994         if (!type_found) {
 995                 pr_err(TTM_PFX "No compatible memory type found\n");
 996                 return -EINVAL;
 997         }
 998
 999         return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
1000 }
1001 EXPORT_SYMBOL(ttm_bo_mem_space);
1002
1003 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1004                               struct ttm_placement *placement,
1005                               struct ttm_operation_ctx *ctx)
1006 {
1007         int ret = 0;
1008         struct ttm_mem_reg mem;
1009
1010         reservation_object_assert_held(bo->resv);
1011
1012         mem.num_pages = bo->num_pages;
1013         mem.size = mem.num_pages << PAGE_SHIFT;
1014         mem.page_alignment = bo->mem.page_alignment;
1015         mem.bus.io_reserved_vm = false;
1016         mem.bus.io_reserved_count = 0;
1017         /*
1018          * Determine where to move the buffer.
1019          */
1020         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1021         if (ret)
1022                 goto out_unlock;
1023         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1024 out_unlock:
1025         if (ret && mem.mm_node)
1026                 ttm_bo_mem_put(bo, &mem);
1027         return ret;
1028 }
1029
1030 static bool ttm_bo_places_compat(const struct ttm_place *places,
1031                                  unsigned num_placement,
1032                                  struct ttm_mem_reg *mem,
1033                                  uint32_t *new_flags)
1034 {
1035         unsigned i;
1036
1037         for (i = 0; i < num_placement; i++) {
1038                 const struct ttm_place *heap = &places[i];
1039
1040                 if (mem->mm_node && (mem->start < heap->fpfn ||
1041                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1042                         continue;
1043
1044                 *new_flags = heap->flags;
1045                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1046                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1047                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1048                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1049                         return true;
1050         }
1051         return false;
1052 }
1053
1054 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1055                        struct ttm_mem_reg *mem,
1056                        uint32_t *new_flags)
1057 {
1058         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1059                                  mem, new_flags))
1060                 return true;
1061
1062         if ((placement->busy_placement != placement->placement ||
1063              placement->num_busy_placement > placement->num_placement) &&
1064             ttm_bo_places_compat(placement->busy_placement,
1065                                  placement->num_busy_placement,
1066                                  mem, new_flags))
1067                 return true;
1068
1069         return false;
1070 }
1071 EXPORT_SYMBOL(ttm_bo_mem_compat);
1072
1073 int ttm_bo_validate(struct ttm_buffer_object *bo,
1074                     struct ttm_placement *placement,
1075                     struct ttm_operation_ctx *ctx)
1076 {
1077         int ret;
1078         uint32_t new_flags;
1079
1080         reservation_object_assert_held(bo->resv);
1081         /*
1082          * Check whether we need to move buffer.
1083          */
1084         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1085                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1086                 if (ret)
1087                         return ret;
1088         } else {
1089                 /*
1090                  * Use the access and other non-mapping-related flag bits from
1091                  * the compatible memory placement flags to the active flags
1092                  */
1093                 ttm_flag_masked(&bo->mem.placement, new_flags,
1094                                 ~TTM_PL_MASK_MEMTYPE);
1095         }
1096         /*
1097          * We might need to add a TTM.
1098          */
1099         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1100                 ret = ttm_tt_create(bo, true);
1101                 if (ret)
1102                         return ret;
1103         }
1104         return 0;
1105 }
1106 EXPORT_SYMBOL(ttm_bo_validate);
1107
1108 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1109                          struct ttm_buffer_object *bo,
1110                          unsigned long size,
1111                          enum ttm_bo_type type,
1112                          struct ttm_placement *placement,
1113                          uint32_t page_alignment,
1114                          struct ttm_operation_ctx *ctx,
1115                          size_t acc_size,
1116                          struct sg_table *sg,
1117                          struct reservation_object *resv,
1118                          void (*destroy) (struct ttm_buffer_object *))
1119 {
1120         int ret = 0;
1121         unsigned long num_pages;
1122         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1123         bool locked;
1124
1125         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1126         if (ret) {
1127                 pr_err("Out of kernel memory\n");
1128                 if (destroy)
1129                         (*destroy)(bo);
1130                 else
1131                         kfree(bo);
1132                 return -ENOMEM;
1133         }
1134
1135         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1136         if (num_pages == 0) {
1137                 pr_err("Illegal buffer object size\n");
1138                 if (destroy)
1139                         (*destroy)(bo);
1140                 else
1141                         kfree(bo);
1142                 ttm_mem_global_free(mem_glob, acc_size);
1143                 return -EINVAL;
1144         }
1145         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1146
1147         kref_init(&bo->kref);
1148         kref_init(&bo->list_kref);
1149         atomic_set(&bo->cpu_writers, 0);
1150         INIT_LIST_HEAD(&bo->lru);
1151         INIT_LIST_HEAD(&bo->ddestroy);
1152         INIT_LIST_HEAD(&bo->swap);
1153         INIT_LIST_HEAD(&bo->io_reserve_lru);
1154         mutex_init(&bo->wu_mutex);
1155         bo->bdev = bdev;
1156         bo->type = type;
1157         bo->num_pages = num_pages;
1158         bo->mem.size = num_pages << PAGE_SHIFT;
1159         bo->mem.mem_type = TTM_PL_SYSTEM;
1160         bo->mem.num_pages = bo->num_pages;
1161         bo->mem.mm_node = NULL;
1162         bo->mem.page_alignment = page_alignment;
1163         bo->mem.bus.io_reserved_vm = false;
1164         bo->mem.bus.io_reserved_count = 0;
1165         bo->moving = NULL;
1166         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1167         bo->acc_size = acc_size;
1168         bo->sg = sg;
1169         if (resv) {
1170                 bo->resv = resv;
1171                 reservation_object_assert_held(bo->resv);
1172         } else {
1173                 bo->resv = &bo->ttm_resv;
1174         }
1175         reservation_object_init(&bo->ttm_resv);
1176         atomic_inc(&bo->bdev->glob->bo_count);
1177         drm_vma_node_reset(&bo->vma_node);
1178         bo->priority = 0;
1179
1180         /*
1181          * For ttm_bo_type_device buffers, allocate
1182          * address space from the device.
1183          */
1184         if (bo->type == ttm_bo_type_device ||
1185             bo->type == ttm_bo_type_sg)
1186                 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
1187                                          bo->mem.num_pages);
1188
1189         /* passed reservation objects should already be locked,
1190          * since otherwise lockdep will be angered in radeon.
1191          */
1192         if (!resv) {
1193                 locked = reservation_object_trylock(bo->resv);
1194                 WARN_ON(!locked);
1195         }
1196
1197         if (likely(!ret))
1198                 ret = ttm_bo_validate(bo, placement, ctx);
1199
1200         if (unlikely(ret)) {
1201                 if (!resv)
1202                         ttm_bo_unreserve(bo);
1203
1204                 ttm_bo_unref(&bo);
1205                 return ret;
1206         }
1207
1208         if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
1209                 spin_lock(&bdev->glob->lru_lock);
1210                 ttm_bo_add_to_lru(bo);
1211                 spin_unlock(&bdev->glob->lru_lock);
1212         }
1213
1214         return ret;
1215 }
1216 EXPORT_SYMBOL(ttm_bo_init_reserved);
1217
1218 int ttm_bo_init(struct ttm_bo_device *bdev,
1219                 struct ttm_buffer_object *bo,
1220                 unsigned long size,
1221                 enum ttm_bo_type type,
1222                 struct ttm_placement *placement,
1223                 uint32_t page_alignment,
1224                 bool interruptible,
1225                 size_t acc_size,
1226                 struct sg_table *sg,
1227                 struct reservation_object *resv,
1228                 void (*destroy) (struct ttm_buffer_object *))
1229 {
1230         struct ttm_operation_ctx ctx = { interruptible, false };
1231         int ret;
1232
1233         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1234                                    page_alignment, &ctx, acc_size,
1235                                    sg, resv, destroy);
1236         if (ret)
1237                 return ret;
1238
1239         if (!resv)
1240                 ttm_bo_unreserve(bo);
1241
1242         return 0;
1243 }
1244 EXPORT_SYMBOL(ttm_bo_init);
1245
1246 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1247                        unsigned long bo_size,
1248                        unsigned struct_size)
1249 {
1250         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1251         size_t size = 0;
1252
1253         size += ttm_round_pot(struct_size);
1254         size += ttm_round_pot(npages * sizeof(void *));
1255         size += ttm_round_pot(sizeof(struct ttm_tt));
1256         return size;
1257 }
1258 EXPORT_SYMBOL(ttm_bo_acc_size);
1259
1260 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1261                            unsigned long bo_size,
1262                            unsigned struct_size)
1263 {
1264         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1265         size_t size = 0;
1266
1267         size += ttm_round_pot(struct_size);
1268         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1269         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1270         return size;
1271 }
1272 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1273
1274 int ttm_bo_create(struct ttm_bo_device *bdev,
1275                         unsigned long size,
1276                         enum ttm_bo_type type,
1277                         struct ttm_placement *placement,
1278                         uint32_t page_alignment,
1279                         bool interruptible,
1280                         struct ttm_buffer_object **p_bo)
1281 {
1282         struct ttm_buffer_object *bo;
1283         size_t acc_size;
1284         int ret;
1285
1286         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1287         if (unlikely(bo == NULL))
1288                 return -ENOMEM;
1289
1290         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1291         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1292                           interruptible, acc_size,
1293                           NULL, NULL, NULL);
1294         if (likely(ret == 0))
1295                 *p_bo = bo;
1296
1297         return ret;
1298 }
1299 EXPORT_SYMBOL(ttm_bo_create);
1300
1301 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1302                                    unsigned mem_type)
1303 {
1304         struct ttm_operation_ctx ctx = {
1305                 .interruptible = false,
1306                 .no_wait_gpu = false,
1307                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1308         };
1309         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1310         struct ttm_bo_global *glob = bdev->glob;
1311         struct dma_fence *fence;
1312         int ret;
1313         unsigned i;
1314
1315         /*
1316          * Can't use standard list traversal since we're unlocking.
1317          */
1318
1319         spin_lock(&glob->lru_lock);
1320         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1321                 while (!list_empty(&man->lru[i])) {
1322                         spin_unlock(&glob->lru_lock);
1323                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
1324                         if (ret)
1325                                 return ret;
1326                         spin_lock(&glob->lru_lock);
1327                 }
1328         }
1329         spin_unlock(&glob->lru_lock);
1330
1331         spin_lock(&man->move_lock);
1332         fence = dma_fence_get(man->move);
1333         spin_unlock(&man->move_lock);
1334
1335         if (fence) {
1336                 ret = dma_fence_wait(fence, false);
1337                 dma_fence_put(fence);
1338                 if (ret)
1339                         return ret;
1340         }
1341
1342         return 0;
1343 }
1344
1345 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1346 {
1347         struct ttm_mem_type_manager *man;
1348         int ret = -EINVAL;
1349
1350         if (mem_type >= TTM_NUM_MEM_TYPES) {
1351                 pr_err("Illegal memory type %d\n", mem_type);
1352                 return ret;
1353         }
1354         man = &bdev->man[mem_type];
1355
1356         if (!man->has_type) {
1357                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1358                        mem_type);
1359                 return ret;
1360         }
1361
1362         man->use_type = false;
1363         man->has_type = false;
1364
1365         ret = 0;
1366         if (mem_type > 0) {
1367                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1368                 if (ret) {
1369                         pr_err("Cleanup eviction failed\n");
1370                         return ret;
1371                 }
1372
1373                 ret = (*man->func->takedown)(man);
1374         }
1375
1376         dma_fence_put(man->move);
1377         man->move = NULL;
1378
1379         return ret;
1380 }
1381 EXPORT_SYMBOL(ttm_bo_clean_mm);
1382
1383 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1384 {
1385         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1386
1387         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1388                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1389                 return -EINVAL;
1390         }
1391
1392         if (!man->has_type) {
1393                 pr_err("Memory type %u has not been initialized\n", mem_type);
1394                 return 0;
1395         }
1396
1397         return ttm_bo_force_list_clean(bdev, mem_type);
1398 }
1399 EXPORT_SYMBOL(ttm_bo_evict_mm);
1400
1401 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1402                         unsigned long p_size)
1403 {
1404         int ret;
1405         struct ttm_mem_type_manager *man;
1406         unsigned i;
1407
1408         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1409         man = &bdev->man[type];
1410         BUG_ON(man->has_type);
1411         man->io_reserve_fastpath = true;
1412         man->use_io_reserve_lru = false;
1413         mutex_init(&man->io_reserve_mutex);
1414         spin_lock_init(&man->move_lock);
1415         INIT_LIST_HEAD(&man->io_reserve_lru);
1416
1417         ret = bdev->driver->init_mem_type(bdev, type, man);
1418         if (ret)
1419                 return ret;
1420         man->bdev = bdev;
1421
1422         if (type != TTM_PL_SYSTEM) {
1423                 ret = (*man->func->init)(man, p_size);
1424                 if (ret)
1425                         return ret;
1426         }
1427         man->has_type = true;
1428         man->use_type = true;
1429         man->size = p_size;
1430
1431         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1432                 INIT_LIST_HEAD(&man->lru[i]);
1433         man->move = NULL;
1434
1435         return 0;
1436 }
1437 EXPORT_SYMBOL(ttm_bo_init_mm);
1438
1439 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1440 {
1441         struct ttm_bo_global *glob =
1442                 container_of(kobj, struct ttm_bo_global, kobj);
1443
1444         __free_page(glob->dummy_read_page);
1445         kfree(glob);
1446 }
1447
1448 void ttm_bo_global_release(struct drm_global_reference *ref)
1449 {
1450         struct ttm_bo_global *glob = ref->object;
1451
1452         kobject_del(&glob->kobj);
1453         kobject_put(&glob->kobj);
1454 }
1455 EXPORT_SYMBOL(ttm_bo_global_release);
1456
1457 int ttm_bo_global_init(struct drm_global_reference *ref)
1458 {
1459         struct ttm_bo_global_ref *bo_ref =
1460                 container_of(ref, struct ttm_bo_global_ref, ref);
1461         struct ttm_bo_global *glob = ref->object;
1462         int ret;
1463         unsigned i;
1464
1465         mutex_init(&glob->device_list_mutex);
1466         spin_lock_init(&glob->lru_lock);
1467         glob->mem_glob = bo_ref->mem_glob;
1468         glob->mem_glob->bo_glob = glob;
1469         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1470
1471         if (unlikely(glob->dummy_read_page == NULL)) {
1472                 ret = -ENOMEM;
1473                 goto out_no_drp;
1474         }
1475
1476         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1477                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1478         INIT_LIST_HEAD(&glob->device_list);
1479         atomic_set(&glob->bo_count, 0);
1480
1481         ret = kobject_init_and_add(
1482                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1483         if (unlikely(ret != 0))
1484                 kobject_put(&glob->kobj);
1485         return ret;
1486 out_no_drp:
1487         kfree(glob);
1488         return ret;
1489 }
1490 EXPORT_SYMBOL(ttm_bo_global_init);
1491
1492
1493 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1494 {
1495         int ret = 0;
1496         unsigned i = TTM_NUM_MEM_TYPES;
1497         struct ttm_mem_type_manager *man;
1498         struct ttm_bo_global *glob = bdev->glob;
1499
1500         while (i--) {
1501                 man = &bdev->man[i];
1502                 if (man->has_type) {
1503                         man->use_type = false;
1504                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1505                                 ret = -EBUSY;
1506                                 pr_err("DRM memory manager type %d is not clean\n",
1507                                        i);
1508                         }
1509                         man->has_type = false;
1510                 }
1511         }
1512
1513         mutex_lock(&glob->device_list_mutex);
1514         list_del(&bdev->device_list);
1515         mutex_unlock(&glob->device_list_mutex);
1516
1517         cancel_delayed_work_sync(&bdev->wq);
1518
1519         if (ttm_bo_delayed_delete(bdev, true))
1520                 pr_debug("Delayed destroy list was clean\n");
1521
1522         spin_lock(&glob->lru_lock);
1523         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1524                 if (list_empty(&bdev->man[0].lru[0]))
1525                         pr_debug("Swap list %d was clean\n", i);
1526         spin_unlock(&glob->lru_lock);
1527
1528         drm_vma_offset_manager_destroy(&bdev->vma_manager);
1529
1530         return ret;
1531 }
1532 EXPORT_SYMBOL(ttm_bo_device_release);
1533
1534 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1535                        struct ttm_bo_global *glob,
1536                        struct ttm_bo_driver *driver,
1537                        struct address_space *mapping,
1538                        uint64_t file_page_offset,
1539                        bool need_dma32)
1540 {
1541         int ret = -EINVAL;
1542
1543         bdev->driver = driver;
1544
1545         memset(bdev->man, 0, sizeof(bdev->man));
1546
1547         /*
1548          * Initialize the system memory buffer type.
1549          * Other types need to be driver / IOCTL initialized.
1550          */
1551         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1552         if (unlikely(ret != 0))
1553                 goto out_no_sys;
1554
1555         drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
1556                                     0x10000000);
1557         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1558         INIT_LIST_HEAD(&bdev->ddestroy);
1559         bdev->dev_mapping = mapping;
1560         bdev->glob = glob;
1561         bdev->need_dma32 = need_dma32;
1562         mutex_lock(&glob->device_list_mutex);
1563         list_add_tail(&bdev->device_list, &glob->device_list);
1564         mutex_unlock(&glob->device_list_mutex);
1565
1566         return 0;
1567 out_no_sys:
1568         return ret;
1569 }
1570 EXPORT_SYMBOL(ttm_bo_device_init);
1571
1572 /*
1573  * buffer object vm functions.
1574  */
1575
1576 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1577 {
1578         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1579
1580         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1581                 if (mem->mem_type == TTM_PL_SYSTEM)
1582                         return false;
1583
1584                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1585                         return false;
1586
1587                 if (mem->placement & TTM_PL_FLAG_CACHED)
1588                         return false;
1589         }
1590         return true;
1591 }
1592
1593 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1594 {
1595         struct ttm_bo_device *bdev = bo->bdev;
1596
1597         drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1598         ttm_mem_io_free_vm(bo);
1599 }
1600
1601 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1602 {
1603         struct ttm_bo_device *bdev = bo->bdev;
1604         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1605
1606         ttm_mem_io_lock(man, false);
1607         ttm_bo_unmap_virtual_locked(bo);
1608         ttm_mem_io_unlock(man);
1609 }
1610
1611
1612 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1613
1614 int ttm_bo_wait(struct ttm_buffer_object *bo,
1615                 bool interruptible, bool no_wait)
1616 {
1617         long timeout = 15 * HZ;
1618
1619         if (no_wait) {
1620                 if (reservation_object_test_signaled_rcu(bo->resv, true))
1621                         return 0;
1622                 else
1623                         return -EBUSY;
1624         }
1625
1626         timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
1627                                                       interruptible, timeout);
1628         if (timeout < 0)
1629                 return timeout;
1630
1631         if (timeout == 0)
1632                 return -EBUSY;
1633
1634         reservation_object_add_excl_fence(bo->resv, NULL);
1635         return 0;
1636 }
1637 EXPORT_SYMBOL(ttm_bo_wait);
1638
1639 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1640 {
1641         int ret = 0;
1642
1643         /*
1644          * Using ttm_bo_reserve makes sure the lru lists are updated.
1645          */
1646
1647         ret = ttm_bo_reserve(bo, true, no_wait, NULL);
1648         if (unlikely(ret != 0))
1649                 return ret;
1650         ret = ttm_bo_wait(bo, true, no_wait);
1651         if (likely(ret == 0))
1652                 atomic_inc(&bo->cpu_writers);
1653         ttm_bo_unreserve(bo);
1654         return ret;
1655 }
1656 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1657
1658 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1659 {
1660         atomic_dec(&bo->cpu_writers);
1661 }
1662 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1663
1664 /**
1665  * A buffer object shrink method that tries to swap out the first
1666  * buffer object on the bo_global::swap_lru list.
1667  */
1668 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1669 {
1670         struct ttm_buffer_object *bo;
1671         int ret = -EBUSY;
1672         bool locked;
1673         unsigned i;
1674
1675         spin_lock(&glob->lru_lock);
1676         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1677                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1678                         if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) {
1679                                 ret = 0;
1680                                 break;
1681                         }
1682                 }
1683                 if (!ret)
1684                         break;
1685         }
1686
1687         if (ret) {
1688                 spin_unlock(&glob->lru_lock);
1689                 return ret;
1690         }
1691
1692         kref_get(&bo->list_kref);
1693
1694         if (!list_empty(&bo->ddestroy)) {
1695                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1696                 kref_put(&bo->list_kref, ttm_bo_release_list);
1697                 return ret;
1698         }
1699
1700         ttm_bo_del_from_lru(bo);
1701         spin_unlock(&glob->lru_lock);
1702
1703         /**
1704          * Move to system cached
1705          */
1706
1707         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1708             bo->ttm->caching_state != tt_cached) {
1709                 struct ttm_operation_ctx ctx = { false, false };
1710                 struct ttm_mem_reg evict_mem;
1711
1712                 evict_mem = bo->mem;
1713                 evict_mem.mm_node = NULL;
1714                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1715                 evict_mem.mem_type = TTM_PL_SYSTEM;
1716
1717                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1718                 if (unlikely(ret != 0))
1719                         goto out;
1720         }
1721
1722         /**
1723          * Make sure BO is idle.
1724          */
1725
1726         ret = ttm_bo_wait(bo, false, false);
1727         if (unlikely(ret != 0))
1728                 goto out;
1729
1730         ttm_bo_unmap_virtual(bo);
1731
1732         /**
1733          * Swap out. Buffer will be swapped in again as soon as
1734          * anyone tries to access a ttm page.
1735          */
1736
1737         if (bo->bdev->driver->swap_notify)
1738                 bo->bdev->driver->swap_notify(bo);
1739
1740         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1741 out:
1742
1743         /**
1744          *
1745          * Unreserve without putting on LRU to avoid swapping out an
1746          * already swapped buffer.
1747          */
1748         if (locked)
1749                 reservation_object_unlock(bo->resv);
1750         kref_put(&bo->list_kref, ttm_bo_release_list);
1751         return ret;
1752 }
1753 EXPORT_SYMBOL(ttm_bo_swapout);
1754
1755 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1756 {
1757         struct ttm_operation_ctx ctx = {
1758                 .interruptible = false,
1759                 .no_wait_gpu = false
1760         };
1761
1762         while (ttm_bo_swapout(bdev->glob, &ctx) == 0)
1763                 ;
1764 }
1765 EXPORT_SYMBOL(ttm_bo_swapout_all);
1766
1767 /**
1768  * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
1769  * unreserved
1770  *
1771  * @bo: Pointer to buffer
1772  */
1773 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
1774 {
1775         int ret;
1776
1777         /*
1778          * In the absense of a wait_unlocked API,
1779          * Use the bo::wu_mutex to avoid triggering livelocks due to
1780          * concurrent use of this function. Note that this use of
1781          * bo::wu_mutex can go away if we change locking order to
1782          * mmap_sem -> bo::reserve.
1783          */
1784         ret = mutex_lock_interruptible(&bo->wu_mutex);
1785         if (unlikely(ret != 0))
1786                 return -ERESTARTSYS;
1787         if (!ww_mutex_is_locked(&bo->resv->lock))
1788                 goto out_unlock;
1789         ret = reservation_object_lock_interruptible(bo->resv, NULL);
1790         if (ret == -EINTR)
1791                 ret = -ERESTARTSYS;
1792         if (unlikely(ret != 0))
1793                 goto out_unlock;
1794         reservation_object_unlock(bo->resv);
1795
1796 out_unlock:
1797         mutex_unlock(&bo->wu_mutex);
1798         return ret;
1799 }