drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include <drm/ttm/ttm_bo_driver.h>
   7
   8 #include "i915_deps.h"
   9 #include "i915_drv.h"
  10 #include "intel_memory_region.h"
  11 #include "intel_region_ttm.h"
  12
  13 #include "gem/i915_gem_object.h"
  14 #include "gem/i915_gem_region.h"
  15 #include "gem/i915_gem_ttm.h"
  16 #include "gem/i915_gem_ttm_move.h"
  17
  18 #include "gt/intel_engine_pm.h"
  19 #include "gt/intel_gt.h"
  20 #include "gt/intel_migrate.h"
  21
  22 /**
  23  * DOC: Selftest failure modes for failsafe migration:
  24  *
  25  * For fail_gpu_migration, the gpu blit scheduled is always a clear blit
  26  * rather than a copy blit, and then we force the failure paths as if
  27  * the blit fence returned an error.
  28  *
  29  * For fail_work_allocation we fail the kmalloc of the async worker, we
  30  * sync the gpu blit. If it then fails, or fail_gpu_migration is set to
  31  * true, then a memcpy operation is performed sync.
  32  */
  33 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
  34 static bool fail_gpu_migration;
  35 static bool fail_work_allocation;
  36
  37 void i915_ttm_migrate_set_failure_modes(bool gpu_migration,
  38                                         bool work_allocation)
  39 {
  40         fail_gpu_migration = gpu_migration;
  41         fail_work_allocation = work_allocation;
  42 }
  43 #endif
  44
  45 static enum i915_cache_level
  46 i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res,
  47                      struct ttm_tt *ttm)
  48 {
  49         return ((HAS_LLC(i915) || HAS_SNOOP(i915)) &&
  50                 !i915_ttm_gtt_binds_lmem(res) &&
  51                 ttm->caching == ttm_cached) ? I915_CACHE_LLC :
  52                 I915_CACHE_NONE;
  53 }
  54
  55 static struct intel_memory_region *
  56 i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type)
  57 {
  58         struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev);
  59
  60         /* There's some room for optimization here... */
  61         GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM &&
  62                    ttm_mem_type < I915_PL_LMEM0);
  63         if (ttm_mem_type == I915_PL_SYSTEM)
  64                 return intel_memory_region_lookup(i915, INTEL_MEMORY_SYSTEM,
  65                                                   0);
  66
  67         return intel_memory_region_lookup(i915, INTEL_MEMORY_LOCAL,
  68                                           ttm_mem_type - I915_PL_LMEM0);
  69 }
  70
  71 /**
  72  * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a
  73  * TTM move
  74  * @obj: The gem object
  75  */
  76 void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj)
  77 {
  78         struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
  79
  80         if (i915_ttm_cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) {
  81                 obj->write_domain = I915_GEM_DOMAIN_WC;
  82                 obj->read_domains = I915_GEM_DOMAIN_WC;
  83         } else {
  84                 obj->write_domain = I915_GEM_DOMAIN_CPU;
  85                 obj->read_domains = I915_GEM_DOMAIN_CPU;
  86         }
  87 }
  88
  89 /**
  90  * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move
  91  * @obj: The gem object
  92  *
  93  * Adjusts the GEM object's region, mem_flags and cache coherency after a
  94  * TTM move.
  95  */
  96 void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj)
  97 {
  98         struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
  99         unsigned int cache_level;
 100         unsigned int i;
 101
 102         /*
 103          * If object was moved to an allowable region, update the object
 104          * region to consider it migrated. Note that if it's currently not
 105          * in an allowable region, it's evicted and we don't update the
 106          * object region.
 107          */
 108         if (intel_region_to_ttm_type(obj->mm.region) != bo->resource->mem_type) {
 109                 for (i = 0; i < obj->mm.n_placements; ++i) {
 110                         struct intel_memory_region *mr = obj->mm.placements[i];
 111
 112                         if (intel_region_to_ttm_type(mr) == bo->resource->mem_type &&
 113                             mr != obj->mm.region) {
 114                                 i915_gem_object_release_memory_region(obj);
 115                                 i915_gem_object_init_memory_region(obj, mr);
 116                                 break;
 117                         }
 118                 }
 119         }
 120
 121         obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM);
 122
 123         obj->mem_flags |= i915_ttm_cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM :
 124                 I915_BO_FLAG_STRUCT_PAGE;
 125
 126         cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource,
 127                                            bo->ttm);
 128         i915_gem_object_set_cache_coherency(obj, cache_level);
 129 }
 130
 131 /**
 132  * i915_ttm_move_notify - Prepare an object for move
 133  * @bo: The ttm buffer object.
 134  *
 135  * This function prepares an object for move by removing all GPU bindings,
 136  * removing all CPU mapings and finally releasing the pages sg-table.
 137  *
 138  * Return: 0 if successful, negative error code on error.
 139  */
 140 int i915_ttm_move_notify(struct ttm_buffer_object *bo)
 141 {
 142         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 143         int ret;
 144
 145         ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
 146         if (ret)
 147                 return ret;
 148
 149         ret = __i915_gem_object_put_pages(obj);
 150         if (ret)
 151                 return ret;
 152
 153         return 0;
 154 }
 155
 156 static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
 157                                              bool clear,
 158                                              struct ttm_resource *dst_mem,
 159                                              struct ttm_tt *dst_ttm,
 160                                              struct sg_table *dst_st,
 161                                              const struct i915_deps *deps)
 162 {
 163         struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915),
 164                                                      bdev);
 165         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 166         struct i915_request *rq;
 167         struct ttm_tt *src_ttm = bo->ttm;
 168         enum i915_cache_level src_level, dst_level;
 169         int ret;
 170
 171         if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(to_gt(i915)))
 172                 return ERR_PTR(-EINVAL);
 173
 174         /* With fail_gpu_migration, we always perform a GPU clear. */
 175         if (I915_SELFTEST_ONLY(fail_gpu_migration))
 176                 clear = true;
 177
 178         dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm);
 179         if (clear) {
 180                 if (bo->type == ttm_bo_type_kernel &&
 181                     !I915_SELFTEST_ONLY(fail_gpu_migration))
 182                         return ERR_PTR(-EINVAL);
 183
 184                 intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
 185                 ret = intel_context_migrate_clear(to_gt(i915)->migrate.context, deps,
 186                                                   dst_st->sgl, dst_level,
 187                                                   i915_ttm_gtt_binds_lmem(dst_mem),
 188                                                   0, &rq);
 189         } else {
 190                 struct i915_refct_sgt *src_rsgt =
 191                         i915_ttm_resource_get_st(obj, bo->resource);
 192
 193                 if (IS_ERR(src_rsgt))
 194                         return ERR_CAST(src_rsgt);
 195
 196                 src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm);
 197                 intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
 198                 ret = intel_context_migrate_copy(to_gt(i915)->migrate.context,
 199                                                  deps, src_rsgt->table.sgl,
 200                                                  src_level,
 201                                                  i915_ttm_gtt_binds_lmem(bo->resource),
 202                                                  dst_st->sgl, dst_level,
 203                                                  i915_ttm_gtt_binds_lmem(dst_mem),
 204                                                  &rq);
 205
 206                 i915_refct_sgt_put(src_rsgt);
 207         }
 208
 209         intel_engine_pm_put(to_gt(i915)->migrate.context->engine);
 210
 211         if (ret && rq) {
 212                 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
 213                 i915_request_put(rq);
 214         }
 215
 216         return ret ? ERR_PTR(ret) : &rq->fence;
 217 }
 218
 219 /**
 220  * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality.
 221  * @_dst_iter: Storage space for the destination kmap iterator.
 222  * @_src_iter: Storage space for the source kmap iterator.
 223  * @dst_iter: Pointer to the destination kmap iterator.
 224  * @src_iter: Pointer to the source kmap iterator.
 225  * @clear: Whether to clear instead of copy.
 226  * @src_rsgt: Refcounted scatter-gather list of source memory.
 227  * @dst_rsgt: Refcounted scatter-gather list of destination memory.
 228  */
 229 struct i915_ttm_memcpy_arg {
 230         union {
 231                 struct ttm_kmap_iter_tt tt;
 232                 struct ttm_kmap_iter_iomap io;
 233         } _dst_iter,
 234         _src_iter;
 235         struct ttm_kmap_iter *dst_iter;
 236         struct ttm_kmap_iter *src_iter;
 237         unsigned long num_pages;
 238         bool clear;
 239         struct i915_refct_sgt *src_rsgt;
 240         struct i915_refct_sgt *dst_rsgt;
 241 };
 242
 243 /**
 244  * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence.
 245  * @fence: The dma-fence.
 246  * @work: The work struct use for the memcpy work.
 247  * @lock: The fence lock. Not used to protect anything else ATM.
 248  * @irq_work: Low latency worker to signal the fence since it can't be done
 249  * from the callback for lockdep reasons.
 250  * @cb: Callback for the accelerated migration fence.
 251  * @arg: The argument for the memcpy functionality.
 252  */
 253 struct i915_ttm_memcpy_work {
 254         struct dma_fence fence;
 255         struct work_struct work;
 256         /* The fence lock */
 257         spinlock_t lock;
 258         struct irq_work irq_work;
 259         struct dma_fence_cb cb;
 260         struct i915_ttm_memcpy_arg arg;
 261 };
 262
 263 static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg)
 264 {
 265         ttm_move_memcpy(arg->clear, arg->num_pages,
 266                         arg->dst_iter, arg->src_iter);
 267 }
 268
 269 static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg,
 270                                  struct ttm_buffer_object *bo, bool clear,
 271                                  struct ttm_resource *dst_mem,
 272                                  struct ttm_tt *dst_ttm,
 273                                  struct i915_refct_sgt *dst_rsgt)
 274 {
 275         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 276         struct intel_memory_region *dst_reg, *src_reg;
 277
 278         dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type);
 279         src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type);
 280         GEM_BUG_ON(!dst_reg || !src_reg);
 281
 282         arg->dst_iter = !i915_ttm_cpu_maps_iomem(dst_mem) ?
 283                 ttm_kmap_iter_tt_init(&arg->_dst_iter.tt, dst_ttm) :
 284                 ttm_kmap_iter_iomap_init(&arg->_dst_iter.io, &dst_reg->iomap,
 285                                          &dst_rsgt->table, dst_reg->region.start);
 286
 287         arg->src_iter = !i915_ttm_cpu_maps_iomem(bo->resource) ?
 288                 ttm_kmap_iter_tt_init(&arg->_src_iter.tt, bo->ttm) :
 289                 ttm_kmap_iter_iomap_init(&arg->_src_iter.io, &src_reg->iomap,
 290                                          &obj->ttm.cached_io_rsgt->table,
 291                                          src_reg->region.start);
 292         arg->clear = clear;
 293         arg->num_pages = bo->base.size >> PAGE_SHIFT;
 294
 295         arg->dst_rsgt = i915_refct_sgt_get(dst_rsgt);
 296         arg->src_rsgt = clear ? NULL :
 297                 i915_ttm_resource_get_st(obj, bo->resource);
 298 }
 299
 300 static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg)
 301 {
 302         i915_refct_sgt_put(arg->src_rsgt);
 303         i915_refct_sgt_put(arg->dst_rsgt);
 304 }
 305
 306 static void __memcpy_work(struct work_struct *work)
 307 {
 308         struct i915_ttm_memcpy_work *copy_work =
 309                 container_of(work, typeof(*copy_work), work);
 310         struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
 311         bool cookie = dma_fence_begin_signalling();
 312
 313         i915_ttm_move_memcpy(arg);
 314         dma_fence_end_signalling(cookie);
 315
 316         dma_fence_signal(&copy_work->fence);
 317
 318         i915_ttm_memcpy_release(arg);
 319         dma_fence_put(&copy_work->fence);
 320 }
 321
 322 static void __memcpy_irq_work(struct irq_work *irq_work)
 323 {
 324         struct i915_ttm_memcpy_work *copy_work =
 325                 container_of(irq_work, typeof(*copy_work), irq_work);
 326         struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
 327
 328         dma_fence_signal(&copy_work->fence);
 329         i915_ttm_memcpy_release(arg);
 330         dma_fence_put(&copy_work->fence);
 331 }
 332
 333 static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 334 {
 335         struct i915_ttm_memcpy_work *copy_work =
 336                 container_of(cb, typeof(*copy_work), cb);
 337
 338         if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) {
 339                 INIT_WORK(&copy_work->work, __memcpy_work);
 340                 queue_work(system_unbound_wq, &copy_work->work);
 341         } else {
 342                 init_irq_work(&copy_work->irq_work, __memcpy_irq_work);
 343                 irq_work_queue(&copy_work->irq_work);
 344         }
 345 }
 346
 347 static const char *get_driver_name(struct dma_fence *fence)
 348 {
 349         return "i915_ttm_memcpy_work";
 350 }
 351
 352 static const char *get_timeline_name(struct dma_fence *fence)
 353 {
 354         return "unbound";
 355 }
 356
 357 static const struct dma_fence_ops dma_fence_memcpy_ops = {
 358         .get_driver_name = get_driver_name,
 359         .get_timeline_name = get_timeline_name,
 360 };
 361
 362 static struct dma_fence *
 363 i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work,
 364                          struct dma_fence *dep)
 365 {
 366         int ret;
 367
 368         spin_lock_init(&work->lock);
 369         dma_fence_init(&work->fence, &dma_fence_memcpy_ops, &work->lock, 0, 0);
 370         dma_fence_get(&work->fence);
 371         ret = dma_fence_add_callback(dep, &work->cb, __memcpy_cb);
 372         if (ret) {
 373                 if (ret != -ENOENT)
 374                         dma_fence_wait(dep, false);
 375
 376                 return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL :
 377                                dep->error);
 378         }
 379
 380         return &work->fence;
 381 }
 382
 383 static struct dma_fence *
 384 __i915_ttm_move(struct ttm_buffer_object *bo,
 385                 const struct ttm_operation_ctx *ctx, bool clear,
 386                 struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm,
 387                 struct i915_refct_sgt *dst_rsgt, bool allow_accel,
 388                 const struct i915_deps *move_deps)
 389 {
 390         struct i915_ttm_memcpy_work *copy_work = NULL;
 391         struct i915_ttm_memcpy_arg _arg, *arg = &_arg;
 392         struct dma_fence *fence = ERR_PTR(-EINVAL);
 393
 394         if (allow_accel) {
 395                 fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm,
 396                                             &dst_rsgt->table, move_deps);
 397
 398                 /*
 399                  * We only need to intercept the error when moving to lmem.
 400                  * When moving to system, TTM or shmem will provide us with
 401                  * cleared pages.
 402                  */
 403                 if (!IS_ERR(fence) && !i915_ttm_gtt_binds_lmem(dst_mem) &&
 404                     !I915_SELFTEST_ONLY(fail_gpu_migration ||
 405                                         fail_work_allocation))
 406                         goto out;
 407         }
 408
 409         /* If we've scheduled gpu migration. Try to arm error intercept. */
 410         if (!IS_ERR(fence)) {
 411                 struct dma_fence *dep = fence;
 412
 413                 if (!I915_SELFTEST_ONLY(fail_work_allocation))
 414                         copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL);
 415
 416                 if (copy_work) {
 417                         arg = &copy_work->arg;
 418                         i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm,
 419                                              dst_rsgt);
 420                         fence = i915_ttm_memcpy_work_arm(copy_work, dep);
 421                 } else {
 422                         dma_fence_wait(dep, false);
 423                         fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ?
 424                                         -EINVAL : fence->error);
 425                 }
 426                 dma_fence_put(dep);
 427
 428                 if (!IS_ERR(fence))
 429                         goto out;
 430         } else if (move_deps) {
 431                 int err = i915_deps_sync(move_deps, ctx);
 432
 433                 if (err)
 434                         return ERR_PTR(err);
 435         }
 436
 437         /* Error intercept failed or no accelerated migration to start with */
 438         if (!copy_work)
 439                 i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm,
 440                                      dst_rsgt);
 441         i915_ttm_move_memcpy(arg);
 442         i915_ttm_memcpy_release(arg);
 443         kfree(copy_work);
 444
 445         return NULL;
 446 out:
 447         if (!fence && copy_work) {
 448                 i915_ttm_memcpy_release(arg);
 449                 kfree(copy_work);
 450         }
 451
 452         return fence;
 453 }
 454
 455 static int
 456 prev_deps(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
 457           struct i915_deps *deps)
 458 {
 459         int ret;
 460
 461         ret = i915_deps_add_dependency(deps, bo->moving, ctx);
 462         if (!ret)
 463                 ret = i915_deps_add_resv(deps, bo->base.resv, ctx);
 464
 465         return ret;
 466 }
 467
 468 /**
 469  * i915_ttm_move - The TTM move callback used by i915.
 470  * @bo: The buffer object.
 471  * @evict: Whether this is an eviction.
 472  * @dst_mem: The destination ttm resource.
 473  * @hop: If we need multihop, what temporary memory type to move to.
 474  *
 475  * Return: 0 if successful, negative error code otherwise.
 476  */
 477 int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,
 478                   struct ttm_operation_ctx *ctx,
 479                   struct ttm_resource *dst_mem,
 480                   struct ttm_place *hop)
 481 {
 482         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 483         struct ttm_resource_manager *dst_man =
 484                 ttm_manager_type(bo->bdev, dst_mem->mem_type);
 485         struct dma_fence *migration_fence = NULL;
 486         struct ttm_tt *ttm = bo->ttm;
 487         struct i915_refct_sgt *dst_rsgt;
 488         bool clear;
 489         int ret;
 490
 491         if (GEM_WARN_ON(!obj)) {
 492                 ttm_bo_move_null(bo, dst_mem);
 493                 return 0;
 494         }
 495
 496         ret = i915_ttm_move_notify(bo);
 497         if (ret)
 498                 return ret;
 499
 500         if (obj->mm.madv != I915_MADV_WILLNEED) {
 501                 i915_ttm_purge(obj);
 502                 ttm_resource_free(bo, &dst_mem);
 503                 return 0;
 504         }
 505
 506         /* Populate ttm with pages if needed. Typically system memory. */
 507         if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) {
 508                 ret = ttm_tt_populate(bo->bdev, ttm, ctx);
 509                 if (ret)
 510                         return ret;
 511         }
 512
 513         dst_rsgt = i915_ttm_resource_get_st(obj, dst_mem);
 514         if (IS_ERR(dst_rsgt))
 515                 return PTR_ERR(dst_rsgt);
 516
 517         clear = !i915_ttm_cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm));
 518         if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) {
 519                 struct i915_deps deps;
 520
 521                 i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
 522                 ret = prev_deps(bo, ctx, &deps);
 523                 if (ret) {
 524                         i915_refct_sgt_put(dst_rsgt);
 525                         return ret;
 526                 }
 527
 528                 migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, bo->ttm,
 529                                                   dst_rsgt, true, &deps);
 530                 i915_deps_fini(&deps);
 531         }
 532
 533         /* We can possibly get an -ERESTARTSYS here */
 534         if (IS_ERR(migration_fence)) {
 535                 i915_refct_sgt_put(dst_rsgt);
 536                 return PTR_ERR(migration_fence);
 537         }
 538
 539         if (migration_fence) {
 540                 ret = ttm_bo_move_accel_cleanup(bo, migration_fence, evict,
 541                                                 true, dst_mem);
 542                 if (ret) {
 543                         dma_fence_wait(migration_fence, false);
 544                         ttm_bo_move_sync_cleanup(bo, dst_mem);
 545                 }
 546                 dma_fence_put(migration_fence);
 547         } else {
 548                 ttm_bo_move_sync_cleanup(bo, dst_mem);
 549         }
 550
 551         i915_ttm_adjust_domains_after_move(obj);
 552         i915_ttm_free_cached_io_rsgt(obj);
 553
 554         if (i915_ttm_gtt_binds_lmem(dst_mem) || i915_ttm_cpu_maps_iomem(dst_mem)) {
 555                 obj->ttm.cached_io_rsgt = dst_rsgt;
 556                 obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl;
 557                 obj->ttm.get_io_page.sg_idx = 0;
 558         } else {
 559                 i915_refct_sgt_put(dst_rsgt);
 560         }
 561
 562         i915_ttm_adjust_lru(obj);
 563         i915_ttm_adjust_gem_after_move(obj);
 564         return 0;
 565 }
 566
 567 /**
 568  * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to
 569  * another
 570  * @dst: The destination object
 571  * @src: The source object
 572  * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used.
 573  * @intr: Whether to perform waits interruptible:
 574  *
 575  * Note: The caller is responsible for assuring that the underlying
 576  * TTM objects are populated if needed and locked.
 577  *
 578  * Return: Zero on success. Negative error code on error. If @intr == true,
 579  * then it may return -ERESTARTSYS or -EINTR.
 580  */
 581 int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
 582                           struct drm_i915_gem_object *src,
 583                           bool allow_accel, bool intr)
 584 {
 585         struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst);
 586         struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src);
 587         struct ttm_operation_ctx ctx = {
 588                 .interruptible = intr,
 589         };
 590         struct i915_refct_sgt *dst_rsgt;
 591         struct dma_fence *copy_fence;
 592         struct i915_deps deps;
 593         int ret;
 594
 595         assert_object_held(dst);
 596         assert_object_held(src);
 597         i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
 598
 599         ret = dma_resv_reserve_shared(src_bo->base.resv, 1);
 600         if (ret)
 601                 return ret;
 602
 603         ret = i915_deps_add_resv(&deps, dst_bo->base.resv, &ctx);
 604         if (ret)
 605                 return ret;
 606
 607         ret = i915_deps_add_resv(&deps, src_bo->base.resv, &ctx);
 608         if (ret)
 609                 return ret;
 610
 611         dst_rsgt = i915_ttm_resource_get_st(dst, dst_bo->resource);
 612         copy_fence = __i915_ttm_move(src_bo, &ctx, false, dst_bo->resource,
 613                                      dst_bo->ttm, dst_rsgt, allow_accel,
 614                                      &deps);
 615
 616         i915_deps_fini(&deps);
 617         i915_refct_sgt_put(dst_rsgt);
 618         if (IS_ERR_OR_NULL(copy_fence))
 619                 return PTR_ERR_OR_ZERO(copy_fence);
 620
 621         dma_resv_add_excl_fence(dst_bo->base.resv, copy_fence);
 622         dma_resv_add_shared_fence(src_bo->base.resv, copy_fence);
 623
 624         dma_fence_put(copy_fence);
 625
 626         return 0;
 627 }