drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include <drm/ttm/ttm_tt.h>
   7
   8 #include "i915_deps.h"
   9 #include "i915_drv.h"
  10 #include "intel_memory_region.h"
  11 #include "intel_region_ttm.h"
  12
  13 #include "gem/i915_gem_object.h"
  14 #include "gem/i915_gem_region.h"
  15 #include "gem/i915_gem_ttm.h"
  16 #include "gem/i915_gem_ttm_move.h"
  17
  18 #include "gt/intel_engine_pm.h"
  19 #include "gt/intel_gt.h"
  20 #include "gt/intel_migrate.h"
  21
  22 /**
  23  * DOC: Selftest failure modes for failsafe migration:
  24  *
  25  * For fail_gpu_migration, the gpu blit scheduled is always a clear blit
  26  * rather than a copy blit, and then we force the failure paths as if
  27  * the blit fence returned an error.
  28  *
  29  * For fail_work_allocation we fail the kmalloc of the async worker, we
  30  * sync the gpu blit. If it then fails, or fail_gpu_migration is set to
  31  * true, then a memcpy operation is performed sync.
  32  */
  33 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
  34 static bool fail_gpu_migration;
  35 static bool fail_work_allocation;
  36 static bool ban_memcpy;
  37
  38 void i915_ttm_migrate_set_failure_modes(bool gpu_migration,
  39                                         bool work_allocation)
  40 {
  41         fail_gpu_migration = gpu_migration;
  42         fail_work_allocation = work_allocation;
  43 }
  44
  45 void i915_ttm_migrate_set_ban_memcpy(bool ban)
  46 {
  47         ban_memcpy = ban;
  48 }
  49 #endif
  50
  51 static enum i915_cache_level
  52 i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res,
  53                      struct ttm_tt *ttm)
  54 {
  55         return ((HAS_LLC(i915) || HAS_SNOOP(i915)) &&
  56                 !i915_ttm_gtt_binds_lmem(res) &&
  57                 ttm->caching == ttm_cached) ? I915_CACHE_LLC :
  58                 I915_CACHE_NONE;
  59 }
  60
  61 static struct intel_memory_region *
  62 i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type)
  63 {
  64         struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev);
  65
  66         /* There's some room for optimization here... */
  67         GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM &&
  68                    ttm_mem_type < I915_PL_LMEM0);
  69         if (ttm_mem_type == I915_PL_SYSTEM)
  70                 return intel_memory_region_lookup(i915, INTEL_MEMORY_SYSTEM,
  71                                                   0);
  72
  73         return intel_memory_region_lookup(i915, INTEL_MEMORY_LOCAL,
  74                                           ttm_mem_type - I915_PL_LMEM0);
  75 }
  76
  77 /**
  78  * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a
  79  * TTM move
  80  * @obj: The gem object
  81  */
  82 void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj)
  83 {
  84         struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
  85
  86         if (i915_ttm_cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) {
  87                 obj->write_domain = I915_GEM_DOMAIN_WC;
  88                 obj->read_domains = I915_GEM_DOMAIN_WC;
  89         } else {
  90                 obj->write_domain = I915_GEM_DOMAIN_CPU;
  91                 obj->read_domains = I915_GEM_DOMAIN_CPU;
  92         }
  93 }
  94
  95 /**
  96  * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move
  97  * @obj: The gem object
  98  *
  99  * Adjusts the GEM object's region, mem_flags and cache coherency after a
 100  * TTM move.
 101  */
 102 void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj)
 103 {
 104         struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
 105         unsigned int cache_level;
 106         unsigned int mem_flags;
 107         unsigned int i;
 108         int mem_type;
 109
 110         /*
 111          * We might have been purged (or swapped out) if the resource is NULL,
 112          * in which case the SYSTEM placement is the closest match to describe
 113          * the current domain. If the object is ever used in this state then we
 114          * will require moving it again.
 115          */
 116         if (!bo->resource) {
 117                 mem_flags = I915_BO_FLAG_STRUCT_PAGE;
 118                 mem_type = I915_PL_SYSTEM;
 119                 cache_level = I915_CACHE_NONE;
 120         } else {
 121                 mem_flags = i915_ttm_cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM :
 122                         I915_BO_FLAG_STRUCT_PAGE;
 123                 mem_type = bo->resource->mem_type;
 124                 cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource,
 125                                                    bo->ttm);
 126         }
 127
 128         /*
 129          * If object was moved to an allowable region, update the object
 130          * region to consider it migrated. Note that if it's currently not
 131          * in an allowable region, it's evicted and we don't update the
 132          * object region.
 133          */
 134         if (intel_region_to_ttm_type(obj->mm.region) != mem_type) {
 135                 for (i = 0; i < obj->mm.n_placements; ++i) {
 136                         struct intel_memory_region *mr = obj->mm.placements[i];
 137
 138                         if (intel_region_to_ttm_type(mr) == mem_type &&
 139                             mr != obj->mm.region) {
 140                                 i915_gem_object_release_memory_region(obj);
 141                                 i915_gem_object_init_memory_region(obj, mr);
 142                                 break;
 143                         }
 144                 }
 145         }
 146
 147         obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM);
 148         obj->mem_flags |= mem_flags;
 149
 150         i915_gem_object_set_cache_coherency(obj, cache_level);
 151 }
 152
 153 /**
 154  * i915_ttm_move_notify - Prepare an object for move
 155  * @bo: The ttm buffer object.
 156  *
 157  * This function prepares an object for move by removing all GPU bindings,
 158  * removing all CPU mapings and finally releasing the pages sg-table.
 159  *
 160  * Return: 0 if successful, negative error code on error.
 161  */
 162 int i915_ttm_move_notify(struct ttm_buffer_object *bo)
 163 {
 164         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 165         int ret;
 166
 167         /*
 168          * Note: The async unbinding here will actually transform the
 169          * blocking wait for unbind into a wait before finally submitting
 170          * evict / migration blit and thus stall the migration timeline
 171          * which may not be good for overall throughput. We should make
 172          * sure we await the unbind fences *after* the migration blit
 173          * instead of *before* as we currently do.
 174          */
 175         ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE |
 176                                      I915_GEM_OBJECT_UNBIND_ASYNC);
 177         if (ret)
 178                 return ret;
 179
 180         ret = __i915_gem_object_put_pages(obj);
 181         if (ret)
 182                 return ret;
 183
 184         return 0;
 185 }
 186
 187 static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
 188                                              bool clear,
 189                                              struct ttm_resource *dst_mem,
 190                                              struct ttm_tt *dst_ttm,
 191                                              struct sg_table *dst_st,
 192                                              const struct i915_deps *deps)
 193 {
 194         struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915),
 195                                                      bdev);
 196         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 197         struct i915_request *rq;
 198         struct ttm_tt *src_ttm = bo->ttm;
 199         enum i915_cache_level src_level, dst_level;
 200         int ret;
 201
 202         if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(to_gt(i915)))
 203                 return ERR_PTR(-EINVAL);
 204
 205         /* With fail_gpu_migration, we always perform a GPU clear. */
 206         if (I915_SELFTEST_ONLY(fail_gpu_migration))
 207                 clear = true;
 208
 209         dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm);
 210         if (clear) {
 211                 if (bo->type == ttm_bo_type_kernel &&
 212                     !I915_SELFTEST_ONLY(fail_gpu_migration))
 213                         return ERR_PTR(-EINVAL);
 214
 215                 intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
 216                 ret = intel_context_migrate_clear(to_gt(i915)->migrate.context, deps,
 217                                                   dst_st->sgl, dst_level,
 218                                                   i915_ttm_gtt_binds_lmem(dst_mem),
 219                                                   0, &rq);
 220         } else {
 221                 struct i915_refct_sgt *src_rsgt =
 222                         i915_ttm_resource_get_st(obj, bo->resource);
 223
 224                 if (IS_ERR(src_rsgt))
 225                         return ERR_CAST(src_rsgt);
 226
 227                 src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm);
 228                 intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
 229                 ret = intel_context_migrate_copy(to_gt(i915)->migrate.context,
 230                                                  deps, src_rsgt->table.sgl,
 231                                                  src_level,
 232                                                  i915_ttm_gtt_binds_lmem(bo->resource),
 233                                                  dst_st->sgl, dst_level,
 234                                                  i915_ttm_gtt_binds_lmem(dst_mem),
 235                                                  &rq);
 236
 237                 i915_refct_sgt_put(src_rsgt);
 238         }
 239
 240         intel_engine_pm_put(to_gt(i915)->migrate.context->engine);
 241
 242         if (ret && rq) {
 243                 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
 244                 i915_request_put(rq);
 245         }
 246
 247         return ret ? ERR_PTR(ret) : &rq->fence;
 248 }
 249
 250 /**
 251  * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality.
 252  * @_dst_iter: Storage space for the destination kmap iterator.
 253  * @_src_iter: Storage space for the source kmap iterator.
 254  * @dst_iter: Pointer to the destination kmap iterator.
 255  * @src_iter: Pointer to the source kmap iterator.
 256  * @clear: Whether to clear instead of copy.
 257  * @src_rsgt: Refcounted scatter-gather list of source memory.
 258  * @dst_rsgt: Refcounted scatter-gather list of destination memory.
 259  */
 260 struct i915_ttm_memcpy_arg {
 261         union {
 262                 struct ttm_kmap_iter_tt tt;
 263                 struct ttm_kmap_iter_iomap io;
 264         } _dst_iter,
 265         _src_iter;
 266         struct ttm_kmap_iter *dst_iter;
 267         struct ttm_kmap_iter *src_iter;
 268         unsigned long num_pages;
 269         bool clear;
 270         struct i915_refct_sgt *src_rsgt;
 271         struct i915_refct_sgt *dst_rsgt;
 272 };
 273
 274 /**
 275  * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence.
 276  * @fence: The dma-fence.
 277  * @work: The work struct use for the memcpy work.
 278  * @lock: The fence lock. Not used to protect anything else ATM.
 279  * @irq_work: Low latency worker to signal the fence since it can't be done
 280  * from the callback for lockdep reasons.
 281  * @cb: Callback for the accelerated migration fence.
 282  * @arg: The argument for the memcpy functionality.
 283  * @i915: The i915 pointer.
 284  * @obj: The GEM object.
 285  * @memcpy_allowed: Instead of processing the @arg, and falling back to memcpy
 286  * or memset, we wedge the device and set the @obj unknown_state, to prevent
 287  * further access to the object with the CPU or GPU.  On some devices we might
 288  * only be permitted to use the blitter engine for such operations.
 289  */
 290 struct i915_ttm_memcpy_work {
 291         struct dma_fence fence;
 292         struct work_struct work;
 293         spinlock_t lock;
 294         struct irq_work irq_work;
 295         struct dma_fence_cb cb;
 296         struct i915_ttm_memcpy_arg arg;
 297         struct drm_i915_private *i915;
 298         struct drm_i915_gem_object *obj;
 299         bool memcpy_allowed;
 300 };
 301
 302 static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg)
 303 {
 304         ttm_move_memcpy(arg->clear, arg->num_pages,
 305                         arg->dst_iter, arg->src_iter);
 306 }
 307
 308 static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg,
 309                                  struct ttm_buffer_object *bo, bool clear,
 310                                  struct ttm_resource *dst_mem,
 311                                  struct ttm_tt *dst_ttm,
 312                                  struct i915_refct_sgt *dst_rsgt)
 313 {
 314         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 315         struct intel_memory_region *dst_reg, *src_reg;
 316
 317         dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type);
 318         src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type);
 319         GEM_BUG_ON(!dst_reg || !src_reg);
 320
 321         arg->dst_iter = !i915_ttm_cpu_maps_iomem(dst_mem) ?
 322                 ttm_kmap_iter_tt_init(&arg->_dst_iter.tt, dst_ttm) :
 323                 ttm_kmap_iter_iomap_init(&arg->_dst_iter.io, &dst_reg->iomap,
 324                                          &dst_rsgt->table, dst_reg->region.start);
 325
 326         arg->src_iter = !i915_ttm_cpu_maps_iomem(bo->resource) ?
 327                 ttm_kmap_iter_tt_init(&arg->_src_iter.tt, bo->ttm) :
 328                 ttm_kmap_iter_iomap_init(&arg->_src_iter.io, &src_reg->iomap,
 329                                          &obj->ttm.cached_io_rsgt->table,
 330                                          src_reg->region.start);
 331         arg->clear = clear;
 332         arg->num_pages = bo->base.size >> PAGE_SHIFT;
 333
 334         arg->dst_rsgt = i915_refct_sgt_get(dst_rsgt);
 335         arg->src_rsgt = clear ? NULL :
 336                 i915_ttm_resource_get_st(obj, bo->resource);
 337 }
 338
 339 static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg)
 340 {
 341         i915_refct_sgt_put(arg->src_rsgt);
 342         i915_refct_sgt_put(arg->dst_rsgt);
 343 }
 344
 345 static void __memcpy_work(struct work_struct *work)
 346 {
 347         struct i915_ttm_memcpy_work *copy_work =
 348                 container_of(work, typeof(*copy_work), work);
 349         struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
 350         bool cookie;
 351
 352         /*
 353          * FIXME: We need to take a closer look here. We should be able to plonk
 354          * this into the fence critical section.
 355          */
 356         if (!copy_work->memcpy_allowed) {
 357                 struct intel_gt *gt;
 358                 unsigned int id;
 359
 360                 for_each_gt(gt, copy_work->i915, id)
 361                         intel_gt_set_wedged(gt);
 362         }
 363
 364         cookie = dma_fence_begin_signalling();
 365
 366         if (copy_work->memcpy_allowed) {
 367                 i915_ttm_move_memcpy(arg);
 368         } else {
 369                 /*
 370                  * Prevent further use of the object. Any future GTT binding or
 371                  * CPU access is not allowed once we signal the fence. Outside
 372                  * of the fence critical section, we then also then wedge the gpu
 373                  * to indicate the device is not functional.
 374                  *
 375                  * The below dma_fence_signal() is our write-memory-barrier.
 376                  */
 377                 copy_work->obj->mm.unknown_state = true;
 378         }
 379
 380         dma_fence_end_signalling(cookie);
 381
 382         dma_fence_signal(&copy_work->fence);
 383
 384         i915_ttm_memcpy_release(arg);
 385         i915_gem_object_put(copy_work->obj);
 386         dma_fence_put(&copy_work->fence);
 387 }
 388
 389 static void __memcpy_irq_work(struct irq_work *irq_work)
 390 {
 391         struct i915_ttm_memcpy_work *copy_work =
 392                 container_of(irq_work, typeof(*copy_work), irq_work);
 393         struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
 394
 395         dma_fence_signal(&copy_work->fence);
 396         i915_ttm_memcpy_release(arg);
 397         i915_gem_object_put(copy_work->obj);
 398         dma_fence_put(&copy_work->fence);
 399 }
 400
 401 static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 402 {
 403         struct i915_ttm_memcpy_work *copy_work =
 404                 container_of(cb, typeof(*copy_work), cb);
 405
 406         if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) {
 407                 INIT_WORK(&copy_work->work, __memcpy_work);
 408                 queue_work(system_unbound_wq, &copy_work->work);
 409         } else {
 410                 init_irq_work(&copy_work->irq_work, __memcpy_irq_work);
 411                 irq_work_queue(&copy_work->irq_work);
 412         }
 413 }
 414
 415 static const char *get_driver_name(struct dma_fence *fence)
 416 {
 417         return "i915_ttm_memcpy_work";
 418 }
 419
 420 static const char *get_timeline_name(struct dma_fence *fence)
 421 {
 422         return "unbound";
 423 }
 424
 425 static const struct dma_fence_ops dma_fence_memcpy_ops = {
 426         .get_driver_name = get_driver_name,
 427         .get_timeline_name = get_timeline_name,
 428 };
 429
 430 static struct dma_fence *
 431 i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work,
 432                          struct dma_fence *dep)
 433 {
 434         int ret;
 435
 436         spin_lock_init(&work->lock);
 437         dma_fence_init(&work->fence, &dma_fence_memcpy_ops, &work->lock, 0, 0);
 438         dma_fence_get(&work->fence);
 439         ret = dma_fence_add_callback(dep, &work->cb, __memcpy_cb);
 440         if (ret) {
 441                 if (ret != -ENOENT)
 442                         dma_fence_wait(dep, false);
 443
 444                 return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL :
 445                                dep->error);
 446         }
 447
 448         return &work->fence;
 449 }
 450
 451 static bool i915_ttm_memcpy_allowed(struct ttm_buffer_object *bo,
 452                                     struct ttm_resource *dst_mem)
 453 {
 454         if (i915_gem_object_needs_ccs_pages(i915_ttm_to_gem(bo)))
 455                 return false;
 456
 457         if (!(i915_ttm_resource_mappable(bo->resource) &&
 458               i915_ttm_resource_mappable(dst_mem)))
 459                 return false;
 460
 461         return I915_SELFTEST_ONLY(ban_memcpy) ? false : true;
 462 }
 463
 464 static struct dma_fence *
 465 __i915_ttm_move(struct ttm_buffer_object *bo,
 466                 const struct ttm_operation_ctx *ctx, bool clear,
 467                 struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm,
 468                 struct i915_refct_sgt *dst_rsgt, bool allow_accel,
 469                 const struct i915_deps *move_deps)
 470 {
 471         const bool memcpy_allowed = i915_ttm_memcpy_allowed(bo, dst_mem);
 472         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 473         struct drm_i915_private *i915 = to_i915(bo->base.dev);
 474         struct i915_ttm_memcpy_work *copy_work = NULL;
 475         struct i915_ttm_memcpy_arg _arg, *arg = &_arg;
 476         struct dma_fence *fence = ERR_PTR(-EINVAL);
 477
 478         if (allow_accel) {
 479                 fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm,
 480                                             &dst_rsgt->table, move_deps);
 481
 482                 /*
 483                  * We only need to intercept the error when moving to lmem.
 484                  * When moving to system, TTM or shmem will provide us with
 485                  * cleared pages.
 486                  */
 487                 if (!IS_ERR(fence) && !i915_ttm_gtt_binds_lmem(dst_mem) &&
 488                     !I915_SELFTEST_ONLY(fail_gpu_migration ||
 489                                         fail_work_allocation))
 490                         goto out;
 491         }
 492
 493         /* If we've scheduled gpu migration. Try to arm error intercept. */
 494         if (!IS_ERR(fence)) {
 495                 struct dma_fence *dep = fence;
 496
 497                 if (!I915_SELFTEST_ONLY(fail_work_allocation))
 498                         copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL);
 499
 500                 if (copy_work) {
 501                         copy_work->i915 = i915;
 502                         copy_work->memcpy_allowed = memcpy_allowed;
 503                         copy_work->obj = i915_gem_object_get(obj);
 504                         arg = &copy_work->arg;
 505                         if (memcpy_allowed)
 506                                 i915_ttm_memcpy_init(arg, bo, clear, dst_mem,
 507                                                      dst_ttm, dst_rsgt);
 508
 509                         fence = i915_ttm_memcpy_work_arm(copy_work, dep);
 510                 } else {
 511                         dma_fence_wait(dep, false);
 512                         fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ?
 513                                         -EINVAL : fence->error);
 514                 }
 515                 dma_fence_put(dep);
 516
 517                 if (!IS_ERR(fence))
 518                         goto out;
 519         } else {
 520                 int err = PTR_ERR(fence);
 521
 522                 if (err == -EINTR || err == -ERESTARTSYS || err == -EAGAIN)
 523                         return fence;
 524
 525                 if (move_deps) {
 526                         err = i915_deps_sync(move_deps, ctx);
 527                         if (err)
 528                                 return ERR_PTR(err);
 529                 }
 530         }
 531
 532         /* Error intercept failed or no accelerated migration to start with */
 533
 534         if (memcpy_allowed) {
 535                 if (!copy_work)
 536                         i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm,
 537                                              dst_rsgt);
 538                 i915_ttm_move_memcpy(arg);
 539                 i915_ttm_memcpy_release(arg);
 540         }
 541         if (copy_work)
 542                 i915_gem_object_put(copy_work->obj);
 543         kfree(copy_work);
 544
 545         return memcpy_allowed ? NULL : ERR_PTR(-EIO);
 546 out:
 547         if (!fence && copy_work) {
 548                 i915_ttm_memcpy_release(arg);
 549                 i915_gem_object_put(copy_work->obj);
 550                 kfree(copy_work);
 551         }
 552
 553         return fence;
 554 }
 555
 556 /**
 557  * i915_ttm_move - The TTM move callback used by i915.
 558  * @bo: The buffer object.
 559  * @evict: Whether this is an eviction.
 560  * @dst_mem: The destination ttm resource.
 561  * @hop: If we need multihop, what temporary memory type to move to.
 562  *
 563  * Return: 0 if successful, negative error code otherwise.
 564  */
 565 int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,
 566                   struct ttm_operation_ctx *ctx,
 567                   struct ttm_resource *dst_mem,
 568                   struct ttm_place *hop)
 569 {
 570         struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
 571         struct ttm_resource_manager *dst_man =
 572                 ttm_manager_type(bo->bdev, dst_mem->mem_type);
 573         struct dma_fence *migration_fence = NULL;
 574         struct ttm_tt *ttm = bo->ttm;
 575         struct i915_refct_sgt *dst_rsgt;
 576         bool clear;
 577         int ret;
 578
 579         if (GEM_WARN_ON(i915_ttm_is_ghost_object(bo))) {
 580                 ttm_bo_move_null(bo, dst_mem);
 581                 return 0;
 582         }
 583
 584         if (!bo->resource) {
 585                 if (dst_mem->mem_type != TTM_PL_SYSTEM) {
 586                         hop->mem_type = TTM_PL_SYSTEM;
 587                         hop->flags = TTM_PL_FLAG_TEMPORARY;
 588                         return -EMULTIHOP;
 589                 }
 590
 591                 /*
 592                  * This is only reached when first creating the object, or if
 593                  * the object was purged or swapped out (pipeline-gutting). For
 594                  * the former we can safely skip all of the below since we are
 595                  * only using a dummy SYSTEM placement here. And with the latter
 596                  * we will always re-enter here with bo->resource set correctly
 597                  * (as per the above), since this is part of a multi-hop
 598                  * sequence, where at the end we can do the move for real.
 599                  *
 600                  * The special case here is when the dst_mem is TTM_PL_SYSTEM,
 601                  * which doens't require any kind of move, so it should be safe
 602                  * to skip all the below and call ttm_bo_move_null() here, where
 603                  * the caller in __i915_ttm_get_pages() will take care of the
 604                  * rest, since we should have a valid ttm_tt.
 605                  */
 606                 ttm_bo_move_null(bo, dst_mem);
 607                 return 0;
 608         }
 609
 610         ret = i915_ttm_move_notify(bo);
 611         if (ret)
 612                 return ret;
 613
 614         if (obj->mm.madv != I915_MADV_WILLNEED) {
 615                 i915_ttm_purge(obj);
 616                 ttm_resource_free(bo, &dst_mem);
 617                 return 0;
 618         }
 619
 620         /* Populate ttm with pages if needed. Typically system memory. */
 621         if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) {
 622                 ret = ttm_tt_populate(bo->bdev, ttm, ctx);
 623                 if (ret)
 624                         return ret;
 625         }
 626
 627         dst_rsgt = i915_ttm_resource_get_st(obj, dst_mem);
 628         if (IS_ERR(dst_rsgt))
 629                 return PTR_ERR(dst_rsgt);
 630
 631         clear = !i915_ttm_cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm));
 632         if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) {
 633                 struct i915_deps deps;
 634
 635                 i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
 636                 ret = i915_deps_add_resv(&deps, bo->base.resv, ctx);
 637                 if (ret) {
 638                         i915_refct_sgt_put(dst_rsgt);
 639                         return ret;
 640                 }
 641
 642                 migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, ttm,
 643                                                   dst_rsgt, true, &deps);
 644                 i915_deps_fini(&deps);
 645         }
 646
 647         /* We can possibly get an -ERESTARTSYS here */
 648         if (IS_ERR(migration_fence)) {
 649                 i915_refct_sgt_put(dst_rsgt);
 650                 return PTR_ERR(migration_fence);
 651         }
 652
 653         if (migration_fence) {
 654                 if (I915_SELFTEST_ONLY(evict && fail_gpu_migration))
 655                         ret = -EIO; /* never feed non-migrate fences into ttm */
 656                 else
 657                         ret = ttm_bo_move_accel_cleanup(bo, migration_fence, evict,
 658                                                         true, dst_mem);
 659                 if (ret) {
 660                         dma_fence_wait(migration_fence, false);
 661                         ttm_bo_move_sync_cleanup(bo, dst_mem);
 662                 }
 663                 dma_fence_put(migration_fence);
 664         } else {
 665                 ttm_bo_move_sync_cleanup(bo, dst_mem);
 666         }
 667
 668         i915_ttm_adjust_domains_after_move(obj);
 669         i915_ttm_free_cached_io_rsgt(obj);
 670
 671         if (i915_ttm_gtt_binds_lmem(dst_mem) || i915_ttm_cpu_maps_iomem(dst_mem)) {
 672                 obj->ttm.cached_io_rsgt = dst_rsgt;
 673                 obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl;
 674                 obj->ttm.get_io_page.sg_idx = 0;
 675         } else {
 676                 i915_refct_sgt_put(dst_rsgt);
 677         }
 678
 679         i915_ttm_adjust_lru(obj);
 680         i915_ttm_adjust_gem_after_move(obj);
 681         return 0;
 682 }
 683
 684 /**
 685  * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to
 686  * another
 687  * @dst: The destination object
 688  * @src: The source object
 689  * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used.
 690  * @intr: Whether to perform waits interruptible:
 691  *
 692  * Note: The caller is responsible for assuring that the underlying
 693  * TTM objects are populated if needed and locked.
 694  *
 695  * Return: Zero on success. Negative error code on error. If @intr == true,
 696  * then it may return -ERESTARTSYS or -EINTR.
 697  */
 698 int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
 699                           struct drm_i915_gem_object *src,
 700                           bool allow_accel, bool intr)
 701 {
 702         struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst);
 703         struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src);
 704         struct ttm_operation_ctx ctx = {
 705                 .interruptible = intr,
 706         };
 707         struct i915_refct_sgt *dst_rsgt;
 708         struct dma_fence *copy_fence;
 709         struct i915_deps deps;
 710         int ret;
 711
 712         assert_object_held(dst);
 713         assert_object_held(src);
 714         i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
 715
 716         ret = dma_resv_reserve_fences(src_bo->base.resv, 1);
 717         if (ret)
 718                 return ret;
 719
 720         ret = dma_resv_reserve_fences(dst_bo->base.resv, 1);
 721         if (ret)
 722                 return ret;
 723
 724         ret = i915_deps_add_resv(&deps, dst_bo->base.resv, &ctx);
 725         if (ret)
 726                 return ret;
 727
 728         ret = i915_deps_add_resv(&deps, src_bo->base.resv, &ctx);
 729         if (ret)
 730                 return ret;
 731
 732         dst_rsgt = i915_ttm_resource_get_st(dst, dst_bo->resource);
 733         copy_fence = __i915_ttm_move(src_bo, &ctx, false, dst_bo->resource,
 734                                      dst_bo->ttm, dst_rsgt, allow_accel,
 735                                      &deps);
 736
 737         i915_deps_fini(&deps);
 738         i915_refct_sgt_put(dst_rsgt);
 739         if (IS_ERR_OR_NULL(copy_fence))
 740                 return PTR_ERR_OR_ZERO(copy_fence);
 741
 742         dma_resv_add_fence(dst_bo->base.resv, copy_fence, DMA_RESV_USAGE_WRITE);
 743         dma_resv_add_fence(src_bo->base.resv, copy_fence, DMA_RESV_USAGE_READ);
 744         dma_fence_put(copy_fence);
 745
 746         return 0;
 747 }