drivers/gpu/drm/xe/xe_vm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include "xe_vm.h"
   7
   8 #include <linux/dma-fence-array.h>
   9 #include <linux/nospec.h>
  10
  11 #include <drm/drm_exec.h>
  12 #include <drm/drm_print.h>
  13 #include <drm/ttm/ttm_execbuf_util.h>
  14 #include <drm/ttm/ttm_tt.h>
  15 #include <drm/xe_drm.h>
  16 #include <linux/delay.h>
  17 #include <linux/kthread.h>
  18 #include <linux/mm.h>
  19 #include <linux/swap.h>
  20
  21 #include "xe_assert.h"
  22 #include "xe_bo.h"
  23 #include "xe_device.h"
  24 #include "xe_drm_client.h"
  25 #include "xe_exec_queue.h"
  26 #include "xe_gt.h"
  27 #include "xe_gt_pagefault.h"
  28 #include "xe_gt_tlb_invalidation.h"
  29 #include "xe_migrate.h"
  30 #include "xe_pat.h"
  31 #include "xe_pm.h"
  32 #include "xe_preempt_fence.h"
  33 #include "xe_pt.h"
  34 #include "xe_res_cursor.h"
  35 #include "xe_sync.h"
  36 #include "xe_trace.h"
  37 #include "generated/xe_wa_oob.h"
  38 #include "xe_wa.h"
  39
  40 #define TEST_VM_ASYNC_OPS_ERROR
  41
  42 static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm)
  43 {
  44         return vm->gpuvm.r_obj;
  45 }
  46
  47 /**
  48  * xe_vma_userptr_check_repin() - Advisory check for repin needed
  49  * @vma: The userptr vma
  50  *
  51  * Check if the userptr vma has been invalidated since last successful
  52  * repin. The check is advisory only and can the function can be called
  53  * without the vm->userptr.notifier_lock held. There is no guarantee that the
  54  * vma userptr will remain valid after a lockless check, so typically
  55  * the call needs to be followed by a proper check under the notifier_lock.
  56  *
  57  * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended.
  58  */
  59 int xe_vma_userptr_check_repin(struct xe_vma *vma)
  60 {
  61         return mmu_interval_check_retry(&vma->userptr.notifier,
  62                                         vma->userptr.notifier_seq) ?
  63                 -EAGAIN : 0;
  64 }
  65
  66 int xe_vma_userptr_pin_pages(struct xe_vma *vma)
  67 {
  68         struct xe_vm *vm = xe_vma_vm(vma);
  69         struct xe_device *xe = vm->xe;
  70         const unsigned long num_pages = xe_vma_size(vma) >> PAGE_SHIFT;
  71         struct page **pages;
  72         bool in_kthread = !current->mm;
  73         unsigned long notifier_seq;
  74         int pinned, ret, i;
  75         bool read_only = xe_vma_read_only(vma);
  76
  77         lockdep_assert_held(&vm->lock);
  78         xe_assert(xe, xe_vma_is_userptr(vma));
  79 retry:
  80         if (vma->gpuva.flags & XE_VMA_DESTROYED)
  81                 return 0;
  82
  83         notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier);
  84         if (notifier_seq == vma->userptr.notifier_seq)
  85                 return 0;
  86
  87         pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL);
  88         if (!pages)
  89                 return -ENOMEM;
  90
  91         if (vma->userptr.sg) {
  92                 dma_unmap_sgtable(xe->drm.dev,
  93                                   vma->userptr.sg,
  94                                   read_only ? DMA_TO_DEVICE :
  95                                   DMA_BIDIRECTIONAL, 0);
  96                 sg_free_table(vma->userptr.sg);
  97                 vma->userptr.sg = NULL;
  98         }
  99
 100         pinned = ret = 0;
 101         if (in_kthread) {
 102                 if (!mmget_not_zero(vma->userptr.notifier.mm)) {
 103                         ret = -EFAULT;
 104                         goto mm_closed;
 105                 }
 106                 kthread_use_mm(vma->userptr.notifier.mm);
 107         }
 108
 109         while (pinned < num_pages) {
 110                 ret = get_user_pages_fast(xe_vma_userptr(vma) +
 111                                           pinned * PAGE_SIZE,
 112                                           num_pages - pinned,
 113                                           read_only ? 0 : FOLL_WRITE,
 114                                           &pages[pinned]);
 115                 if (ret < 0) {
 116                         if (in_kthread)
 117                                 ret = 0;
 118                         break;
 119                 }
 120
 121                 pinned += ret;
 122                 ret = 0;
 123         }
 124
 125         if (in_kthread) {
 126                 kthread_unuse_mm(vma->userptr.notifier.mm);
 127                 mmput(vma->userptr.notifier.mm);
 128         }
 129 mm_closed:
 130         if (ret)
 131                 goto out;
 132
 133         ret = sg_alloc_table_from_pages_segment(&vma->userptr.sgt, pages,
 134                                                 pinned, 0,
 135                                                 (u64)pinned << PAGE_SHIFT,
 136                                                 xe_sg_segment_size(xe->drm.dev),
 137                                                 GFP_KERNEL);
 138         if (ret) {
 139                 vma->userptr.sg = NULL;
 140                 goto out;
 141         }
 142         vma->userptr.sg = &vma->userptr.sgt;
 143
 144         ret = dma_map_sgtable(xe->drm.dev, vma->userptr.sg,
 145                               read_only ? DMA_TO_DEVICE :
 146                               DMA_BIDIRECTIONAL,
 147                               DMA_ATTR_SKIP_CPU_SYNC |
 148                               DMA_ATTR_NO_KERNEL_MAPPING);
 149         if (ret) {
 150                 sg_free_table(vma->userptr.sg);
 151                 vma->userptr.sg = NULL;
 152                 goto out;
 153         }
 154
 155         for (i = 0; i < pinned; ++i) {
 156                 if (!read_only) {
 157                         lock_page(pages[i]);
 158                         set_page_dirty(pages[i]);
 159                         unlock_page(pages[i]);
 160                 }
 161
 162                 mark_page_accessed(pages[i]);
 163         }
 164
 165 out:
 166         release_pages(pages, pinned);
 167         kvfree(pages);
 168
 169         if (!(ret < 0)) {
 170                 vma->userptr.notifier_seq = notifier_seq;
 171                 if (xe_vma_userptr_check_repin(vma) == -EAGAIN)
 172                         goto retry;
 173         }
 174
 175         return ret < 0 ? ret : 0;
 176 }
 177
 178 static bool preempt_fences_waiting(struct xe_vm *vm)
 179 {
 180         struct xe_exec_queue *q;
 181
 182         lockdep_assert_held(&vm->lock);
 183         xe_vm_assert_held(vm);
 184
 185         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
 186                 if (!q->compute.pfence ||
 187                     (q->compute.pfence && test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 188                                                    &q->compute.pfence->flags))) {
 189                         return true;
 190                 }
 191         }
 192
 193         return false;
 194 }
 195
 196 static void free_preempt_fences(struct list_head *list)
 197 {
 198         struct list_head *link, *next;
 199
 200         list_for_each_safe(link, next, list)
 201                 xe_preempt_fence_free(to_preempt_fence_from_link(link));
 202 }
 203
 204 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list,
 205                                 unsigned int *count)
 206 {
 207         lockdep_assert_held(&vm->lock);
 208         xe_vm_assert_held(vm);
 209
 210         if (*count >= vm->preempt.num_exec_queues)
 211                 return 0;
 212
 213         for (; *count < vm->preempt.num_exec_queues; ++(*count)) {
 214                 struct xe_preempt_fence *pfence = xe_preempt_fence_alloc();
 215
 216                 if (IS_ERR(pfence))
 217                         return PTR_ERR(pfence);
 218
 219                 list_move_tail(xe_preempt_fence_link(pfence), list);
 220         }
 221
 222         return 0;
 223 }
 224
 225 static int wait_for_existing_preempt_fences(struct xe_vm *vm)
 226 {
 227         struct xe_exec_queue *q;
 228
 229         xe_vm_assert_held(vm);
 230
 231         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
 232                 if (q->compute.pfence) {
 233                         long timeout = dma_fence_wait(q->compute.pfence, false);
 234
 235                         if (timeout < 0)
 236                                 return -ETIME;
 237                         dma_fence_put(q->compute.pfence);
 238                         q->compute.pfence = NULL;
 239                 }
 240         }
 241
 242         return 0;
 243 }
 244
 245 static bool xe_vm_is_idle(struct xe_vm *vm)
 246 {
 247         struct xe_exec_queue *q;
 248
 249         xe_vm_assert_held(vm);
 250         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
 251                 if (!xe_exec_queue_is_idle(q))
 252                         return false;
 253         }
 254
 255         return true;
 256 }
 257
 258 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list)
 259 {
 260         struct list_head *link;
 261         struct xe_exec_queue *q;
 262
 263         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
 264                 struct dma_fence *fence;
 265
 266                 link = list->next;
 267                 xe_assert(vm->xe, link != list);
 268
 269                 fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link),
 270                                              q, q->compute.context,
 271                                              ++q->compute.seqno);
 272                 dma_fence_put(q->compute.pfence);
 273                 q->compute.pfence = fence;
 274         }
 275 }
 276
 277 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo)
 278 {
 279         struct xe_exec_queue *q;
 280         int err;
 281
 282         if (!vm->preempt.num_exec_queues)
 283                 return 0;
 284
 285         err = xe_bo_lock(bo, true);
 286         if (err)
 287                 return err;
 288
 289         err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues);
 290         if (err)
 291                 goto out_unlock;
 292
 293         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link)
 294                 if (q->compute.pfence) {
 295                         dma_resv_add_fence(bo->ttm.base.resv,
 296                                            q->compute.pfence,
 297                                            DMA_RESV_USAGE_BOOKKEEP);
 298                 }
 299
 300 out_unlock:
 301         xe_bo_unlock(bo);
 302         return err;
 303 }
 304
 305 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
 306                                                 struct drm_exec *exec)
 307 {
 308         struct xe_exec_queue *q;
 309
 310         lockdep_assert_held(&vm->lock);
 311         xe_vm_assert_held(vm);
 312
 313         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) {
 314                 q->ops->resume(q);
 315
 316                 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->compute.pfence,
 317                                          DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
 318         }
 319 }
 320
 321 int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 322 {
 323         struct drm_gpuvm_exec vm_exec = {
 324                 .vm = &vm->gpuvm,
 325                 .flags = DRM_EXEC_INTERRUPTIBLE_WAIT,
 326                 .num_fences = 1,
 327         };
 328         struct drm_exec *exec = &vm_exec.exec;
 329         struct dma_fence *pfence;
 330         int err;
 331         bool wait;
 332
 333         xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm));
 334
 335         down_write(&vm->lock);
 336         err = drm_gpuvm_exec_lock(&vm_exec);
 337         if (err)
 338                 return err;
 339
 340         pfence = xe_preempt_fence_create(q, q->compute.context,
 341                                          ++q->compute.seqno);
 342         if (!pfence) {
 343                 err = -ENOMEM;
 344                 goto out_unlock;
 345         }
 346
 347         list_add(&q->compute.link, &vm->preempt.exec_queues);
 348         ++vm->preempt.num_exec_queues;
 349         q->compute.pfence = pfence;
 350
 351         down_read(&vm->userptr.notifier_lock);
 352
 353         drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence,
 354                                  DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
 355
 356         /*
 357          * Check to see if a preemption on VM is in flight or userptr
 358          * invalidation, if so trigger this preempt fence to sync state with
 359          * other preempt fences on the VM.
 360          */
 361         wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm);
 362         if (wait)
 363                 dma_fence_enable_sw_signaling(pfence);
 364
 365         up_read(&vm->userptr.notifier_lock);
 366
 367 out_unlock:
 368         drm_exec_fini(exec);
 369         up_write(&vm->lock);
 370
 371         return err;
 372 }
 373
 374 /**
 375  * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM
 376  * @vm: The VM.
 377  * @q: The exec_queue
 378  */
 379 void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 380 {
 381         if (!xe_vm_in_preempt_fence_mode(vm))
 382                 return;
 383
 384         down_write(&vm->lock);
 385         list_del(&q->compute.link);
 386         --vm->preempt.num_exec_queues;
 387         if (q->compute.pfence) {
 388                 dma_fence_enable_sw_signaling(q->compute.pfence);
 389                 dma_fence_put(q->compute.pfence);
 390                 q->compute.pfence = NULL;
 391         }
 392         up_write(&vm->lock);
 393 }
 394
 395 /**
 396  * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs
 397  * that need repinning.
 398  * @vm: The VM.
 399  *
 400  * This function checks for whether the VM has userptrs that need repinning,
 401  * and provides a release-type barrier on the userptr.notifier_lock after
 402  * checking.
 403  *
 404  * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are.
 405  */
 406 int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
 407 {
 408         lockdep_assert_held_read(&vm->userptr.notifier_lock);
 409
 410         return (list_empty(&vm->userptr.repin_list) &&
 411                 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
 412 }
 413
 414 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
 415
 416 static void xe_vm_kill(struct xe_vm *vm)
 417 {
 418         struct xe_exec_queue *q;
 419
 420         lockdep_assert_held(&vm->lock);
 421
 422         xe_vm_lock(vm, false);
 423         vm->flags |= XE_VM_FLAG_BANNED;
 424         trace_xe_vm_kill(vm);
 425
 426         list_for_each_entry(q, &vm->preempt.exec_queues, compute.link)
 427                 q->ops->kill(q);
 428         xe_vm_unlock(vm);
 429
 430         /* TODO: Inform user the VM is banned */
 431 }
 432
 433 /**
 434  * xe_vm_validate_should_retry() - Whether to retry after a validate error.
 435  * @exec: The drm_exec object used for locking before validation.
 436  * @err: The error returned from ttm_bo_validate().
 437  * @end: A ktime_t cookie that should be set to 0 before first use and
 438  * that should be reused on subsequent calls.
 439  *
 440  * With multiple active VMs, under memory pressure, it is possible that
 441  * ttm_bo_validate() run into -EDEADLK and in such case returns -ENOMEM.
 442  * Until ttm properly handles locking in such scenarios, best thing the
 443  * driver can do is retry with a timeout. Check if that is necessary, and
 444  * if so unlock the drm_exec's objects while keeping the ticket to prepare
 445  * for a rerun.
 446  *
 447  * Return: true if a retry after drm_exec_init() is recommended;
 448  * false otherwise.
 449  */
 450 bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end)
 451 {
 452         ktime_t cur;
 453
 454         if (err != -ENOMEM)
 455                 return false;
 456
 457         cur = ktime_get();
 458         *end = *end ? : ktime_add_ms(cur, XE_VM_REBIND_RETRY_TIMEOUT_MS);
 459         if (!ktime_before(cur, *end))
 460                 return false;
 461
 462         msleep(20);
 463         return true;
 464 }
 465
 466 static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
 467 {
 468         struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
 469         struct drm_gpuva *gpuva;
 470         int ret;
 471
 472         lockdep_assert_held(&vm->lock);
 473         drm_gpuvm_bo_for_each_va(gpuva, vm_bo)
 474                 list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind,
 475                                &vm->rebind_list);
 476
 477         ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false);
 478         if (ret)
 479                 return ret;
 480
 481         vm_bo->evicted = false;
 482         return 0;
 483 }
 484
 485 static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
 486                                  bool *done)
 487 {
 488         int err;
 489
 490         /*
 491          * 1 fence for each preempt fence plus a fence for each tile from a
 492          * possible rebind
 493          */
 494         err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, vm->preempt.num_exec_queues +
 495                                    vm->xe->info.tile_count);
 496         if (err)
 497                 return err;
 498
 499         if (xe_vm_is_idle(vm)) {
 500                 vm->preempt.rebind_deactivated = true;
 501                 *done = true;
 502                 return 0;
 503         }
 504
 505         if (!preempt_fences_waiting(vm)) {
 506                 *done = true;
 507                 return 0;
 508         }
 509
 510         err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, vm->preempt.num_exec_queues);
 511         if (err)
 512                 return err;
 513
 514         err = wait_for_existing_preempt_fences(vm);
 515         if (err)
 516                 return err;
 517
 518         return drm_gpuvm_validate(&vm->gpuvm, exec);
 519 }
 520
 521 static void preempt_rebind_work_func(struct work_struct *w)
 522 {
 523         struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work);
 524         struct drm_exec exec;
 525         struct dma_fence *rebind_fence;
 526         unsigned int fence_count = 0;
 527         LIST_HEAD(preempt_fences);
 528         ktime_t end = 0;
 529         int err = 0;
 530         long wait;
 531         int __maybe_unused tries = 0;
 532
 533         xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm));
 534         trace_xe_vm_rebind_worker_enter(vm);
 535
 536         down_write(&vm->lock);
 537
 538         if (xe_vm_is_closed_or_banned(vm)) {
 539                 up_write(&vm->lock);
 540                 trace_xe_vm_rebind_worker_exit(vm);
 541                 return;
 542         }
 543
 544 retry:
 545         if (xe_vm_userptr_check_repin(vm)) {
 546                 err = xe_vm_userptr_pin(vm);
 547                 if (err)
 548                         goto out_unlock_outer;
 549         }
 550
 551         drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
 552
 553         drm_exec_until_all_locked(&exec) {
 554                 bool done = false;
 555
 556                 err = xe_preempt_work_begin(&exec, vm, &done);
 557                 drm_exec_retry_on_contention(&exec);
 558                 if (err || done) {
 559                         drm_exec_fini(&exec);
 560                         if (err && xe_vm_validate_should_retry(&exec, err, &end))
 561                                 err = -EAGAIN;
 562
 563                         goto out_unlock_outer;
 564                 }
 565         }
 566
 567         err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
 568         if (err)
 569                 goto out_unlock;
 570
 571         rebind_fence = xe_vm_rebind(vm, true);
 572         if (IS_ERR(rebind_fence)) {
 573                 err = PTR_ERR(rebind_fence);
 574                 goto out_unlock;
 575         }
 576
 577         if (rebind_fence) {
 578                 dma_fence_wait(rebind_fence, false);
 579                 dma_fence_put(rebind_fence);
 580         }
 581
 582         /* Wait on munmap style VM unbinds */
 583         wait = dma_resv_wait_timeout(xe_vm_resv(vm),
 584                                      DMA_RESV_USAGE_KERNEL,
 585                                      false, MAX_SCHEDULE_TIMEOUT);
 586         if (wait <= 0) {
 587                 err = -ETIME;
 588                 goto out_unlock;
 589         }
 590
 591 #define retry_required(__tries, __vm) \
 592         (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \
 593         (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \
 594         __xe_vm_userptr_needs_repin(__vm))
 595
 596         down_read(&vm->userptr.notifier_lock);
 597         if (retry_required(tries, vm)) {
 598                 up_read(&vm->userptr.notifier_lock);
 599                 err = -EAGAIN;
 600                 goto out_unlock;
 601         }
 602
 603 #undef retry_required
 604
 605         spin_lock(&vm->xe->ttm.lru_lock);
 606         ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
 607         spin_unlock(&vm->xe->ttm.lru_lock);
 608
 609         /* Point of no return. */
 610         arm_preempt_fences(vm, &preempt_fences);
 611         resume_and_reinstall_preempt_fences(vm, &exec);
 612         up_read(&vm->userptr.notifier_lock);
 613
 614 out_unlock:
 615         drm_exec_fini(&exec);
 616 out_unlock_outer:
 617         if (err == -EAGAIN) {
 618                 trace_xe_vm_rebind_worker_retry(vm);
 619                 goto retry;
 620         }
 621
 622         if (err) {
 623                 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
 624                 xe_vm_kill(vm);
 625         }
 626         up_write(&vm->lock);
 627
 628         free_preempt_fences(&preempt_fences);
 629
 630         trace_xe_vm_rebind_worker_exit(vm);
 631 }
 632
 633 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
 634                                    const struct mmu_notifier_range *range,
 635                                    unsigned long cur_seq)
 636 {
 637         struct xe_vma *vma = container_of(mni, struct xe_vma, userptr.notifier);
 638         struct xe_vm *vm = xe_vma_vm(vma);
 639         struct dma_resv_iter cursor;
 640         struct dma_fence *fence;
 641         long err;
 642
 643         xe_assert(vm->xe, xe_vma_is_userptr(vma));
 644         trace_xe_vma_userptr_invalidate(vma);
 645
 646         if (!mmu_notifier_range_blockable(range))
 647                 return false;
 648
 649         down_write(&vm->userptr.notifier_lock);
 650         mmu_interval_set_seq(mni, cur_seq);
 651
 652         /* No need to stop gpu access if the userptr is not yet bound. */
 653         if (!vma->userptr.initial_bind) {
 654                 up_write(&vm->userptr.notifier_lock);
 655                 return true;
 656         }
 657
 658         /*
 659          * Tell exec and rebind worker they need to repin and rebind this
 660          * userptr.
 661          */
 662         if (!xe_vm_in_fault_mode(vm) &&
 663             !(vma->gpuva.flags & XE_VMA_DESTROYED) && vma->tile_present) {
 664                 spin_lock(&vm->userptr.invalidated_lock);
 665                 list_move_tail(&vma->userptr.invalidate_link,
 666                                &vm->userptr.invalidated);
 667                 spin_unlock(&vm->userptr.invalidated_lock);
 668         }
 669
 670         up_write(&vm->userptr.notifier_lock);
 671
 672         /*
 673          * Preempt fences turn into schedule disables, pipeline these.
 674          * Note that even in fault mode, we need to wait for binds and
 675          * unbinds to complete, and those are attached as BOOKMARK fences
 676          * to the vm.
 677          */
 678         dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
 679                             DMA_RESV_USAGE_BOOKKEEP);
 680         dma_resv_for_each_fence_unlocked(&cursor, fence)
 681                 dma_fence_enable_sw_signaling(fence);
 682         dma_resv_iter_end(&cursor);
 683
 684         err = dma_resv_wait_timeout(xe_vm_resv(vm),
 685                                     DMA_RESV_USAGE_BOOKKEEP,
 686                                     false, MAX_SCHEDULE_TIMEOUT);
 687         XE_WARN_ON(err <= 0);
 688
 689         if (xe_vm_in_fault_mode(vm)) {
 690                 err = xe_vm_invalidate_vma(vma);
 691                 XE_WARN_ON(err);
 692         }
 693
 694         trace_xe_vma_userptr_invalidate_complete(vma);
 695
 696         return true;
 697 }
 698
 699 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
 700         .invalidate = vma_userptr_invalidate,
 701 };
 702
 703 int xe_vm_userptr_pin(struct xe_vm *vm)
 704 {
 705         struct xe_vma *vma, *next;
 706         int err = 0;
 707         LIST_HEAD(tmp_evict);
 708
 709         lockdep_assert_held_write(&vm->lock);
 710
 711         /* Collect invalidated userptrs */
 712         spin_lock(&vm->userptr.invalidated_lock);
 713         list_for_each_entry_safe(vma, next, &vm->userptr.invalidated,
 714                                  userptr.invalidate_link) {
 715                 list_del_init(&vma->userptr.invalidate_link);
 716                 list_move_tail(&vma->combined_links.userptr,
 717                                &vm->userptr.repin_list);
 718         }
 719         spin_unlock(&vm->userptr.invalidated_lock);
 720
 721         /* Pin and move to temporary list */
 722         list_for_each_entry_safe(vma, next, &vm->userptr.repin_list,
 723                                  combined_links.userptr) {
 724                 err = xe_vma_userptr_pin_pages(vma);
 725                 if (err < 0)
 726                         return err;
 727
 728                 list_move_tail(&vma->combined_links.userptr, &vm->rebind_list);
 729         }
 730
 731         return 0;
 732 }
 733
 734 /**
 735  * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs
 736  * that need repinning.
 737  * @vm: The VM.
 738  *
 739  * This function does an advisory check for whether the VM has userptrs that
 740  * need repinning.
 741  *
 742  * Return: 0 if there are no indications of userptrs needing repinning,
 743  * -EAGAIN if there are.
 744  */
 745 int xe_vm_userptr_check_repin(struct xe_vm *vm)
 746 {
 747         return (list_empty_careful(&vm->userptr.repin_list) &&
 748                 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
 749 }
 750
 751 static struct dma_fence *
 752 xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
 753                struct xe_sync_entry *syncs, u32 num_syncs,
 754                bool first_op, bool last_op);
 755
 756 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 757 {
 758         struct dma_fence *fence = NULL;
 759         struct xe_vma *vma, *next;
 760
 761         lockdep_assert_held(&vm->lock);
 762         if (xe_vm_in_lr_mode(vm) && !rebind_worker)
 763                 return NULL;
 764
 765         xe_vm_assert_held(vm);
 766         list_for_each_entry_safe(vma, next, &vm->rebind_list,
 767                                  combined_links.rebind) {
 768                 xe_assert(vm->xe, vma->tile_present);
 769
 770                 list_del_init(&vma->combined_links.rebind);
 771                 dma_fence_put(fence);
 772                 if (rebind_worker)
 773                         trace_xe_vma_rebind_worker(vma);
 774                 else
 775                         trace_xe_vma_rebind_exec(vma);
 776                 fence = xe_vm_bind_vma(vma, NULL, NULL, 0, false, false);
 777                 if (IS_ERR(fence))
 778                         return fence;
 779         }
 780
 781         return fence;
 782 }
 783
 784 #define VMA_CREATE_FLAG_READ_ONLY       BIT(0)
 785 #define VMA_CREATE_FLAG_IS_NULL         BIT(1)
 786
 787 static struct xe_vma *xe_vma_create(struct xe_vm *vm,
 788                                     struct xe_bo *bo,
 789                                     u64 bo_offset_or_userptr,
 790                                     u64 start, u64 end,
 791                                     u16 pat_index, unsigned int flags)
 792 {
 793         struct xe_vma *vma;
 794         struct xe_tile *tile;
 795         u8 id;
 796         bool read_only = (flags & VMA_CREATE_FLAG_READ_ONLY);
 797         bool is_null = (flags & VMA_CREATE_FLAG_IS_NULL);
 798
 799         xe_assert(vm->xe, start < end);
 800         xe_assert(vm->xe, end < vm->size);
 801
 802         if (!bo && !is_null)    /* userptr */
 803                 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 804         else
 805                 vma = kzalloc(sizeof(*vma) - sizeof(struct xe_userptr),
 806                               GFP_KERNEL);
 807         if (!vma) {
 808                 vma = ERR_PTR(-ENOMEM);
 809                 return vma;
 810         }
 811
 812         INIT_LIST_HEAD(&vma->combined_links.rebind);
 813
 814         INIT_LIST_HEAD(&vma->gpuva.gem.entry);
 815         vma->gpuva.vm = &vm->gpuvm;
 816         vma->gpuva.va.addr = start;
 817         vma->gpuva.va.range = end - start + 1;
 818         if (read_only)
 819                 vma->gpuva.flags |= XE_VMA_READ_ONLY;
 820         if (is_null)
 821                 vma->gpuva.flags |= DRM_GPUVA_SPARSE;
 822
 823         for_each_tile(tile, vm->xe, id)
 824                 vma->tile_mask |= 0x1 << id;
 825
 826         if (GRAPHICS_VER(vm->xe) >= 20 || vm->xe->info.platform == XE_PVC)
 827                 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT;
 828
 829         vma->pat_index = pat_index;
 830
 831         if (bo) {
 832                 struct drm_gpuvm_bo *vm_bo;
 833
 834                 xe_bo_assert_held(bo);
 835
 836                 vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base);
 837                 if (IS_ERR(vm_bo)) {
 838                         kfree(vma);
 839                         return ERR_CAST(vm_bo);
 840                 }
 841
 842                 drm_gpuvm_bo_extobj_add(vm_bo);
 843                 drm_gem_object_get(&bo->ttm.base);
 844                 vma->gpuva.gem.obj = &bo->ttm.base;
 845                 vma->gpuva.gem.offset = bo_offset_or_userptr;
 846                 drm_gpuva_link(&vma->gpuva, vm_bo);
 847                 drm_gpuvm_bo_put(vm_bo);
 848         } else /* userptr or null */ {
 849                 if (!is_null) {
 850                         u64 size = end - start + 1;
 851                         int err;
 852
 853                         INIT_LIST_HEAD(&vma->userptr.invalidate_link);
 854                         vma->gpuva.gem.offset = bo_offset_or_userptr;
 855
 856                         err = mmu_interval_notifier_insert(&vma->userptr.notifier,
 857                                                            current->mm,
 858                                                            xe_vma_userptr(vma), size,
 859                                                            &vma_userptr_notifier_ops);
 860                         if (err) {
 861                                 kfree(vma);
 862                                 vma = ERR_PTR(err);
 863                                 return vma;
 864                         }
 865
 866                         vma->userptr.notifier_seq = LONG_MAX;
 867                 }
 868
 869                 xe_vm_get(vm);
 870         }
 871
 872         return vma;
 873 }
 874
 875 static void xe_vma_destroy_late(struct xe_vma *vma)
 876 {
 877         struct xe_vm *vm = xe_vma_vm(vma);
 878         struct xe_device *xe = vm->xe;
 879         bool read_only = xe_vma_read_only(vma);
 880
 881         if (xe_vma_is_userptr(vma)) {
 882                 if (vma->userptr.sg) {
 883                         dma_unmap_sgtable(xe->drm.dev,
 884                                           vma->userptr.sg,
 885                                           read_only ? DMA_TO_DEVICE :
 886                                           DMA_BIDIRECTIONAL, 0);
 887                         sg_free_table(vma->userptr.sg);
 888                         vma->userptr.sg = NULL;
 889                 }
 890
 891                 /*
 892                  * Since userptr pages are not pinned, we can't remove
 893                  * the notifer until we're sure the GPU is not accessing
 894                  * them anymore
 895                  */
 896                 mmu_interval_notifier_remove(&vma->userptr.notifier);
 897                 xe_vm_put(vm);
 898         } else if (xe_vma_is_null(vma)) {
 899                 xe_vm_put(vm);
 900         } else {
 901                 xe_bo_put(xe_vma_bo(vma));
 902         }
 903
 904         kfree(vma);
 905 }
 906
 907 static void vma_destroy_work_func(struct work_struct *w)
 908 {
 909         struct xe_vma *vma =
 910                 container_of(w, struct xe_vma, destroy_work);
 911
 912         xe_vma_destroy_late(vma);
 913 }
 914
 915 static void vma_destroy_cb(struct dma_fence *fence,
 916                            struct dma_fence_cb *cb)
 917 {
 918         struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb);
 919
 920         INIT_WORK(&vma->destroy_work, vma_destroy_work_func);
 921         queue_work(system_unbound_wq, &vma->destroy_work);
 922 }
 923
 924 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
 925 {
 926         struct xe_vm *vm = xe_vma_vm(vma);
 927
 928         lockdep_assert_held_write(&vm->lock);
 929         xe_assert(vm->xe, list_empty(&vma->combined_links.destroy));
 930
 931         if (xe_vma_is_userptr(vma)) {
 932                 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED);
 933
 934                 spin_lock(&vm->userptr.invalidated_lock);
 935                 list_del(&vma->userptr.invalidate_link);
 936                 spin_unlock(&vm->userptr.invalidated_lock);
 937         } else if (!xe_vma_is_null(vma)) {
 938                 xe_bo_assert_held(xe_vma_bo(vma));
 939
 940                 drm_gpuva_unlink(&vma->gpuva);
 941         }
 942
 943         xe_vm_assert_held(vm);
 944         if (fence) {
 945                 int ret = dma_fence_add_callback(fence, &vma->destroy_cb,
 946                                                  vma_destroy_cb);
 947
 948                 if (ret) {
 949                         XE_WARN_ON(ret != -ENOENT);
 950                         xe_vma_destroy_late(vma);
 951                 }
 952         } else {
 953                 xe_vma_destroy_late(vma);
 954         }
 955 }
 956
 957 /**
 958  * xe_vm_prepare_vma() - drm_exec utility to lock a vma
 959  * @exec: The drm_exec object we're currently locking for.
 960  * @vma: The vma for witch we want to lock the vm resv and any attached
 961  * object's resv.
 962  * @num_shared: The number of dma-fence slots to pre-allocate in the
 963  * objects' reservation objects.
 964  *
 965  * Return: 0 on success, negative error code on error. In particular
 966  * may return -EDEADLK on WW transaction contention and -EINTR if
 967  * an interruptible wait is terminated by a signal.
 968  */
 969 int xe_vm_prepare_vma(struct drm_exec *exec, struct xe_vma *vma,
 970                       unsigned int num_shared)
 971 {
 972         struct xe_vm *vm = xe_vma_vm(vma);
 973         struct xe_bo *bo = xe_vma_bo(vma);
 974         int err;
 975
 976         XE_WARN_ON(!vm);
 977         err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), num_shared);
 978         if (!err && bo && !bo->vm)
 979                 err = drm_exec_prepare_obj(exec, &bo->ttm.base, num_shared);
 980
 981         return err;
 982 }
 983
 984 static void xe_vma_destroy_unlocked(struct xe_vma *vma)
 985 {
 986         struct drm_exec exec;
 987         int err;
 988
 989         drm_exec_init(&exec, 0, 0);
 990         drm_exec_until_all_locked(&exec) {
 991                 err = xe_vm_prepare_vma(&exec, vma, 0);
 992                 drm_exec_retry_on_contention(&exec);
 993                 if (XE_WARN_ON(err))
 994                         break;
 995         }
 996
 997         xe_vma_destroy(vma, NULL);
 998
 999         drm_exec_fini(&exec);
1000 }
1001
1002 struct xe_vma *
1003 xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range)
1004 {
1005         struct drm_gpuva *gpuva;
1006
1007         lockdep_assert_held(&vm->lock);
1008
1009         if (xe_vm_is_closed_or_banned(vm))
1010                 return NULL;
1011
1012         xe_assert(vm->xe, start + range <= vm->size);
1013
1014         gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range);
1015
1016         return gpuva ? gpuva_to_vma(gpuva) : NULL;
1017 }
1018
1019 static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma)
1020 {
1021         int err;
1022
1023         xe_assert(vm->xe, xe_vma_vm(vma) == vm);
1024         lockdep_assert_held(&vm->lock);
1025
1026         err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva);
1027         XE_WARN_ON(err);        /* Shouldn't be possible */
1028
1029         return err;
1030 }
1031
1032 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma)
1033 {
1034         xe_assert(vm->xe, xe_vma_vm(vma) == vm);
1035         lockdep_assert_held(&vm->lock);
1036
1037         drm_gpuva_remove(&vma->gpuva);
1038         if (vm->usm.last_fault_vma == vma)
1039                 vm->usm.last_fault_vma = NULL;
1040 }
1041
1042 static struct drm_gpuva_op *xe_vm_op_alloc(void)
1043 {
1044         struct xe_vma_op *op;
1045
1046         op = kzalloc(sizeof(*op), GFP_KERNEL);
1047
1048         if (unlikely(!op))
1049                 return NULL;
1050
1051         return &op->base;
1052 }
1053
1054 static void xe_vm_free(struct drm_gpuvm *gpuvm);
1055
1056 static struct drm_gpuvm_ops gpuvm_ops = {
1057         .op_alloc = xe_vm_op_alloc,
1058         .vm_bo_validate = xe_gpuvm_validate,
1059         .vm_free = xe_vm_free,
1060 };
1061
1062 static u64 pde_encode_pat_index(struct xe_device *xe, u16 pat_index)
1063 {
1064         u64 pte = 0;
1065
1066         if (pat_index & BIT(0))
1067                 pte |= XE_PPGTT_PTE_PAT0;
1068
1069         if (pat_index & BIT(1))
1070                 pte |= XE_PPGTT_PTE_PAT1;
1071
1072         return pte;
1073 }
1074
1075 static u64 pte_encode_pat_index(struct xe_device *xe, u16 pat_index,
1076                                 u32 pt_level)
1077 {
1078         u64 pte = 0;
1079
1080         if (pat_index & BIT(0))
1081                 pte |= XE_PPGTT_PTE_PAT0;
1082
1083         if (pat_index & BIT(1))
1084                 pte |= XE_PPGTT_PTE_PAT1;
1085
1086         if (pat_index & BIT(2)) {
1087                 if (pt_level)
1088                         pte |= XE_PPGTT_PDE_PDPE_PAT2;
1089                 else
1090                         pte |= XE_PPGTT_PTE_PAT2;
1091         }
1092
1093         if (pat_index & BIT(3))
1094                 pte |= XELPG_PPGTT_PTE_PAT3;
1095
1096         if (pat_index & (BIT(4)))
1097                 pte |= XE2_PPGTT_PTE_PAT4;
1098
1099         return pte;
1100 }
1101
1102 static u64 pte_encode_ps(u32 pt_level)
1103 {
1104         XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL);
1105
1106         if (pt_level == 1)
1107                 return XE_PDE_PS_2M;
1108         else if (pt_level == 2)
1109                 return XE_PDPE_PS_1G;
1110
1111         return 0;
1112 }
1113
1114 static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset,
1115                               const u16 pat_index)
1116 {
1117         struct xe_device *xe = xe_bo_device(bo);
1118         u64 pde;
1119
1120         pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
1121         pde |= XE_PAGE_PRESENT | XE_PAGE_RW;
1122         pde |= pde_encode_pat_index(xe, pat_index);
1123
1124         return pde;
1125 }
1126
1127 static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
1128                               u16 pat_index, u32 pt_level)
1129 {
1130         struct xe_device *xe = xe_bo_device(bo);
1131         u64 pte;
1132
1133         pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
1134         pte |= XE_PAGE_PRESENT | XE_PAGE_RW;
1135         pte |= pte_encode_pat_index(xe, pat_index, pt_level);
1136         pte |= pte_encode_ps(pt_level);
1137
1138         if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
1139                 pte |= XE_PPGTT_PTE_DM;
1140
1141         return pte;
1142 }
1143
1144 static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma,
1145                                u16 pat_index, u32 pt_level)
1146 {
1147         struct xe_device *xe = xe_vma_vm(vma)->xe;
1148
1149         pte |= XE_PAGE_PRESENT;
1150
1151         if (likely(!xe_vma_read_only(vma)))
1152                 pte |= XE_PAGE_RW;
1153
1154         pte |= pte_encode_pat_index(xe, pat_index, pt_level);
1155         pte |= pte_encode_ps(pt_level);
1156
1157         if (unlikely(xe_vma_is_null(vma)))
1158                 pte |= XE_PTE_NULL;
1159
1160         return pte;
1161 }
1162
1163 static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr,
1164                                 u16 pat_index,
1165                                 u32 pt_level, bool devmem, u64 flags)
1166 {
1167         u64 pte;
1168
1169         /* Avoid passing random bits directly as flags */
1170         xe_assert(xe, !(flags & ~XE_PTE_PS64));
1171
1172         pte = addr;
1173         pte |= XE_PAGE_PRESENT | XE_PAGE_RW;
1174         pte |= pte_encode_pat_index(xe, pat_index, pt_level);
1175         pte |= pte_encode_ps(pt_level);
1176
1177         if (devmem)
1178                 pte |= XE_PPGTT_PTE_DM;
1179
1180         pte |= flags;
1181
1182         return pte;
1183 }
1184
1185 static const struct xe_pt_ops xelp_pt_ops = {
1186         .pte_encode_bo = xelp_pte_encode_bo,
1187         .pte_encode_vma = xelp_pte_encode_vma,
1188         .pte_encode_addr = xelp_pte_encode_addr,
1189         .pde_encode_bo = xelp_pde_encode_bo,
1190 };
1191
1192 static void vm_destroy_work_func(struct work_struct *w);
1193
1194 /**
1195  * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the
1196  * given tile and vm.
1197  * @xe: xe device.
1198  * @tile: tile to set up for.
1199  * @vm: vm to set up for.
1200  *
1201  * Sets up a pagetable tree with one page-table per level and a single
1202  * leaf PTE. All pagetable entries point to the single page-table or,
1203  * for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and
1204  * writes become NOPs.
1205  *
1206  * Return: 0 on success, negative error code on error.
1207  */
1208 static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile,
1209                                 struct xe_vm *vm)
1210 {
1211         u8 id = tile->id;
1212         int i;
1213
1214         for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) {
1215                 vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i);
1216                 if (IS_ERR(vm->scratch_pt[id][i]))
1217                         return PTR_ERR(vm->scratch_pt[id][i]);
1218
1219                 xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]);
1220         }
1221
1222         return 0;
1223 }
1224
1225 static void xe_vm_free_scratch(struct xe_vm *vm)
1226 {
1227         struct xe_tile *tile;
1228         u8 id;
1229
1230         if (!xe_vm_has_scratch(vm))
1231                 return;
1232
1233         for_each_tile(tile, vm->xe, id) {
1234                 u32 i;
1235
1236                 if (!vm->pt_root[id])
1237                         continue;
1238
1239                 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i)
1240                         if (vm->scratch_pt[id][i])
1241                                 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL);
1242         }
1243 }
1244
1245 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
1246 {
1247         struct drm_gem_object *vm_resv_obj;
1248         struct xe_vm *vm;
1249         int err, number_tiles = 0;
1250         struct xe_tile *tile;
1251         u8 id;
1252
1253         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1254         if (!vm)
1255                 return ERR_PTR(-ENOMEM);
1256
1257         vm->xe = xe;
1258
1259         vm->size = 1ull << xe->info.va_bits;
1260
1261         vm->flags = flags;
1262
1263         init_rwsem(&vm->lock);
1264
1265         INIT_LIST_HEAD(&vm->rebind_list);
1266
1267         INIT_LIST_HEAD(&vm->userptr.repin_list);
1268         INIT_LIST_HEAD(&vm->userptr.invalidated);
1269         init_rwsem(&vm->userptr.notifier_lock);
1270         spin_lock_init(&vm->userptr.invalidated_lock);
1271
1272         INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
1273
1274         INIT_LIST_HEAD(&vm->preempt.exec_queues);
1275         vm->preempt.min_run_period_ms = 10;     /* FIXME: Wire up to uAPI */
1276
1277         for_each_tile(tile, xe, id)
1278                 xe_range_fence_tree_init(&vm->rftree[id]);
1279
1280         vm->pt_ops = &xelp_pt_ops;
1281
1282         if (!(flags & XE_VM_FLAG_MIGRATION))
1283                 xe_device_mem_access_get(xe);
1284
1285         vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
1286         if (!vm_resv_obj) {
1287                 err = -ENOMEM;
1288                 goto err_no_resv;
1289         }
1290
1291         drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm,
1292                        vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops);
1293
1294         drm_gem_object_put(vm_resv_obj);
1295
1296         err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
1297         if (err)
1298                 goto err_close;
1299
1300         if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
1301                 vm->flags |= XE_VM_FLAG_64K;
1302
1303         for_each_tile(tile, xe, id) {
1304                 if (flags & XE_VM_FLAG_MIGRATION &&
1305                     tile->id != XE_VM_FLAG_TILE_ID(flags))
1306                         continue;
1307
1308                 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level);
1309                 if (IS_ERR(vm->pt_root[id])) {
1310                         err = PTR_ERR(vm->pt_root[id]);
1311                         vm->pt_root[id] = NULL;
1312                         goto err_unlock_close;
1313                 }
1314         }
1315
1316         if (xe_vm_has_scratch(vm)) {
1317                 for_each_tile(tile, xe, id) {
1318                         if (!vm->pt_root[id])
1319                                 continue;
1320
1321                         err = xe_vm_create_scratch(xe, tile, vm);
1322                         if (err)
1323                                 goto err_unlock_close;
1324                 }
1325                 vm->batch_invalidate_tlb = true;
1326         }
1327
1328         if (flags & XE_VM_FLAG_LR_MODE) {
1329                 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func);
1330                 vm->flags |= XE_VM_FLAG_LR_MODE;
1331                 vm->batch_invalidate_tlb = false;
1332         }
1333
1334         /* Fill pt_root after allocating scratch tables */
1335         for_each_tile(tile, xe, id) {
1336                 if (!vm->pt_root[id])
1337                         continue;
1338
1339                 xe_pt_populate_empty(tile, vm, vm->pt_root[id]);
1340         }
1341         dma_resv_unlock(xe_vm_resv(vm));
1342
1343         /* Kernel migration VM shouldn't have a circular loop.. */
1344         if (!(flags & XE_VM_FLAG_MIGRATION)) {
1345                 for_each_tile(tile, xe, id) {
1346                         struct xe_gt *gt = tile->primary_gt;
1347                         struct xe_vm *migrate_vm;
1348                         struct xe_exec_queue *q;
1349                         u32 create_flags = EXEC_QUEUE_FLAG_VM;
1350
1351                         if (!vm->pt_root[id])
1352                                 continue;
1353
1354                         migrate_vm = xe_migrate_get_vm(tile->migrate);
1355                         q = xe_exec_queue_create_class(xe, gt, migrate_vm,
1356                                                        XE_ENGINE_CLASS_COPY,
1357                                                        create_flags);
1358                         xe_vm_put(migrate_vm);
1359                         if (IS_ERR(q)) {
1360                                 err = PTR_ERR(q);
1361                                 goto err_close;
1362                         }
1363                         vm->q[id] = q;
1364                         number_tiles++;
1365                 }
1366         }
1367
1368         if (number_tiles > 1)
1369                 vm->composite_fence_ctx = dma_fence_context_alloc(1);
1370
1371         mutex_lock(&xe->usm.lock);
1372         if (flags & XE_VM_FLAG_FAULT_MODE)
1373                 xe->usm.num_vm_in_fault_mode++;
1374         else if (!(flags & XE_VM_FLAG_MIGRATION))
1375                 xe->usm.num_vm_in_non_fault_mode++;
1376         mutex_unlock(&xe->usm.lock);
1377
1378         trace_xe_vm_create(vm);
1379
1380         return vm;
1381
1382 err_unlock_close:
1383         dma_resv_unlock(xe_vm_resv(vm));
1384 err_close:
1385         xe_vm_close_and_put(vm);
1386         return ERR_PTR(err);
1387
1388 err_no_resv:
1389         for_each_tile(tile, xe, id)
1390                 xe_range_fence_tree_fini(&vm->rftree[id]);
1391         kfree(vm);
1392         if (!(flags & XE_VM_FLAG_MIGRATION))
1393                 xe_device_mem_access_put(xe);
1394         return ERR_PTR(err);
1395 }
1396
1397 static void xe_vm_close(struct xe_vm *vm)
1398 {
1399         down_write(&vm->lock);
1400         vm->size = 0;
1401         up_write(&vm->lock);
1402 }
1403
1404 void xe_vm_close_and_put(struct xe_vm *vm)
1405 {
1406         LIST_HEAD(contested);
1407         struct xe_device *xe = vm->xe;
1408         struct xe_tile *tile;
1409         struct xe_vma *vma, *next_vma;
1410         struct drm_gpuva *gpuva, *next;
1411         u8 id;
1412
1413         xe_assert(xe, !vm->preempt.num_exec_queues);
1414
1415         xe_vm_close(vm);
1416         if (xe_vm_in_preempt_fence_mode(vm))
1417                 flush_work(&vm->preempt.rebind_work);
1418
1419         down_write(&vm->lock);
1420         for_each_tile(tile, xe, id) {
1421                 if (vm->q[id])
1422                         xe_exec_queue_last_fence_put(vm->q[id], vm);
1423         }
1424         up_write(&vm->lock);
1425
1426         for_each_tile(tile, xe, id) {
1427                 if (vm->q[id]) {
1428                         xe_exec_queue_kill(vm->q[id]);
1429                         xe_exec_queue_put(vm->q[id]);
1430                         vm->q[id] = NULL;
1431                 }
1432         }
1433
1434         down_write(&vm->lock);
1435         xe_vm_lock(vm, false);
1436         drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) {
1437                 vma = gpuva_to_vma(gpuva);
1438
1439                 if (xe_vma_has_no_bo(vma)) {
1440                         down_read(&vm->userptr.notifier_lock);
1441                         vma->gpuva.flags |= XE_VMA_DESTROYED;
1442                         up_read(&vm->userptr.notifier_lock);
1443                 }
1444
1445                 xe_vm_remove_vma(vm, vma);
1446
1447                 /* easy case, remove from VMA? */
1448                 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) {
1449                         list_del_init(&vma->combined_links.rebind);
1450                         xe_vma_destroy(vma, NULL);
1451                         continue;
1452                 }
1453
1454                 list_move_tail(&vma->combined_links.destroy, &contested);
1455                 vma->gpuva.flags |= XE_VMA_DESTROYED;
1456         }
1457
1458         /*
1459          * All vm operations will add shared fences to resv.
1460          * The only exception is eviction for a shared object,
1461          * but even so, the unbind when evicted would still
1462          * install a fence to resv. Hence it's safe to
1463          * destroy the pagetables immediately.
1464          */
1465         xe_vm_free_scratch(vm);
1466
1467         for_each_tile(tile, xe, id) {
1468                 if (vm->pt_root[id]) {
1469                         xe_pt_destroy(vm->pt_root[id], vm->flags, NULL);
1470                         vm->pt_root[id] = NULL;
1471                 }
1472         }
1473         xe_vm_unlock(vm);
1474
1475         /*
1476          * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL
1477          * Since we hold a refcount to the bo, we can remove and free
1478          * the members safely without locking.
1479          */
1480         list_for_each_entry_safe(vma, next_vma, &contested,
1481                                  combined_links.destroy) {
1482                 list_del_init(&vma->combined_links.destroy);
1483                 xe_vma_destroy_unlocked(vma);
1484         }
1485
1486         up_write(&vm->lock);
1487
1488         mutex_lock(&xe->usm.lock);
1489         if (vm->flags & XE_VM_FLAG_FAULT_MODE)
1490                 xe->usm.num_vm_in_fault_mode--;
1491         else if (!(vm->flags & XE_VM_FLAG_MIGRATION))
1492                 xe->usm.num_vm_in_non_fault_mode--;
1493         mutex_unlock(&xe->usm.lock);
1494
1495         for_each_tile(tile, xe, id)
1496                 xe_range_fence_tree_fini(&vm->rftree[id]);
1497
1498         xe_vm_put(vm);
1499 }
1500
1501 static void vm_destroy_work_func(struct work_struct *w)
1502 {
1503         struct xe_vm *vm =
1504                 container_of(w, struct xe_vm, destroy_work);
1505         struct xe_device *xe = vm->xe;
1506         struct xe_tile *tile;
1507         u8 id;
1508         void *lookup;
1509
1510         /* xe_vm_close_and_put was not called? */
1511         xe_assert(xe, !vm->size);
1512
1513         if (!(vm->flags & XE_VM_FLAG_MIGRATION)) {
1514                 xe_device_mem_access_put(xe);
1515
1516                 if (xe->info.has_asid && vm->usm.asid) {
1517                         mutex_lock(&xe->usm.lock);
1518                         lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
1519                         xe_assert(xe, lookup == vm);
1520                         mutex_unlock(&xe->usm.lock);
1521                 }
1522         }
1523
1524         for_each_tile(tile, xe, id)
1525                 XE_WARN_ON(vm->pt_root[id]);
1526
1527         trace_xe_vm_free(vm);
1528         dma_fence_put(vm->rebind_fence);
1529         kfree(vm);
1530 }
1531
1532 static void xe_vm_free(struct drm_gpuvm *gpuvm)
1533 {
1534         struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
1535
1536         /* To destroy the VM we need to be able to sleep */
1537         queue_work(system_unbound_wq, &vm->destroy_work);
1538 }
1539
1540 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
1541 {
1542         struct xe_vm *vm;
1543
1544         mutex_lock(&xef->vm.lock);
1545         vm = xa_load(&xef->vm.xa, id);
1546         if (vm)
1547                 xe_vm_get(vm);
1548         mutex_unlock(&xef->vm.lock);
1549
1550         return vm;
1551 }
1552
1553 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile)
1554 {
1555         return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0,
1556                                          tile_to_xe(tile)->pat.idx[XE_CACHE_WB]);
1557 }
1558
1559 static struct xe_exec_queue *
1560 to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
1561 {
1562         return q ? q : vm->q[0];
1563 }
1564
1565 static struct dma_fence *
1566 xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
1567                  struct xe_sync_entry *syncs, u32 num_syncs,
1568                  bool first_op, bool last_op)
1569 {
1570         struct xe_vm *vm = xe_vma_vm(vma);
1571         struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1572         struct xe_tile *tile;
1573         struct dma_fence *fence = NULL;
1574         struct dma_fence **fences = NULL;
1575         struct dma_fence_array *cf = NULL;
1576         int cur_fence = 0, i;
1577         int number_tiles = hweight8(vma->tile_present);
1578         int err;
1579         u8 id;
1580
1581         trace_xe_vma_unbind(vma);
1582
1583         if (number_tiles > 1) {
1584                 fences = kmalloc_array(number_tiles, sizeof(*fences),
1585                                        GFP_KERNEL);
1586                 if (!fences)
1587                         return ERR_PTR(-ENOMEM);
1588         }
1589
1590         for_each_tile(tile, vm->xe, id) {
1591                 if (!(vma->tile_present & BIT(id)))
1592                         goto next;
1593
1594                 fence = __xe_pt_unbind_vma(tile, vma, q ? q : vm->q[id],
1595                                            first_op ? syncs : NULL,
1596                                            first_op ? num_syncs : 0);
1597                 if (IS_ERR(fence)) {
1598                         err = PTR_ERR(fence);
1599                         goto err_fences;
1600                 }
1601
1602                 if (fences)
1603                         fences[cur_fence++] = fence;
1604
1605 next:
1606                 if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list))
1607                         q = list_next_entry(q, multi_gt_list);
1608         }
1609
1610         if (fences) {
1611                 cf = dma_fence_array_create(number_tiles, fences,
1612                                             vm->composite_fence_ctx,
1613                                             vm->composite_fence_seqno++,
1614                                             false);
1615                 if (!cf) {
1616                         --vm->composite_fence_seqno;
1617                         err = -ENOMEM;
1618                         goto err_fences;
1619                 }
1620         }
1621
1622         fence = cf ? &cf->base : !fence ?
1623                 xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence;
1624         if (last_op) {
1625                 for (i = 0; i < num_syncs; i++)
1626                         xe_sync_entry_signal(&syncs[i], NULL, fence);
1627         }
1628
1629         return fence;
1630
1631 err_fences:
1632         if (fences) {
1633                 while (cur_fence)
1634                         dma_fence_put(fences[--cur_fence]);
1635                 kfree(fences);
1636         }
1637
1638         return ERR_PTR(err);
1639 }
1640
1641 static struct dma_fence *
1642 xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
1643                struct xe_sync_entry *syncs, u32 num_syncs,
1644                bool first_op, bool last_op)
1645 {
1646         struct xe_tile *tile;
1647         struct dma_fence *fence;
1648         struct dma_fence **fences = NULL;
1649         struct dma_fence_array *cf = NULL;
1650         struct xe_vm *vm = xe_vma_vm(vma);
1651         int cur_fence = 0, i;
1652         int number_tiles = hweight8(vma->tile_mask);
1653         int err;
1654         u8 id;
1655
1656         trace_xe_vma_bind(vma);
1657
1658         if (number_tiles > 1) {
1659                 fences = kmalloc_array(number_tiles, sizeof(*fences),
1660                                        GFP_KERNEL);
1661                 if (!fences)
1662                         return ERR_PTR(-ENOMEM);
1663         }
1664
1665         for_each_tile(tile, vm->xe, id) {
1666                 if (!(vma->tile_mask & BIT(id)))
1667                         goto next;
1668
1669                 fence = __xe_pt_bind_vma(tile, vma, q ? q : vm->q[id],
1670                                          first_op ? syncs : NULL,
1671                                          first_op ? num_syncs : 0,
1672                                          vma->tile_present & BIT(id));
1673                 if (IS_ERR(fence)) {
1674                         err = PTR_ERR(fence);
1675                         goto err_fences;
1676                 }
1677
1678                 if (fences)
1679                         fences[cur_fence++] = fence;
1680
1681 next:
1682                 if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list))
1683                         q = list_next_entry(q, multi_gt_list);
1684         }
1685
1686         if (fences) {
1687                 cf = dma_fence_array_create(number_tiles, fences,
1688                                             vm->composite_fence_ctx,
1689                                             vm->composite_fence_seqno++,
1690                                             false);
1691                 if (!cf) {
1692                         --vm->composite_fence_seqno;
1693                         err = -ENOMEM;
1694                         goto err_fences;
1695                 }
1696         }
1697
1698         if (last_op) {
1699                 for (i = 0; i < num_syncs; i++)
1700                         xe_sync_entry_signal(&syncs[i], NULL,
1701                                              cf ? &cf->base : fence);
1702         }
1703
1704         return cf ? &cf->base : fence;
1705
1706 err_fences:
1707         if (fences) {
1708                 while (cur_fence)
1709                         dma_fence_put(fences[--cur_fence]);
1710                 kfree(fences);
1711         }
1712
1713         return ERR_PTR(err);
1714 }
1715
1716 static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
1717                         struct xe_exec_queue *q, struct xe_sync_entry *syncs,
1718                         u32 num_syncs, bool immediate, bool first_op,
1719                         bool last_op)
1720 {
1721         struct dma_fence *fence;
1722         struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1723
1724         xe_vm_assert_held(vm);
1725
1726         if (immediate) {
1727                 fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op,
1728                                        last_op);
1729                 if (IS_ERR(fence))
1730                         return PTR_ERR(fence);
1731         } else {
1732                 int i;
1733
1734                 xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
1735
1736                 fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
1737                 if (last_op) {
1738                         for (i = 0; i < num_syncs; i++)
1739                                 xe_sync_entry_signal(&syncs[i], NULL, fence);
1740                 }
1741         }
1742
1743         if (last_op)
1744                 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
1745         dma_fence_put(fence);
1746
1747         return 0;
1748 }
1749
1750 static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
1751                       struct xe_bo *bo, struct xe_sync_entry *syncs,
1752                       u32 num_syncs, bool immediate, bool first_op,
1753                       bool last_op)
1754 {
1755         int err;
1756
1757         xe_vm_assert_held(vm);
1758         xe_bo_assert_held(bo);
1759
1760         if (bo && immediate) {
1761                 err = xe_bo_validate(bo, vm, true);
1762                 if (err)
1763                         return err;
1764         }
1765
1766         return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op,
1767                             last_op);
1768 }
1769
1770 static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
1771                         struct xe_exec_queue *q, struct xe_sync_entry *syncs,
1772                         u32 num_syncs, bool first_op, bool last_op)
1773 {
1774         struct dma_fence *fence;
1775         struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1776
1777         xe_vm_assert_held(vm);
1778         xe_bo_assert_held(xe_vma_bo(vma));
1779
1780         fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op);
1781         if (IS_ERR(fence))
1782                 return PTR_ERR(fence);
1783
1784         xe_vma_destroy(vma, fence);
1785         if (last_op)
1786                 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
1787         dma_fence_put(fence);
1788
1789         return 0;
1790 }
1791
1792 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \
1793                                     DRM_XE_VM_CREATE_FLAG_LR_MODE | \
1794                                     DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
1795
1796 int xe_vm_create_ioctl(struct drm_device *dev, void *data,
1797                        struct drm_file *file)
1798 {
1799         struct xe_device *xe = to_xe_device(dev);
1800         struct xe_file *xef = to_xe_file(file);
1801         struct drm_xe_vm_create *args = data;
1802         struct xe_tile *tile;
1803         struct xe_vm *vm;
1804         u32 id, asid;
1805         int err;
1806         u32 flags = 0;
1807
1808         if (XE_IOCTL_DBG(xe, args->extensions))
1809                 return -EINVAL;
1810
1811         if (XE_WA(xe_root_mmio_gt(xe), 14016763929))
1812                 args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
1813
1814         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE &&
1815                          !xe->info.has_usm))
1816                 return -EINVAL;
1817
1818         if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1819                 return -EINVAL;
1820
1821         if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS))
1822                 return -EINVAL;
1823
1824         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE &&
1825                          args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE))
1826                 return -EINVAL;
1827
1828         if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) &&
1829                          args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE))
1830                 return -EINVAL;
1831
1832         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE &&
1833                          xe_device_in_non_fault_mode(xe)))
1834                 return -EINVAL;
1835
1836         if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) &&
1837                          xe_device_in_fault_mode(xe)))
1838                 return -EINVAL;
1839
1840         if (XE_IOCTL_DBG(xe, args->extensions))
1841                 return -EINVAL;
1842
1843         if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE)
1844                 flags |= XE_VM_FLAG_SCRATCH_PAGE;
1845         if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE)
1846                 flags |= XE_VM_FLAG_LR_MODE;
1847         if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
1848                 flags |= XE_VM_FLAG_FAULT_MODE;
1849
1850         vm = xe_vm_create(xe, flags);
1851         if (IS_ERR(vm))
1852                 return PTR_ERR(vm);
1853
1854         mutex_lock(&xef->vm.lock);
1855         err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL);
1856         mutex_unlock(&xef->vm.lock);
1857         if (err) {
1858                 xe_vm_close_and_put(vm);
1859                 return err;
1860         }
1861
1862         if (xe->info.has_asid) {
1863                 mutex_lock(&xe->usm.lock);
1864                 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm,
1865                                       XA_LIMIT(1, XE_MAX_ASID - 1),
1866                                       &xe->usm.next_asid, GFP_KERNEL);
1867                 mutex_unlock(&xe->usm.lock);
1868                 if (err < 0) {
1869                         xe_vm_close_and_put(vm);
1870                         return err;
1871                 }
1872                 err = 0;
1873                 vm->usm.asid = asid;
1874         }
1875
1876         args->vm_id = id;
1877         vm->xef = xef;
1878
1879         /* Record BO memory for VM pagetable created against client */
1880         for_each_tile(tile, xe, id)
1881                 if (vm->pt_root[id])
1882                         xe_drm_client_add_bo(vm->xef->client, vm->pt_root[id]->bo);
1883
1884 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
1885         /* Warning: Security issue - never enable by default */
1886         args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE);
1887 #endif
1888
1889         return 0;
1890 }
1891
1892 int xe_vm_destroy_ioctl(struct drm_device *dev, void *data,
1893                         struct drm_file *file)
1894 {
1895         struct xe_device *xe = to_xe_device(dev);
1896         struct xe_file *xef = to_xe_file(file);
1897         struct drm_xe_vm_destroy *args = data;
1898         struct xe_vm *vm;
1899         int err = 0;
1900
1901         if (XE_IOCTL_DBG(xe, args->pad) ||
1902             XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1903                 return -EINVAL;
1904
1905         mutex_lock(&xef->vm.lock);
1906         vm = xa_load(&xef->vm.xa, args->vm_id);
1907         if (XE_IOCTL_DBG(xe, !vm))
1908                 err = -ENOENT;
1909         else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues))
1910                 err = -EBUSY;
1911         else
1912                 xa_erase(&xef->vm.xa, args->vm_id);
1913         mutex_unlock(&xef->vm.lock);
1914
1915         if (!err)
1916                 xe_vm_close_and_put(vm);
1917
1918         return err;
1919 }
1920
1921 static const u32 region_to_mem_type[] = {
1922         XE_PL_TT,
1923         XE_PL_VRAM0,
1924         XE_PL_VRAM1,
1925 };
1926
1927 static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
1928                           struct xe_exec_queue *q, u32 region,
1929                           struct xe_sync_entry *syncs, u32 num_syncs,
1930                           bool first_op, bool last_op)
1931 {
1932         struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1933         int err;
1934
1935         xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type));
1936
1937         if (!xe_vma_has_no_bo(vma)) {
1938                 err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
1939                 if (err)
1940                         return err;
1941         }
1942
1943         if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
1944                 return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
1945                                   true, first_op, last_op);
1946         } else {
1947                 int i;
1948
1949                 /* Nothing to do, signal fences now */
1950                 if (last_op) {
1951                         for (i = 0; i < num_syncs; i++) {
1952                                 struct dma_fence *fence =
1953                                         xe_exec_queue_last_fence_get(wait_exec_queue, vm);
1954
1955                                 xe_sync_entry_signal(&syncs[i], NULL, fence);
1956                         }
1957                 }
1958
1959                 return 0;
1960         }
1961 }
1962
1963 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma,
1964                              bool post_commit)
1965 {
1966         down_read(&vm->userptr.notifier_lock);
1967         vma->gpuva.flags |= XE_VMA_DESTROYED;
1968         up_read(&vm->userptr.notifier_lock);
1969         if (post_commit)
1970                 xe_vm_remove_vma(vm, vma);
1971 }
1972
1973 #undef ULL
1974 #define ULL     unsigned long long
1975
1976 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
1977 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
1978 {
1979         struct xe_vma *vma;
1980
1981         switch (op->op) {
1982         case DRM_GPUVA_OP_MAP:
1983                 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx",
1984                        (ULL)op->map.va.addr, (ULL)op->map.va.range);
1985                 break;
1986         case DRM_GPUVA_OP_REMAP:
1987                 vma = gpuva_to_vma(op->remap.unmap->va);
1988                 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
1989                        (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
1990                        op->remap.unmap->keep ? 1 : 0);
1991                 if (op->remap.prev)
1992                         vm_dbg(&xe->drm,
1993                                "REMAP:PREV: addr=0x%016llx, range=0x%016llx",
1994                                (ULL)op->remap.prev->va.addr,
1995                                (ULL)op->remap.prev->va.range);
1996                 if (op->remap.next)
1997                         vm_dbg(&xe->drm,
1998                                "REMAP:NEXT: addr=0x%016llx, range=0x%016llx",
1999                                (ULL)op->remap.next->va.addr,
2000                                (ULL)op->remap.next->va.range);
2001                 break;
2002         case DRM_GPUVA_OP_UNMAP:
2003                 vma = gpuva_to_vma(op->unmap.va);
2004                 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2005                        (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2006                        op->unmap.keep ? 1 : 0);
2007                 break;
2008         case DRM_GPUVA_OP_PREFETCH:
2009                 vma = gpuva_to_vma(op->prefetch.va);
2010                 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx",
2011                        (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma));
2012                 break;
2013         default:
2014                 drm_warn(&xe->drm, "NOT POSSIBLE");
2015         }
2016 }
2017 #else
2018 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
2019 {
2020 }
2021 #endif
2022
2023 /*
2024  * Create operations list from IOCTL arguments, setup operations fields so parse
2025  * and commit steps are decoupled from IOCTL arguments. This step can fail.
2026  */
2027 static struct drm_gpuva_ops *
2028 vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo,
2029                          u64 bo_offset_or_userptr, u64 addr, u64 range,
2030                          u32 operation, u32 flags,
2031                          u32 prefetch_region, u16 pat_index)
2032 {
2033         struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL;
2034         struct drm_gpuva_ops *ops;
2035         struct drm_gpuva_op *__op;
2036         struct xe_vma_op *op;
2037         struct drm_gpuvm_bo *vm_bo;
2038         int err;
2039
2040         lockdep_assert_held_write(&vm->lock);
2041
2042         vm_dbg(&vm->xe->drm,
2043                "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx",
2044                operation, (ULL)addr, (ULL)range,
2045                (ULL)bo_offset_or_userptr);
2046
2047         switch (operation) {
2048         case DRM_XE_VM_BIND_OP_MAP:
2049         case DRM_XE_VM_BIND_OP_MAP_USERPTR:
2050                 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, addr, range,
2051                                                   obj, bo_offset_or_userptr);
2052                 break;
2053         case DRM_XE_VM_BIND_OP_UNMAP:
2054                 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range);
2055                 break;
2056         case DRM_XE_VM_BIND_OP_PREFETCH:
2057                 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range);
2058                 break;
2059         case DRM_XE_VM_BIND_OP_UNMAP_ALL:
2060                 xe_assert(vm->xe, bo);
2061
2062                 err = xe_bo_lock(bo, true);
2063                 if (err)
2064                         return ERR_PTR(err);
2065
2066                 vm_bo = drm_gpuvm_bo_obtain(&vm->gpuvm, obj);
2067                 if (IS_ERR(vm_bo)) {
2068                         xe_bo_unlock(bo);
2069                         return ERR_CAST(vm_bo);
2070                 }
2071
2072                 ops = drm_gpuvm_bo_unmap_ops_create(vm_bo);
2073                 drm_gpuvm_bo_put(vm_bo);
2074                 xe_bo_unlock(bo);
2075                 break;
2076         default:
2077                 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2078                 ops = ERR_PTR(-EINVAL);
2079         }
2080         if (IS_ERR(ops))
2081                 return ops;
2082
2083 #ifdef TEST_VM_ASYNC_OPS_ERROR
2084         if (operation & FORCE_ASYNC_OP_ERROR) {
2085                 op = list_first_entry_or_null(&ops->list, struct xe_vma_op,
2086                                               base.entry);
2087                 if (op)
2088                         op->inject_error = true;
2089         }
2090 #endif
2091
2092         drm_gpuva_for_each_op(__op, ops) {
2093                 struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2094
2095                 if (__op->op == DRM_GPUVA_OP_MAP) {
2096                         op->map.immediate =
2097                                 flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE;
2098                         op->map.read_only =
2099                                 flags & DRM_XE_VM_BIND_FLAG_READONLY;
2100                         op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL;
2101                         op->map.pat_index = pat_index;
2102                 } else if (__op->op == DRM_GPUVA_OP_PREFETCH) {
2103                         op->prefetch.region = prefetch_region;
2104                 }
2105
2106                 print_op(vm->xe, __op);
2107         }
2108
2109         return ops;
2110 }
2111
2112 static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
2113                               u16 pat_index, unsigned int flags)
2114 {
2115         struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
2116         struct drm_exec exec;
2117         struct xe_vma *vma;
2118         int err;
2119
2120         lockdep_assert_held_write(&vm->lock);
2121
2122         if (bo) {
2123                 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
2124                 drm_exec_until_all_locked(&exec) {
2125                         err = 0;
2126                         if (!bo->vm) {
2127                                 err = drm_exec_lock_obj(&exec, xe_vm_obj(vm));
2128                                 drm_exec_retry_on_contention(&exec);
2129                         }
2130                         if (!err) {
2131                                 err = drm_exec_lock_obj(&exec, &bo->ttm.base);
2132                                 drm_exec_retry_on_contention(&exec);
2133                         }
2134                         if (err) {
2135                                 drm_exec_fini(&exec);
2136                                 return ERR_PTR(err);
2137                         }
2138                 }
2139         }
2140         vma = xe_vma_create(vm, bo, op->gem.offset,
2141                             op->va.addr, op->va.addr +
2142                             op->va.range - 1, pat_index, flags);
2143         if (bo)
2144                 drm_exec_fini(&exec);
2145
2146         if (xe_vma_is_userptr(vma)) {
2147                 err = xe_vma_userptr_pin_pages(vma);
2148                 if (err) {
2149                         prep_vma_destroy(vm, vma, false);
2150                         xe_vma_destroy_unlocked(vma);
2151                         return ERR_PTR(err);
2152                 }
2153         } else if (!xe_vma_has_no_bo(vma) && !bo->vm) {
2154                 err = add_preempt_fences(vm, bo);
2155                 if (err) {
2156                         prep_vma_destroy(vm, vma, false);
2157                         xe_vma_destroy_unlocked(vma);
2158                         return ERR_PTR(err);
2159                 }
2160         }
2161
2162         return vma;
2163 }
2164
2165 static u64 xe_vma_max_pte_size(struct xe_vma *vma)
2166 {
2167         if (vma->gpuva.flags & XE_VMA_PTE_1G)
2168                 return SZ_1G;
2169         else if (vma->gpuva.flags & XE_VMA_PTE_2M)
2170                 return SZ_2M;
2171
2172         return SZ_4K;
2173 }
2174
2175 static u64 xe_vma_set_pte_size(struct xe_vma *vma, u64 size)
2176 {
2177         switch (size) {
2178         case SZ_1G:
2179                 vma->gpuva.flags |= XE_VMA_PTE_1G;
2180                 break;
2181         case SZ_2M:
2182                 vma->gpuva.flags |= XE_VMA_PTE_2M;
2183                 break;
2184         }
2185
2186         return SZ_4K;
2187 }
2188
2189 static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
2190 {
2191         int err = 0;
2192
2193         lockdep_assert_held_write(&vm->lock);
2194
2195         switch (op->base.op) {
2196         case DRM_GPUVA_OP_MAP:
2197                 err |= xe_vm_insert_vma(vm, op->map.vma);
2198                 if (!err)
2199                         op->flags |= XE_VMA_OP_COMMITTED;
2200                 break;
2201         case DRM_GPUVA_OP_REMAP:
2202         {
2203                 u8 tile_present =
2204                         gpuva_to_vma(op->base.remap.unmap->va)->tile_present;
2205
2206                 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va),
2207                                  true);
2208                 op->flags |= XE_VMA_OP_COMMITTED;
2209
2210                 if (op->remap.prev) {
2211                         err |= xe_vm_insert_vma(vm, op->remap.prev);
2212                         if (!err)
2213                                 op->flags |= XE_VMA_OP_PREV_COMMITTED;
2214                         if (!err && op->remap.skip_prev) {
2215                                 op->remap.prev->tile_present =
2216                                         tile_present;
2217                                 op->remap.prev = NULL;
2218                         }
2219                 }
2220                 if (op->remap.next) {
2221                         err |= xe_vm_insert_vma(vm, op->remap.next);
2222                         if (!err)
2223                                 op->flags |= XE_VMA_OP_NEXT_COMMITTED;
2224                         if (!err && op->remap.skip_next) {
2225                                 op->remap.next->tile_present =
2226                                         tile_present;
2227                                 op->remap.next = NULL;
2228                         }
2229                 }
2230
2231                 /* Adjust for partial unbind after removin VMA from VM */
2232                 if (!err) {
2233                         op->base.remap.unmap->va->va.addr = op->remap.start;
2234                         op->base.remap.unmap->va->va.range = op->remap.range;
2235                 }
2236                 break;
2237         }
2238         case DRM_GPUVA_OP_UNMAP:
2239                 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true);
2240                 op->flags |= XE_VMA_OP_COMMITTED;
2241                 break;
2242         case DRM_GPUVA_OP_PREFETCH:
2243                 op->flags |= XE_VMA_OP_COMMITTED;
2244                 break;
2245         default:
2246                 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2247         }
2248
2249         return err;
2250 }
2251
2252
2253 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
2254                                    struct drm_gpuva_ops *ops,
2255                                    struct xe_sync_entry *syncs, u32 num_syncs,
2256                                    struct list_head *ops_list, bool last)
2257 {
2258         struct xe_vma_op *last_op = NULL;
2259         struct drm_gpuva_op *__op;
2260         int err = 0;
2261
2262         lockdep_assert_held_write(&vm->lock);
2263
2264         drm_gpuva_for_each_op(__op, ops) {
2265                 struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2266                 struct xe_vma *vma;
2267                 bool first = list_empty(ops_list);
2268                 unsigned int flags = 0;
2269
2270                 INIT_LIST_HEAD(&op->link);
2271                 list_add_tail(&op->link, ops_list);
2272
2273                 if (first) {
2274                         op->flags |= XE_VMA_OP_FIRST;
2275                         op->num_syncs = num_syncs;
2276                         op->syncs = syncs;
2277                 }
2278
2279                 op->q = q;
2280
2281                 switch (op->base.op) {
2282                 case DRM_GPUVA_OP_MAP:
2283                 {
2284                         flags |= op->map.read_only ?
2285                                 VMA_CREATE_FLAG_READ_ONLY : 0;
2286                         flags |= op->map.is_null ?
2287                                 VMA_CREATE_FLAG_IS_NULL : 0;
2288
2289                         vma = new_vma(vm, &op->base.map, op->map.pat_index,
2290                                       flags);
2291                         if (IS_ERR(vma))
2292                                 return PTR_ERR(vma);
2293
2294                         op->map.vma = vma;
2295                         break;
2296                 }
2297                 case DRM_GPUVA_OP_REMAP:
2298                 {
2299                         struct xe_vma *old =
2300                                 gpuva_to_vma(op->base.remap.unmap->va);
2301
2302                         op->remap.start = xe_vma_start(old);
2303                         op->remap.range = xe_vma_size(old);
2304
2305                         if (op->base.remap.prev) {
2306                                 flags |= op->base.remap.unmap->va->flags &
2307                                         XE_VMA_READ_ONLY ?
2308                                         VMA_CREATE_FLAG_READ_ONLY : 0;
2309                                 flags |= op->base.remap.unmap->va->flags &
2310                                         DRM_GPUVA_SPARSE ?
2311                                         VMA_CREATE_FLAG_IS_NULL : 0;
2312
2313                                 vma = new_vma(vm, op->base.remap.prev,
2314                                               old->pat_index, flags);
2315                                 if (IS_ERR(vma))
2316                                         return PTR_ERR(vma);
2317
2318                                 op->remap.prev = vma;
2319
2320                                 /*
2321                                  * Userptr creates a new SG mapping so
2322                                  * we must also rebind.
2323                                  */
2324                                 op->remap.skip_prev = !xe_vma_is_userptr(old) &&
2325                                         IS_ALIGNED(xe_vma_end(vma),
2326                                                    xe_vma_max_pte_size(old));
2327                                 if (op->remap.skip_prev) {
2328                                         xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old));
2329                                         op->remap.range -=
2330                                                 xe_vma_end(vma) -
2331                                                 xe_vma_start(old);
2332                                         op->remap.start = xe_vma_end(vma);
2333                                 }
2334                         }
2335
2336                         if (op->base.remap.next) {
2337                                 flags |= op->base.remap.unmap->va->flags &
2338                                         XE_VMA_READ_ONLY ?
2339                                         VMA_CREATE_FLAG_READ_ONLY : 0;
2340                                 flags |= op->base.remap.unmap->va->flags &
2341                                         DRM_GPUVA_SPARSE ?
2342                                         VMA_CREATE_FLAG_IS_NULL : 0;
2343
2344                                 vma = new_vma(vm, op->base.remap.next,
2345                                               old->pat_index, flags);
2346                                 if (IS_ERR(vma))
2347                                         return PTR_ERR(vma);
2348
2349                                 op->remap.next = vma;
2350
2351                                 /*
2352                                  * Userptr creates a new SG mapping so
2353                                  * we must also rebind.
2354                                  */
2355                                 op->remap.skip_next = !xe_vma_is_userptr(old) &&
2356                                         IS_ALIGNED(xe_vma_start(vma),
2357                                                    xe_vma_max_pte_size(old));
2358                                 if (op->remap.skip_next) {
2359                                         xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old));
2360                                         op->remap.range -=
2361                                                 xe_vma_end(old) -
2362                                                 xe_vma_start(vma);
2363                                 }
2364                         }
2365                         break;
2366                 }
2367                 case DRM_GPUVA_OP_UNMAP:
2368                 case DRM_GPUVA_OP_PREFETCH:
2369                         /* Nothing to do */
2370                         break;
2371                 default:
2372                         drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2373                 }
2374
2375                 last_op = op;
2376
2377                 err = xe_vma_op_commit(vm, op);
2378                 if (err)
2379                         return err;
2380         }
2381
2382         /* FIXME: Unhandled corner case */
2383         XE_WARN_ON(!last_op && last && !list_empty(ops_list));
2384
2385         if (!last_op)
2386                 return 0;
2387
2388         last_op->ops = ops;
2389         if (last) {
2390                 last_op->flags |= XE_VMA_OP_LAST;
2391                 last_op->num_syncs = num_syncs;
2392                 last_op->syncs = syncs;
2393         }
2394
2395         return 0;
2396 }
2397
2398 static int op_execute(struct drm_exec *exec, struct xe_vm *vm,
2399                       struct xe_vma *vma, struct xe_vma_op *op)
2400 {
2401         int err;
2402
2403         lockdep_assert_held_write(&vm->lock);
2404
2405         err = xe_vm_prepare_vma(exec, vma, 1);
2406         if (err)
2407                 return err;
2408
2409         xe_vm_assert_held(vm);
2410         xe_bo_assert_held(xe_vma_bo(vma));
2411
2412         switch (op->base.op) {
2413         case DRM_GPUVA_OP_MAP:
2414                 err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
2415                                  op->syncs, op->num_syncs,
2416                                  op->map.immediate || !xe_vm_in_fault_mode(vm),
2417                                  op->flags & XE_VMA_OP_FIRST,
2418                                  op->flags & XE_VMA_OP_LAST);
2419                 break;
2420         case DRM_GPUVA_OP_REMAP:
2421         {
2422                 bool prev = !!op->remap.prev;
2423                 bool next = !!op->remap.next;
2424
2425                 if (!op->remap.unmap_done) {
2426                         if (prev || next)
2427                                 vma->gpuva.flags |= XE_VMA_FIRST_REBIND;
2428                         err = xe_vm_unbind(vm, vma, op->q, op->syncs,
2429                                            op->num_syncs,
2430                                            op->flags & XE_VMA_OP_FIRST,
2431                                            op->flags & XE_VMA_OP_LAST &&
2432                                            !prev && !next);
2433                         if (err)
2434                                 break;
2435                         op->remap.unmap_done = true;
2436                 }
2437
2438                 if (prev) {
2439                         op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND;
2440                         err = xe_vm_bind(vm, op->remap.prev, op->q,
2441                                          xe_vma_bo(op->remap.prev), op->syncs,
2442                                          op->num_syncs, true, false,
2443                                          op->flags & XE_VMA_OP_LAST && !next);
2444                         op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
2445                         if (err)
2446                                 break;
2447                         op->remap.prev = NULL;
2448                 }
2449
2450                 if (next) {
2451                         op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND;
2452                         err = xe_vm_bind(vm, op->remap.next, op->q,
2453                                          xe_vma_bo(op->remap.next),
2454                                          op->syncs, op->num_syncs,
2455                                          true, false,
2456                                          op->flags & XE_VMA_OP_LAST);
2457                         op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
2458                         if (err)
2459                                 break;
2460                         op->remap.next = NULL;
2461                 }
2462
2463                 break;
2464         }
2465         case DRM_GPUVA_OP_UNMAP:
2466                 err = xe_vm_unbind(vm, vma, op->q, op->syncs,
2467                                    op->num_syncs, op->flags & XE_VMA_OP_FIRST,
2468                                    op->flags & XE_VMA_OP_LAST);
2469                 break;
2470         case DRM_GPUVA_OP_PREFETCH:
2471                 err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
2472                                      op->syncs, op->num_syncs,
2473                                      op->flags & XE_VMA_OP_FIRST,
2474                                      op->flags & XE_VMA_OP_LAST);
2475                 break;
2476         default:
2477                 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2478         }
2479
2480         if (err)
2481                 trace_xe_vma_fail(vma);
2482
2483         return err;
2484 }
2485
2486 static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
2487                                struct xe_vma_op *op)
2488 {
2489         struct drm_exec exec;
2490         int err;
2491
2492 retry_userptr:
2493         drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
2494         drm_exec_until_all_locked(&exec) {
2495                 err = op_execute(&exec, vm, vma, op);
2496                 drm_exec_retry_on_contention(&exec);
2497                 if (err)
2498                         break;
2499         }
2500         drm_exec_fini(&exec);
2501
2502         if (err == -EAGAIN && xe_vma_is_userptr(vma)) {
2503                 lockdep_assert_held_write(&vm->lock);
2504                 err = xe_vma_userptr_pin_pages(vma);
2505                 if (!err)
2506                         goto retry_userptr;
2507
2508                 trace_xe_vma_fail(vma);
2509         }
2510
2511         return err;
2512 }
2513
2514 static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
2515 {
2516         int ret = 0;
2517
2518         lockdep_assert_held_write(&vm->lock);
2519
2520 #ifdef TEST_VM_ASYNC_OPS_ERROR
2521         if (op->inject_error) {
2522                 op->inject_error = false;
2523                 return -ENOMEM;
2524         }
2525 #endif
2526
2527         switch (op->base.op) {
2528         case DRM_GPUVA_OP_MAP:
2529                 ret = __xe_vma_op_execute(vm, op->map.vma, op);
2530                 break;
2531         case DRM_GPUVA_OP_REMAP:
2532         {
2533                 struct xe_vma *vma;
2534
2535                 if (!op->remap.unmap_done)
2536                         vma = gpuva_to_vma(op->base.remap.unmap->va);
2537                 else if (op->remap.prev)
2538                         vma = op->remap.prev;
2539                 else
2540                         vma = op->remap.next;
2541
2542                 ret = __xe_vma_op_execute(vm, vma, op);
2543                 break;
2544         }
2545         case DRM_GPUVA_OP_UNMAP:
2546                 ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
2547                                           op);
2548                 break;
2549         case DRM_GPUVA_OP_PREFETCH:
2550                 ret = __xe_vma_op_execute(vm,
2551                                           gpuva_to_vma(op->base.prefetch.va),
2552                                           op);
2553                 break;
2554         default:
2555                 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2556         }
2557
2558         return ret;
2559 }
2560
2561 static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
2562 {
2563         bool last = op->flags & XE_VMA_OP_LAST;
2564
2565         if (last) {
2566                 while (op->num_syncs--)
2567                         xe_sync_entry_cleanup(&op->syncs[op->num_syncs]);
2568                 kfree(op->syncs);
2569                 if (op->q)
2570                         xe_exec_queue_put(op->q);
2571         }
2572         if (!list_empty(&op->link))
2573                 list_del(&op->link);
2574         if (op->ops)
2575                 drm_gpuva_ops_free(&vm->gpuvm, op->ops);
2576         if (last)
2577                 xe_vm_put(vm);
2578 }
2579
2580 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
2581                              bool post_commit, bool prev_post_commit,
2582                              bool next_post_commit)
2583 {
2584         lockdep_assert_held_write(&vm->lock);
2585
2586         switch (op->base.op) {
2587         case DRM_GPUVA_OP_MAP:
2588                 if (op->map.vma) {
2589                         prep_vma_destroy(vm, op->map.vma, post_commit);
2590                         xe_vma_destroy_unlocked(op->map.vma);
2591                 }
2592                 break;
2593         case DRM_GPUVA_OP_UNMAP:
2594         {
2595                 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
2596
2597                 if (vma) {
2598                         down_read(&vm->userptr.notifier_lock);
2599                         vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2600                         up_read(&vm->userptr.notifier_lock);
2601                         if (post_commit)
2602                                 xe_vm_insert_vma(vm, vma);
2603                 }
2604                 break;
2605         }
2606         case DRM_GPUVA_OP_REMAP:
2607         {
2608                 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va);
2609
2610                 if (op->remap.prev) {
2611                         prep_vma_destroy(vm, op->remap.prev, prev_post_commit);
2612                         xe_vma_destroy_unlocked(op->remap.prev);
2613                 }
2614                 if (op->remap.next) {
2615                         prep_vma_destroy(vm, op->remap.next, next_post_commit);
2616                         xe_vma_destroy_unlocked(op->remap.next);
2617                 }
2618                 if (vma) {
2619                         down_read(&vm->userptr.notifier_lock);
2620                         vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2621                         up_read(&vm->userptr.notifier_lock);
2622                         if (post_commit)
2623                                 xe_vm_insert_vma(vm, vma);
2624                 }
2625                 break;
2626         }
2627         case DRM_GPUVA_OP_PREFETCH:
2628                 /* Nothing to do */
2629                 break;
2630         default:
2631                 drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2632         }
2633 }
2634
2635 static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
2636                                      struct drm_gpuva_ops **ops,
2637                                      int num_ops_list)
2638 {
2639         int i;
2640
2641         for (i = num_ops_list - 1; i; ++i) {
2642                 struct drm_gpuva_ops *__ops = ops[i];
2643                 struct drm_gpuva_op *__op;
2644
2645                 if (!__ops)
2646                         continue;
2647
2648                 drm_gpuva_for_each_op_reverse(__op, __ops) {
2649                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2650
2651                         xe_vma_op_unwind(vm, op,
2652                                          op->flags & XE_VMA_OP_COMMITTED,
2653                                          op->flags & XE_VMA_OP_PREV_COMMITTED,
2654                                          op->flags & XE_VMA_OP_NEXT_COMMITTED);
2655                 }
2656
2657                 drm_gpuva_ops_free(&vm->gpuvm, __ops);
2658         }
2659 }
2660
2661 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
2662                                      struct list_head *ops_list)
2663 {
2664         struct xe_vma_op *op, *next;
2665         int err;
2666
2667         lockdep_assert_held_write(&vm->lock);
2668
2669         list_for_each_entry_safe(op, next, ops_list, link) {
2670                 err = xe_vma_op_execute(vm, op);
2671                 if (err) {
2672                         drm_warn(&vm->xe->drm, "VM op(%d) failed with %d",
2673                                  op->base.op, err);
2674                         /*
2675                          * FIXME: Killing VM rather than proper error handling
2676                          */
2677                         xe_vm_kill(vm);
2678                         return -ENOSPC;
2679                 }
2680                 xe_vma_op_cleanup(vm, op);
2681         }
2682
2683         return 0;
2684 }
2685
2686 #ifdef TEST_VM_ASYNC_OPS_ERROR
2687 #define SUPPORTED_FLAGS \
2688         (FORCE_ASYNC_OP_ERROR | DRM_XE_VM_BIND_FLAG_READONLY | \
2689          DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL | 0xffff)
2690 #else
2691 #define SUPPORTED_FLAGS \
2692         (DRM_XE_VM_BIND_FLAG_READONLY | \
2693          DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL | \
2694          0xffff)
2695 #endif
2696 #define XE_64K_PAGE_MASK 0xffffull
2697 #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)
2698
2699 #define MAX_BINDS       512     /* FIXME: Picking random upper limit */
2700
2701 static int vm_bind_ioctl_check_args(struct xe_device *xe,
2702                                     struct drm_xe_vm_bind *args,
2703                                     struct drm_xe_vm_bind_op **bind_ops)
2704 {
2705         int err;
2706         int i;
2707
2708         if (XE_IOCTL_DBG(xe, args->pad || args->pad2) ||
2709             XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
2710                 return -EINVAL;
2711
2712         if (XE_IOCTL_DBG(xe, args->extensions) ||
2713             XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
2714                 return -EINVAL;
2715
2716         if (args->num_binds > 1) {
2717                 u64 __user *bind_user =
2718                         u64_to_user_ptr(args->vector_of_binds);
2719
2720                 *bind_ops = kmalloc(sizeof(struct drm_xe_vm_bind_op) *
2721                                     args->num_binds, GFP_KERNEL);
2722                 if (!*bind_ops)
2723                         return -ENOMEM;
2724
2725                 err = __copy_from_user(*bind_ops, bind_user,
2726                                        sizeof(struct drm_xe_vm_bind_op) *
2727                                        args->num_binds);
2728                 if (XE_IOCTL_DBG(xe, err)) {
2729                         err = -EFAULT;
2730                         goto free_bind_ops;
2731                 }
2732         } else {
2733                 *bind_ops = &args->bind;
2734         }
2735
2736         for (i = 0; i < args->num_binds; ++i) {
2737                 u64 range = (*bind_ops)[i].range;
2738                 u64 addr = (*bind_ops)[i].addr;
2739                 u32 op = (*bind_ops)[i].op;
2740                 u32 flags = (*bind_ops)[i].flags;
2741                 u32 obj = (*bind_ops)[i].obj;
2742                 u64 obj_offset = (*bind_ops)[i].obj_offset;
2743                 u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance;
2744                 bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL;
2745                 u16 pat_index = (*bind_ops)[i].pat_index;
2746                 u16 coh_mode;
2747
2748                 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) {
2749                         err = -EINVAL;
2750                         goto free_bind_ops;
2751                 }
2752
2753                 pat_index = array_index_nospec(pat_index, xe->pat.n_entries);
2754                 (*bind_ops)[i].pat_index = pat_index;
2755                 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
2756                 if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */
2757                         err = -EINVAL;
2758                         goto free_bind_ops;
2759                 }
2760
2761                 if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) {
2762                         err = -EINVAL;
2763                         goto free_bind_ops;
2764                 }
2765
2766                 if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) ||
2767                     XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) ||
2768                     XE_IOCTL_DBG(xe, obj && is_null) ||
2769                     XE_IOCTL_DBG(xe, obj_offset && is_null) ||
2770                     XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP &&
2771                                  is_null) ||
2772                     XE_IOCTL_DBG(xe, !obj &&
2773                                  op == DRM_XE_VM_BIND_OP_MAP &&
2774                                  !is_null) ||
2775                     XE_IOCTL_DBG(xe, !obj &&
2776                                  op == DRM_XE_VM_BIND_OP_UNMAP_ALL) ||
2777                     XE_IOCTL_DBG(xe, addr &&
2778                                  op == DRM_XE_VM_BIND_OP_UNMAP_ALL) ||
2779                     XE_IOCTL_DBG(xe, range &&
2780                                  op == DRM_XE_VM_BIND_OP_UNMAP_ALL) ||
2781                     XE_IOCTL_DBG(xe, obj &&
2782                                  op == DRM_XE_VM_BIND_OP_MAP_USERPTR) ||
2783                     XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
2784                                  op == DRM_XE_VM_BIND_OP_MAP_USERPTR) ||
2785                     XE_IOCTL_DBG(xe, obj &&
2786                                  op == DRM_XE_VM_BIND_OP_PREFETCH) ||
2787                     XE_IOCTL_DBG(xe, prefetch_region &&
2788                                  op != DRM_XE_VM_BIND_OP_PREFETCH) ||
2789                     XE_IOCTL_DBG(xe, !(BIT(prefetch_region) &
2790                                        xe->info.mem_region_mask)) ||
2791                     XE_IOCTL_DBG(xe, obj &&
2792                                  op == DRM_XE_VM_BIND_OP_UNMAP)) {
2793                         err = -EINVAL;
2794                         goto free_bind_ops;
2795                 }
2796
2797                 if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) ||
2798                     XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) ||
2799                     XE_IOCTL_DBG(xe, range & ~PAGE_MASK) ||
2800                     XE_IOCTL_DBG(xe, !range &&
2801                                  op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) {
2802                         err = -EINVAL;
2803                         goto free_bind_ops;
2804                 }
2805         }
2806
2807         return 0;
2808
2809 free_bind_ops:
2810         if (args->num_binds > 1)
2811                 kfree(*bind_ops);
2812         return err;
2813 }
2814
2815 static int vm_bind_ioctl_signal_fences(struct xe_vm *vm,
2816                                        struct xe_exec_queue *q,
2817                                        struct xe_sync_entry *syncs,
2818                                        int num_syncs)
2819 {
2820         struct dma_fence *fence;
2821         int i, err = 0;
2822
2823         fence = xe_sync_in_fence_get(syncs, num_syncs,
2824                                      to_wait_exec_queue(vm, q), vm);
2825         if (IS_ERR(fence))
2826                 return PTR_ERR(fence);
2827
2828         for (i = 0; i < num_syncs; i++)
2829                 xe_sync_entry_signal(&syncs[i], NULL, fence);
2830
2831         xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm,
2832                                      fence);
2833         dma_fence_put(fence);
2834
2835         return err;
2836 }
2837
2838 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2839 {
2840         struct xe_device *xe = to_xe_device(dev);
2841         struct xe_file *xef = to_xe_file(file);
2842         struct drm_xe_vm_bind *args = data;
2843         struct drm_xe_sync __user *syncs_user;
2844         struct xe_bo **bos = NULL;
2845         struct drm_gpuva_ops **ops = NULL;
2846         struct xe_vm *vm;
2847         struct xe_exec_queue *q = NULL;
2848         u32 num_syncs;
2849         struct xe_sync_entry *syncs = NULL;
2850         struct drm_xe_vm_bind_op *bind_ops;
2851         LIST_HEAD(ops_list);
2852         int err;
2853         int i;
2854
2855         err = vm_bind_ioctl_check_args(xe, args, &bind_ops);
2856         if (err)
2857                 return err;
2858
2859         if (args->exec_queue_id) {
2860                 q = xe_exec_queue_lookup(xef, args->exec_queue_id);
2861                 if (XE_IOCTL_DBG(xe, !q)) {
2862                         err = -ENOENT;
2863                         goto free_objs;
2864                 }
2865
2866                 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) {
2867                         err = -EINVAL;
2868                         goto put_exec_queue;
2869                 }
2870         }
2871
2872         vm = xe_vm_lookup(xef, args->vm_id);
2873         if (XE_IOCTL_DBG(xe, !vm)) {
2874                 err = -EINVAL;
2875                 goto put_exec_queue;
2876         }
2877
2878         err = down_write_killable(&vm->lock);
2879         if (err)
2880                 goto put_vm;
2881
2882         if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
2883                 err = -ENOENT;
2884                 goto release_vm_lock;
2885         }
2886
2887         for (i = 0; i < args->num_binds; ++i) {
2888                 u64 range = bind_ops[i].range;
2889                 u64 addr = bind_ops[i].addr;
2890
2891                 if (XE_IOCTL_DBG(xe, range > vm->size) ||
2892                     XE_IOCTL_DBG(xe, addr > vm->size - range)) {
2893                         err = -EINVAL;
2894                         goto release_vm_lock;
2895                 }
2896         }
2897
2898         if (args->num_binds) {
2899                 bos = kcalloc(args->num_binds, sizeof(*bos), GFP_KERNEL);
2900                 if (!bos) {
2901                         err = -ENOMEM;
2902                         goto release_vm_lock;
2903                 }
2904
2905                 ops = kcalloc(args->num_binds, sizeof(*ops), GFP_KERNEL);
2906                 if (!ops) {
2907                         err = -ENOMEM;
2908                         goto release_vm_lock;
2909                 }
2910         }
2911
2912         for (i = 0; i < args->num_binds; ++i) {
2913                 struct drm_gem_object *gem_obj;
2914                 u64 range = bind_ops[i].range;
2915                 u64 addr = bind_ops[i].addr;
2916                 u32 obj = bind_ops[i].obj;
2917                 u64 obj_offset = bind_ops[i].obj_offset;
2918                 u16 pat_index = bind_ops[i].pat_index;
2919                 u16 coh_mode;
2920
2921                 if (!obj)
2922                         continue;
2923
2924                 gem_obj = drm_gem_object_lookup(file, obj);
2925                 if (XE_IOCTL_DBG(xe, !gem_obj)) {
2926                         err = -ENOENT;
2927                         goto put_obj;
2928                 }
2929                 bos[i] = gem_to_xe_bo(gem_obj);
2930
2931                 if (XE_IOCTL_DBG(xe, range > bos[i]->size) ||
2932                     XE_IOCTL_DBG(xe, obj_offset >
2933                                  bos[i]->size - range)) {
2934                         err = -EINVAL;
2935                         goto put_obj;
2936                 }
2937
2938                 if (bos[i]->flags & XE_BO_INTERNAL_64K) {
2939                         if (XE_IOCTL_DBG(xe, obj_offset &
2940                                          XE_64K_PAGE_MASK) ||
2941                             XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) ||
2942                             XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) {
2943                                 err = -EINVAL;
2944                                 goto put_obj;
2945                         }
2946                 }
2947
2948                 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
2949                 if (bos[i]->cpu_caching) {
2950                         if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
2951                                          bos[i]->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) {
2952                                 err = -EINVAL;
2953                                 goto put_obj;
2954                         }
2955                 } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) {
2956                         /*
2957                          * Imported dma-buf from a different device should
2958                          * require 1way or 2way coherency since we don't know
2959                          * how it was mapped on the CPU. Just assume is it
2960                          * potentially cached on CPU side.
2961                          */
2962                         err = -EINVAL;
2963                         goto put_obj;
2964                 }
2965         }
2966
2967         if (args->num_syncs) {
2968                 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL);
2969                 if (!syncs) {
2970                         err = -ENOMEM;
2971                         goto put_obj;
2972                 }
2973         }
2974
2975         syncs_user = u64_to_user_ptr(args->syncs);
2976         for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
2977                 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
2978                                           &syncs_user[num_syncs],
2979                                           (xe_vm_in_lr_mode(vm) ?
2980                                            SYNC_PARSE_FLAG_LR_MODE : 0) |
2981                                           (!args->num_binds ?
2982                                            SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0));
2983                 if (err)
2984                         goto free_syncs;
2985         }
2986
2987         if (!args->num_binds) {
2988                 err = -ENODATA;
2989                 goto free_syncs;
2990         }
2991
2992         for (i = 0; i < args->num_binds; ++i) {
2993                 u64 range = bind_ops[i].range;
2994                 u64 addr = bind_ops[i].addr;
2995                 u32 op = bind_ops[i].op;
2996                 u32 flags = bind_ops[i].flags;
2997                 u64 obj_offset = bind_ops[i].obj_offset;
2998                 u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance;
2999                 u16 pat_index = bind_ops[i].pat_index;
3000
3001                 ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset,
3002                                                   addr, range, op, flags,
3003                                                   prefetch_region, pat_index);
3004                 if (IS_ERR(ops[i])) {
3005                         err = PTR_ERR(ops[i]);
3006                         ops[i] = NULL;
3007                         goto unwind_ops;
3008                 }
3009
3010                 err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs,
3011                                               &ops_list,
3012                                               i == args->num_binds - 1);
3013                 if (err)
3014                         goto unwind_ops;
3015         }
3016
3017         /* Nothing to do */
3018         if (list_empty(&ops_list)) {
3019                 err = -ENODATA;
3020                 goto unwind_ops;
3021         }
3022
3023         xe_vm_get(vm);
3024         if (q)
3025                 xe_exec_queue_get(q);
3026
3027         err = vm_bind_ioctl_ops_execute(vm, &ops_list);
3028
3029         up_write(&vm->lock);
3030
3031         if (q)
3032                 xe_exec_queue_put(q);
3033         xe_vm_put(vm);
3034
3035         for (i = 0; bos && i < args->num_binds; ++i)
3036                 xe_bo_put(bos[i]);
3037
3038         kfree(bos);
3039         kfree(ops);
3040         if (args->num_binds > 1)
3041                 kfree(bind_ops);
3042
3043         return err;
3044
3045 unwind_ops:
3046         vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
3047 free_syncs:
3048         if (err == -ENODATA)
3049                 err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs);
3050         while (num_syncs--)
3051                 xe_sync_entry_cleanup(&syncs[num_syncs]);
3052
3053         kfree(syncs);
3054 put_obj:
3055         for (i = 0; i < args->num_binds; ++i)
3056                 xe_bo_put(bos[i]);
3057 release_vm_lock:
3058         up_write(&vm->lock);
3059 put_vm:
3060         xe_vm_put(vm);
3061 put_exec_queue:
3062         if (q)
3063                 xe_exec_queue_put(q);
3064 free_objs:
3065         kfree(bos);
3066         kfree(ops);
3067         if (args->num_binds > 1)
3068                 kfree(bind_ops);
3069         return err;
3070 }
3071
3072 /**
3073  * xe_vm_lock() - Lock the vm's dma_resv object
3074  * @vm: The struct xe_vm whose lock is to be locked
3075  * @intr: Whether to perform any wait interruptible
3076  *
3077  * Return: 0 on success, -EINTR if @intr is true and the wait for a
3078  * contended lock was interrupted. If @intr is false, the function
3079  * always returns 0.
3080  */
3081 int xe_vm_lock(struct xe_vm *vm, bool intr)
3082 {
3083         if (intr)
3084                 return dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
3085
3086         return dma_resv_lock(xe_vm_resv(vm), NULL);
3087 }
3088
3089 /**
3090  * xe_vm_unlock() - Unlock the vm's dma_resv object
3091  * @vm: The struct xe_vm whose lock is to be released.
3092  *
3093  * Unlock a buffer object lock that was locked by xe_vm_lock().
3094  */
3095 void xe_vm_unlock(struct xe_vm *vm)
3096 {
3097         dma_resv_unlock(xe_vm_resv(vm));
3098 }
3099
3100 /**
3101  * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
3102  * @vma: VMA to invalidate
3103  *
3104  * Walks a list of page tables leaves which it memset the entries owned by this
3105  * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
3106  * complete.
3107  *
3108  * Returns 0 for success, negative error code otherwise.
3109  */
3110 int xe_vm_invalidate_vma(struct xe_vma *vma)
3111 {
3112         struct xe_device *xe = xe_vma_vm(vma)->xe;
3113         struct xe_tile *tile;
3114         u32 tile_needs_invalidate = 0;
3115         int seqno[XE_MAX_TILES_PER_DEVICE];
3116         u8 id;
3117         int ret;
3118
3119         xe_assert(xe, xe_vm_in_fault_mode(xe_vma_vm(vma)));
3120         xe_assert(xe, !xe_vma_is_null(vma));
3121         trace_xe_vma_usm_invalidate(vma);
3122
3123         /* Check that we don't race with page-table updates */
3124         if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
3125                 if (xe_vma_is_userptr(vma)) {
3126                         WARN_ON_ONCE(!mmu_interval_check_retry
3127                                      (&vma->userptr.notifier,
3128                                       vma->userptr.notifier_seq));
3129                         WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(xe_vma_vm(vma)),
3130                                                              DMA_RESV_USAGE_BOOKKEEP));
3131
3132                 } else {
3133                         xe_bo_assert_held(xe_vma_bo(vma));
3134                 }
3135         }
3136
3137         for_each_tile(tile, xe, id) {
3138                 if (xe_pt_zap_ptes(tile, vma)) {
3139                         tile_needs_invalidate |= BIT(id);
3140                         xe_device_wmb(xe);
3141                         /*
3142                          * FIXME: We potentially need to invalidate multiple
3143                          * GTs within the tile
3144                          */
3145                         seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma);
3146                         if (seqno[id] < 0)
3147                                 return seqno[id];
3148                 }
3149         }
3150
3151         for_each_tile(tile, xe, id) {
3152                 if (tile_needs_invalidate & BIT(id)) {
3153                         ret = xe_gt_tlb_invalidation_wait(tile->primary_gt, seqno[id]);
3154                         if (ret < 0)
3155                                 return ret;
3156                 }
3157         }
3158
3159         vma->usm.tile_invalidated = vma->tile_mask;
3160
3161         return 0;
3162 }
3163
3164 int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id)
3165 {
3166         struct drm_gpuva *gpuva;
3167         bool is_vram;
3168         uint64_t addr;
3169
3170         if (!down_read_trylock(&vm->lock)) {
3171                 drm_printf(p, " Failed to acquire VM lock to dump capture");
3172                 return 0;
3173         }
3174         if (vm->pt_root[gt_id]) {
3175                 addr = xe_bo_addr(vm->pt_root[gt_id]->bo, 0, XE_PAGE_SIZE);
3176                 is_vram = xe_bo_is_vram(vm->pt_root[gt_id]->bo);
3177                 drm_printf(p, " VM root: A:0x%llx %s\n", addr,
3178                            is_vram ? "VRAM" : "SYS");
3179         }
3180
3181         drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
3182                 struct xe_vma *vma = gpuva_to_vma(gpuva);
3183                 bool is_userptr = xe_vma_is_userptr(vma);
3184                 bool is_null = xe_vma_is_null(vma);
3185
3186                 if (is_null) {
3187                         addr = 0;
3188                 } else if (is_userptr) {
3189                         struct xe_res_cursor cur;
3190
3191                         if (vma->userptr.sg) {
3192                                 xe_res_first_sg(vma->userptr.sg, 0, XE_PAGE_SIZE,
3193                                                 &cur);
3194                                 addr = xe_res_dma(&cur);
3195                         } else {
3196                                 addr = 0;
3197                         }
3198                 } else {
3199                         addr = __xe_bo_addr(xe_vma_bo(vma), 0, XE_PAGE_SIZE);
3200                         is_vram = xe_bo_is_vram(xe_vma_bo(vma));
3201                 }
3202                 drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n",
3203                            xe_vma_start(vma), xe_vma_end(vma) - 1,
3204                            xe_vma_size(vma),
3205                            addr, is_null ? "NULL" : is_userptr ? "USR" :
3206                            is_vram ? "VRAM" : "SYS");
3207         }
3208         up_read(&vm->lock);
3209
3210         return 0;
3211 }