drivers/gpu/drm/xe/xe_vm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include "xe_vm.h"
   7
   8 #include <linux/dma-fence-array.h>
   9
  10 #include <drm/drm_print.h>
  11 #include <drm/ttm/ttm_execbuf_util.h>
  12 #include <drm/ttm/ttm_tt.h>
  13 #include <drm/xe_drm.h>
  14 #include <linux/delay.h>
  15 #include <linux/kthread.h>
  16 #include <linux/mm.h>
  17 #include <linux/swap.h>
  18
  19 #include "xe_bo.h"
  20 #include "xe_device.h"
  21 #include "xe_engine.h"
  22 #include "xe_gt.h"
  23 #include "xe_gt_pagefault.h"
  24 #include "xe_gt_tlb_invalidation.h"
  25 #include "xe_migrate.h"
  26 #include "xe_pm.h"
  27 #include "xe_preempt_fence.h"
  28 #include "xe_pt.h"
  29 #include "xe_res_cursor.h"
  30 #include "xe_sync.h"
  31 #include "xe_trace.h"
  32
  33 #define TEST_VM_ASYNC_OPS_ERROR
  34
  35 /**
  36  * xe_vma_userptr_check_repin() - Advisory check for repin needed
  37  * @vma: The userptr vma
  38  *
  39  * Check if the userptr vma has been invalidated since last successful
  40  * repin. The check is advisory only and can the function can be called
  41  * without the vm->userptr.notifier_lock held. There is no guarantee that the
  42  * vma userptr will remain valid after a lockless check, so typically
  43  * the call needs to be followed by a proper check under the notifier_lock.
  44  *
  45  * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended.
  46  */
  47 int xe_vma_userptr_check_repin(struct xe_vma *vma)
  48 {
  49         return mmu_interval_check_retry(&vma->userptr.notifier,
  50                                         vma->userptr.notifier_seq) ?
  51                 -EAGAIN : 0;
  52 }
  53
  54 int xe_vma_userptr_pin_pages(struct xe_vma *vma)
  55 {
  56         struct xe_vm *vm = xe_vma_vm(vma);
  57         struct xe_device *xe = vm->xe;
  58         const unsigned long num_pages = xe_vma_size(vma) >> PAGE_SHIFT;
  59         struct page **pages;
  60         bool in_kthread = !current->mm;
  61         unsigned long notifier_seq;
  62         int pinned, ret, i;
  63         bool read_only = xe_vma_read_only(vma);
  64
  65         lockdep_assert_held(&vm->lock);
  66         XE_BUG_ON(!xe_vma_is_userptr(vma));
  67 retry:
  68         if (vma->gpuva.flags & XE_VMA_DESTROYED)
  69                 return 0;
  70
  71         notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier);
  72         if (notifier_seq == vma->userptr.notifier_seq)
  73                 return 0;
  74
  75         pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL);
  76         if (!pages)
  77                 return -ENOMEM;
  78
  79         if (vma->userptr.sg) {
  80                 dma_unmap_sgtable(xe->drm.dev,
  81                                   vma->userptr.sg,
  82                                   read_only ? DMA_TO_DEVICE :
  83                                   DMA_BIDIRECTIONAL, 0);
  84                 sg_free_table(vma->userptr.sg);
  85                 vma->userptr.sg = NULL;
  86         }
  87
  88         pinned = ret = 0;
  89         if (in_kthread) {
  90                 if (!mmget_not_zero(vma->userptr.notifier.mm)) {
  91                         ret = -EFAULT;
  92                         goto mm_closed;
  93                 }
  94                 kthread_use_mm(vma->userptr.notifier.mm);
  95         }
  96
  97         while (pinned < num_pages) {
  98                 ret = get_user_pages_fast(xe_vma_userptr(vma) +
  99                                           pinned * PAGE_SIZE,
 100                                           num_pages - pinned,
 101                                           read_only ? 0 : FOLL_WRITE,
 102                                           &pages[pinned]);
 103                 if (ret < 0) {
 104                         if (in_kthread)
 105                                 ret = 0;
 106                         break;
 107                 }
 108
 109                 pinned += ret;
 110                 ret = 0;
 111         }
 112
 113         if (in_kthread) {
 114                 kthread_unuse_mm(vma->userptr.notifier.mm);
 115                 mmput(vma->userptr.notifier.mm);
 116         }
 117 mm_closed:
 118         if (ret)
 119                 goto out;
 120
 121         ret = sg_alloc_table_from_pages_segment(&vma->userptr.sgt, pages,
 122                                                 pinned, 0,
 123                                                 (u64)pinned << PAGE_SHIFT,
 124                                                 xe_sg_segment_size(xe->drm.dev),
 125                                                 GFP_KERNEL);
 126         if (ret) {
 127                 vma->userptr.sg = NULL;
 128                 goto out;
 129         }
 130         vma->userptr.sg = &vma->userptr.sgt;
 131
 132         ret = dma_map_sgtable(xe->drm.dev, vma->userptr.sg,
 133                               read_only ? DMA_TO_DEVICE :
 134                               DMA_BIDIRECTIONAL,
 135                               DMA_ATTR_SKIP_CPU_SYNC |
 136                               DMA_ATTR_NO_KERNEL_MAPPING);
 137         if (ret) {
 138                 sg_free_table(vma->userptr.sg);
 139                 vma->userptr.sg = NULL;
 140                 goto out;
 141         }
 142
 143         for (i = 0; i < pinned; ++i) {
 144                 if (!read_only) {
 145                         lock_page(pages[i]);
 146                         set_page_dirty(pages[i]);
 147                         unlock_page(pages[i]);
 148                 }
 149
 150                 mark_page_accessed(pages[i]);
 151         }
 152
 153 out:
 154         release_pages(pages, pinned);
 155         kvfree(pages);
 156
 157         if (!(ret < 0)) {
 158                 vma->userptr.notifier_seq = notifier_seq;
 159                 if (xe_vma_userptr_check_repin(vma) == -EAGAIN)
 160                         goto retry;
 161         }
 162
 163         return ret < 0 ? ret : 0;
 164 }
 165
 166 static bool preempt_fences_waiting(struct xe_vm *vm)
 167 {
 168         struct xe_engine *e;
 169
 170         lockdep_assert_held(&vm->lock);
 171         xe_vm_assert_held(vm);
 172
 173         list_for_each_entry(e, &vm->preempt.engines, compute.link) {
 174                 if (!e->compute.pfence || (e->compute.pfence &&
 175                     test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 176                              &e->compute.pfence->flags))) {
 177                         return true;
 178                 }
 179         }
 180
 181         return false;
 182 }
 183
 184 static void free_preempt_fences(struct list_head *list)
 185 {
 186         struct list_head *link, *next;
 187
 188         list_for_each_safe(link, next, list)
 189                 xe_preempt_fence_free(to_preempt_fence_from_link(link));
 190 }
 191
 192 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list,
 193                                 unsigned int *count)
 194 {
 195         lockdep_assert_held(&vm->lock);
 196         xe_vm_assert_held(vm);
 197
 198         if (*count >= vm->preempt.num_engines)
 199                 return 0;
 200
 201         for (; *count < vm->preempt.num_engines; ++(*count)) {
 202                 struct xe_preempt_fence *pfence = xe_preempt_fence_alloc();
 203
 204                 if (IS_ERR(pfence))
 205                         return PTR_ERR(pfence);
 206
 207                 list_move_tail(xe_preempt_fence_link(pfence), list);
 208         }
 209
 210         return 0;
 211 }
 212
 213 static int wait_for_existing_preempt_fences(struct xe_vm *vm)
 214 {
 215         struct xe_engine *e;
 216
 217         xe_vm_assert_held(vm);
 218
 219         list_for_each_entry(e, &vm->preempt.engines, compute.link) {
 220                 if (e->compute.pfence) {
 221                         long timeout = dma_fence_wait(e->compute.pfence, false);
 222
 223                         if (timeout < 0)
 224                                 return -ETIME;
 225                         dma_fence_put(e->compute.pfence);
 226                         e->compute.pfence = NULL;
 227                 }
 228         }
 229
 230         return 0;
 231 }
 232
 233 static bool xe_vm_is_idle(struct xe_vm *vm)
 234 {
 235         struct xe_engine *e;
 236
 237         xe_vm_assert_held(vm);
 238         list_for_each_entry(e, &vm->preempt.engines, compute.link) {
 239                 if (!xe_engine_is_idle(e))
 240                         return false;
 241         }
 242
 243         return true;
 244 }
 245
 246 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list)
 247 {
 248         struct list_head *link;
 249         struct xe_engine *e;
 250
 251         list_for_each_entry(e, &vm->preempt.engines, compute.link) {
 252                 struct dma_fence *fence;
 253
 254                 link = list->next;
 255                 XE_BUG_ON(link == list);
 256
 257                 fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link),
 258                                              e, e->compute.context,
 259                                              ++e->compute.seqno);
 260                 dma_fence_put(e->compute.pfence);
 261                 e->compute.pfence = fence;
 262         }
 263 }
 264
 265 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo)
 266 {
 267         struct xe_engine *e;
 268         struct ww_acquire_ctx ww;
 269         int err;
 270
 271         err = xe_bo_lock(bo, &ww, vm->preempt.num_engines, true);
 272         if (err)
 273                 return err;
 274
 275         list_for_each_entry(e, &vm->preempt.engines, compute.link)
 276                 if (e->compute.pfence) {
 277                         dma_resv_add_fence(bo->ttm.base.resv,
 278                                            e->compute.pfence,
 279                                            DMA_RESV_USAGE_BOOKKEEP);
 280                 }
 281
 282         xe_bo_unlock(bo, &ww);
 283         return 0;
 284 }
 285
 286 /**
 287  * xe_vm_fence_all_extobjs() - Add a fence to vm's external objects' resv
 288  * @vm: The vm.
 289  * @fence: The fence to add.
 290  * @usage: The resv usage for the fence.
 291  *
 292  * Loops over all of the vm's external object bindings and adds a @fence
 293  * with the given @usage to all of the external object's reservation
 294  * objects.
 295  */
 296 void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence,
 297                              enum dma_resv_usage usage)
 298 {
 299         struct xe_vma *vma;
 300
 301         list_for_each_entry(vma, &vm->extobj.list, extobj.link)
 302                 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, usage);
 303 }
 304
 305 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm)
 306 {
 307         struct xe_engine *e;
 308
 309         lockdep_assert_held(&vm->lock);
 310         xe_vm_assert_held(vm);
 311
 312         list_for_each_entry(e, &vm->preempt.engines, compute.link) {
 313                 e->ops->resume(e);
 314
 315                 dma_resv_add_fence(xe_vm_resv(vm), e->compute.pfence,
 316                                    DMA_RESV_USAGE_BOOKKEEP);
 317                 xe_vm_fence_all_extobjs(vm, e->compute.pfence,
 318                                         DMA_RESV_USAGE_BOOKKEEP);
 319         }
 320 }
 321
 322 int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e)
 323 {
 324         struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV];
 325         struct ttm_validate_buffer *tv;
 326         struct ww_acquire_ctx ww;
 327         struct list_head objs;
 328         struct dma_fence *pfence;
 329         int err;
 330         bool wait;
 331
 332         XE_BUG_ON(!xe_vm_in_compute_mode(vm));
 333
 334         down_write(&vm->lock);
 335
 336         err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs, true, 1);
 337         if (err)
 338                 goto out_unlock_outer;
 339
 340         pfence = xe_preempt_fence_create(e, e->compute.context,
 341                                          ++e->compute.seqno);
 342         if (!pfence) {
 343                 err = -ENOMEM;
 344                 goto out_unlock;
 345         }
 346
 347         list_add(&e->compute.link, &vm->preempt.engines);
 348         ++vm->preempt.num_engines;
 349         e->compute.pfence = pfence;
 350
 351         down_read(&vm->userptr.notifier_lock);
 352
 353         dma_resv_add_fence(xe_vm_resv(vm), pfence,
 354                            DMA_RESV_USAGE_BOOKKEEP);
 355
 356         xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP);
 357
 358         /*
 359          * Check to see if a preemption on VM is in flight or userptr
 360          * invalidation, if so trigger this preempt fence to sync state with
 361          * other preempt fences on the VM.
 362          */
 363         wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm);
 364         if (wait)
 365                 dma_fence_enable_sw_signaling(pfence);
 366
 367         up_read(&vm->userptr.notifier_lock);
 368
 369 out_unlock:
 370         xe_vm_unlock_dma_resv(vm, tv_onstack, tv, &ww, &objs);
 371 out_unlock_outer:
 372         up_write(&vm->lock);
 373
 374         return err;
 375 }
 376
 377 /**
 378  * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs
 379  * that need repinning.
 380  * @vm: The VM.
 381  *
 382  * This function checks for whether the VM has userptrs that need repinning,
 383  * and provides a release-type barrier on the userptr.notifier_lock after
 384  * checking.
 385  *
 386  * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are.
 387  */
 388 int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
 389 {
 390         lockdep_assert_held_read(&vm->userptr.notifier_lock);
 391
 392         return (list_empty(&vm->userptr.repin_list) &&
 393                 list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
 394 }
 395
 396 /**
 397  * xe_vm_lock_dma_resv() - Lock the vm dma_resv object and the dma_resv
 398  * objects of the vm's external buffer objects.
 399  * @vm: The vm.
 400  * @ww: Pointer to a struct ww_acquire_ctx locking context.
 401  * @tv_onstack: Array size XE_ONSTACK_TV of storage for the struct
 402  * ttm_validate_buffers used for locking.
 403  * @tv: Pointer to a pointer that on output contains the actual storage used.
 404  * @objs: List head for the buffer objects locked.
 405  * @intr: Whether to lock interruptible.
 406  * @num_shared: Number of dma-fence slots to reserve in the locked objects.
 407  *
 408  * Locks the vm dma-resv objects and all the dma-resv objects of the
 409  * buffer objects on the vm external object list. The TTM utilities require
 410  * a list of struct ttm_validate_buffers pointing to the actual buffer
 411  * objects to lock. Storage for those struct ttm_validate_buffers should
 412  * be provided in @tv_onstack, and is typically reserved on the stack
 413  * of the caller. If the size of @tv_onstack isn't sufficient, then
 414  * storage will be allocated internally using kvmalloc().
 415  *
 416  * The function performs deadlock handling internally, and after a
 417  * successful return the ww locking transaction should be considered
 418  * sealed.
 419  *
 420  * Return: 0 on success, Negative error code on error. In particular if
 421  * @intr is set to true, -EINTR or -ERESTARTSYS may be returned. In case
 422  * of error, any locking performed has been reverted.
 423  */
 424 int xe_vm_lock_dma_resv(struct xe_vm *vm, struct ww_acquire_ctx *ww,
 425                         struct ttm_validate_buffer *tv_onstack,
 426                         struct ttm_validate_buffer **tv,
 427                         struct list_head *objs,
 428                         bool intr,
 429                         unsigned int num_shared)
 430 {
 431         struct ttm_validate_buffer *tv_vm, *tv_bo;
 432         struct xe_vma *vma, *next;
 433         LIST_HEAD(dups);
 434         int err;
 435
 436         lockdep_assert_held(&vm->lock);
 437
 438         if (vm->extobj.entries < XE_ONSTACK_TV) {
 439                 tv_vm = tv_onstack;
 440         } else {
 441                 tv_vm = kvmalloc_array(vm->extobj.entries + 1, sizeof(*tv_vm),
 442                                        GFP_KERNEL);
 443                 if (!tv_vm)
 444                         return -ENOMEM;
 445         }
 446         tv_bo = tv_vm + 1;
 447
 448         INIT_LIST_HEAD(objs);
 449         list_for_each_entry(vma, &vm->extobj.list, extobj.link) {
 450                 tv_bo->num_shared = num_shared;
 451                 tv_bo->bo = &xe_vma_bo(vma)->ttm;
 452
 453                 list_add_tail(&tv_bo->head, objs);
 454                 tv_bo++;
 455         }
 456         tv_vm->num_shared = num_shared;
 457         tv_vm->bo = xe_vm_ttm_bo(vm);
 458         list_add_tail(&tv_vm->head, objs);
 459         err = ttm_eu_reserve_buffers(ww, objs, intr, &dups);
 460         if (err)
 461                 goto out_err;
 462
 463         spin_lock(&vm->notifier.list_lock);
 464         list_for_each_entry_safe(vma, next, &vm->notifier.rebind_list,
 465                                  notifier.rebind_link) {
 466                 xe_bo_assert_held(xe_vma_bo(vma));
 467
 468                 list_del_init(&vma->notifier.rebind_link);
 469                 if (vma->tile_present && !(vma->gpuva.flags & XE_VMA_DESTROYED))
 470                         list_move_tail(&vma->combined_links.rebind,
 471                                        &vm->rebind_list);
 472         }
 473         spin_unlock(&vm->notifier.list_lock);
 474
 475         *tv = tv_vm;
 476         return 0;
 477
 478 out_err:
 479         if (tv_vm != tv_onstack)
 480                 kvfree(tv_vm);
 481
 482         return err;
 483 }
 484
 485 /**
 486  * xe_vm_unlock_dma_resv() - Unlock reservation objects locked by
 487  * xe_vm_lock_dma_resv()
 488  * @vm: The vm.
 489  * @tv_onstack: The @tv_onstack array given to xe_vm_lock_dma_resv().
 490  * @tv: The value of *@tv given by xe_vm_lock_dma_resv().
 491  * @ww: The ww_acquire_context used for locking.
 492  * @objs: The list returned from xe_vm_lock_dma_resv().
 493  *
 494  * Unlocks the reservation objects and frees any memory allocated by
 495  * xe_vm_lock_dma_resv().
 496  */
 497 void xe_vm_unlock_dma_resv(struct xe_vm *vm,
 498                            struct ttm_validate_buffer *tv_onstack,
 499                            struct ttm_validate_buffer *tv,
 500                            struct ww_acquire_ctx *ww,
 501                            struct list_head *objs)
 502 {
 503         /*
 504          * Nothing should've been able to enter the list while we were locked,
 505          * since we've held the dma-resvs of all the vm's external objects,
 506          * and holding the dma_resv of an object is required for list
 507          * addition, and we shouldn't add ourselves.
 508          */
 509         XE_WARN_ON(!list_empty(&vm->notifier.rebind_list));
 510
 511         ttm_eu_backoff_reservation(ww, objs);
 512         if (tv && tv != tv_onstack)
 513                 kvfree(tv);
 514 }
 515
 516 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
 517
 518 static void xe_vm_kill(struct xe_vm *vm)
 519 {
 520         struct ww_acquire_ctx ww;
 521         struct xe_engine *e;
 522
 523         lockdep_assert_held(&vm->lock);
 524
 525         xe_vm_lock(vm, &ww, 0, false);
 526         vm->flags |= XE_VM_FLAG_BANNED;
 527         trace_xe_vm_kill(vm);
 528
 529         list_for_each_entry(e, &vm->preempt.engines, compute.link)
 530                 e->ops->kill(e);
 531         xe_vm_unlock(vm, &ww);
 532
 533         /* TODO: Inform user the VM is banned */
 534 }
 535
 536 static void preempt_rebind_work_func(struct work_struct *w)
 537 {
 538         struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work);
 539         struct xe_vma *vma;
 540         struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV];
 541         struct ttm_validate_buffer *tv;
 542         struct ww_acquire_ctx ww;
 543         struct list_head objs;
 544         struct dma_fence *rebind_fence;
 545         unsigned int fence_count = 0;
 546         LIST_HEAD(preempt_fences);
 547         ktime_t end = 0;
 548         int err;
 549         long wait;
 550         int __maybe_unused tries = 0;
 551
 552         XE_BUG_ON(!xe_vm_in_compute_mode(vm));
 553         trace_xe_vm_rebind_worker_enter(vm);
 554
 555         down_write(&vm->lock);
 556
 557         if (xe_vm_is_closed_or_banned(vm)) {
 558                 up_write(&vm->lock);
 559                 trace_xe_vm_rebind_worker_exit(vm);
 560                 return;
 561         }
 562
 563 retry:
 564         if (vm->async_ops.error)
 565                 goto out_unlock_outer;
 566
 567         /*
 568          * Extreme corner where we exit a VM error state with a munmap style VM
 569          * unbind inflight which requires a rebind. In this case the rebind
 570          * needs to install some fences into the dma-resv slots. The worker to
 571          * do this queued, let that worker make progress by dropping vm->lock
 572          * and trying this again.
 573          */
 574         if (vm->async_ops.munmap_rebind_inflight) {
 575                 up_write(&vm->lock);
 576                 flush_work(&vm->async_ops.work);
 577                 goto retry;
 578         }
 579
 580         if (xe_vm_userptr_check_repin(vm)) {
 581                 err = xe_vm_userptr_pin(vm);
 582                 if (err)
 583                         goto out_unlock_outer;
 584         }
 585
 586         err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs,
 587                                   false, vm->preempt.num_engines);
 588         if (err)
 589                 goto out_unlock_outer;
 590
 591         if (xe_vm_is_idle(vm)) {
 592                 vm->preempt.rebind_deactivated = true;
 593                 goto out_unlock;
 594         }
 595
 596         /* Fresh preempt fences already installed. Everyting is running. */
 597         if (!preempt_fences_waiting(vm))
 598                 goto out_unlock;
 599
 600         /*
 601          * This makes sure vm is completely suspended and also balances
 602          * xe_engine suspend- and resume; we resume *all* vm engines below.
 603          */
 604         err = wait_for_existing_preempt_fences(vm);
 605         if (err)
 606                 goto out_unlock;
 607
 608         err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
 609         if (err)
 610                 goto out_unlock;
 611
 612         list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
 613                 if (xe_vma_has_no_bo(vma) ||
 614                     vma->gpuva.flags & XE_VMA_DESTROYED)
 615                         continue;
 616
 617                 err = xe_bo_validate(xe_vma_bo(vma), vm, false);
 618                 if (err)
 619                         goto out_unlock;
 620         }
 621
 622         rebind_fence = xe_vm_rebind(vm, true);
 623         if (IS_ERR(rebind_fence)) {
 624                 err = PTR_ERR(rebind_fence);
 625                 goto out_unlock;
 626         }
 627
 628         if (rebind_fence) {
 629                 dma_fence_wait(rebind_fence, false);
 630                 dma_fence_put(rebind_fence);
 631         }
 632
 633         /* Wait on munmap style VM unbinds */
 634         wait = dma_resv_wait_timeout(xe_vm_resv(vm),
 635                                      DMA_RESV_USAGE_KERNEL,
 636                                      false, MAX_SCHEDULE_TIMEOUT);
 637         if (wait <= 0) {
 638                 err = -ETIME;
 639                 goto out_unlock;
 640         }
 641
 642 #define retry_required(__tries, __vm) \
 643         (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \
 644         (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \
 645         __xe_vm_userptr_needs_repin(__vm))
 646
 647         down_read(&vm->userptr.notifier_lock);
 648         if (retry_required(tries, vm)) {
 649                 up_read(&vm->userptr.notifier_lock);
 650                 err = -EAGAIN;
 651                 goto out_unlock;
 652         }
 653
 654 #undef retry_required
 655
 656         spin_lock(&vm->xe->ttm.lru_lock);
 657         ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
 658         spin_unlock(&vm->xe->ttm.lru_lock);
 659
 660         /* Point of no return. */
 661         arm_preempt_fences(vm, &preempt_fences);
 662         resume_and_reinstall_preempt_fences(vm);
 663         up_read(&vm->userptr.notifier_lock);
 664
 665 out_unlock:
 666         xe_vm_unlock_dma_resv(vm, tv_onstack, tv, &ww, &objs);
 667 out_unlock_outer:
 668         if (err == -EAGAIN) {
 669                 trace_xe_vm_rebind_worker_retry(vm);
 670                 goto retry;
 671         }
 672
 673         /*
 674          * With multiple active VMs, under memory pressure, it is possible that
 675          * ttm_bo_validate() run into -EDEADLK and in such case returns -ENOMEM.
 676          * Until ttm properly handles locking in such scenarios, best thing the
 677          * driver can do is retry with a timeout. Killing the VM or putting it
 678          * in error state after timeout or other error scenarios is still TBD.
 679          */
 680         if (err == -ENOMEM) {
 681                 ktime_t cur = ktime_get();
 682
 683                 end = end ? : ktime_add_ms(cur, XE_VM_REBIND_RETRY_TIMEOUT_MS);
 684                 if (ktime_before(cur, end)) {
 685                         msleep(20);
 686                         trace_xe_vm_rebind_worker_retry(vm);
 687                         goto retry;
 688                 }
 689         }
 690         if (err) {
 691                 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
 692                 xe_vm_kill(vm);
 693         }
 694         up_write(&vm->lock);
 695
 696         free_preempt_fences(&preempt_fences);
 697
 698         trace_xe_vm_rebind_worker_exit(vm);
 699 }
 700
 701 static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
 702                                    const struct mmu_notifier_range *range,
 703                                    unsigned long cur_seq)
 704 {
 705         struct xe_vma *vma = container_of(mni, struct xe_vma, userptr.notifier);
 706         struct xe_vm *vm = xe_vma_vm(vma);
 707         struct dma_resv_iter cursor;
 708         struct dma_fence *fence;
 709         long err;
 710
 711         XE_BUG_ON(!xe_vma_is_userptr(vma));
 712         trace_xe_vma_userptr_invalidate(vma);
 713
 714         if (!mmu_notifier_range_blockable(range))
 715                 return false;
 716
 717         down_write(&vm->userptr.notifier_lock);
 718         mmu_interval_set_seq(mni, cur_seq);
 719
 720         /* No need to stop gpu access if the userptr is not yet bound. */
 721         if (!vma->userptr.initial_bind) {
 722                 up_write(&vm->userptr.notifier_lock);
 723                 return true;
 724         }
 725
 726         /*
 727          * Tell exec and rebind worker they need to repin and rebind this
 728          * userptr.
 729          */
 730         if (!xe_vm_in_fault_mode(vm) &&
 731             !(vma->gpuva.flags & XE_VMA_DESTROYED) && vma->tile_present) {
 732                 spin_lock(&vm->userptr.invalidated_lock);
 733                 list_move_tail(&vma->userptr.invalidate_link,
 734                                &vm->userptr.invalidated);
 735                 spin_unlock(&vm->userptr.invalidated_lock);
 736         }
 737
 738         up_write(&vm->userptr.notifier_lock);
 739
 740         /*
 741          * Preempt fences turn into schedule disables, pipeline these.
 742          * Note that even in fault mode, we need to wait for binds and
 743          * unbinds to complete, and those are attached as BOOKMARK fences
 744          * to the vm.
 745          */
 746         dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
 747                             DMA_RESV_USAGE_BOOKKEEP);
 748         dma_resv_for_each_fence_unlocked(&cursor, fence)
 749                 dma_fence_enable_sw_signaling(fence);
 750         dma_resv_iter_end(&cursor);
 751
 752         err = dma_resv_wait_timeout(xe_vm_resv(vm),
 753                                     DMA_RESV_USAGE_BOOKKEEP,
 754                                     false, MAX_SCHEDULE_TIMEOUT);
 755         XE_WARN_ON(err <= 0);
 756
 757         if (xe_vm_in_fault_mode(vm)) {
 758                 err = xe_vm_invalidate_vma(vma);
 759                 XE_WARN_ON(err);
 760         }
 761
 762         trace_xe_vma_userptr_invalidate_complete(vma);
 763
 764         return true;
 765 }
 766
 767 static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
 768         .invalidate = vma_userptr_invalidate,
 769 };
 770
 771 int xe_vm_userptr_pin(struct xe_vm *vm)
 772 {
 773         struct xe_vma *vma, *next;
 774         int err = 0;
 775         LIST_HEAD(tmp_evict);
 776
 777         lockdep_assert_held_write(&vm->lock);
 778
 779         /* Collect invalidated userptrs */
 780         spin_lock(&vm->userptr.invalidated_lock);
 781         list_for_each_entry_safe(vma, next, &vm->userptr.invalidated,
 782                                  userptr.invalidate_link) {
 783                 list_del_init(&vma->userptr.invalidate_link);
 784                 if (list_empty(&vma->combined_links.userptr))
 785                         list_move_tail(&vma->combined_links.userptr,
 786                                        &vm->userptr.repin_list);
 787         }
 788         spin_unlock(&vm->userptr.invalidated_lock);
 789
 790         /* Pin and move to temporary list */
 791         list_for_each_entry_safe(vma, next, &vm->userptr.repin_list,
 792                                  combined_links.userptr) {
 793                 err = xe_vma_userptr_pin_pages(vma);
 794                 if (err < 0)
 795                         goto out_err;
 796
 797                 list_move_tail(&vma->combined_links.userptr, &tmp_evict);
 798         }
 799
 800         /* Take lock and move to rebind_list for rebinding. */
 801         err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
 802         if (err)
 803                 goto out_err;
 804
 805         list_for_each_entry_safe(vma, next, &tmp_evict, combined_links.userptr)
 806                 list_move_tail(&vma->combined_links.rebind, &vm->rebind_list);
 807
 808         dma_resv_unlock(xe_vm_resv(vm));
 809
 810         return 0;
 811
 812 out_err:
 813         list_splice_tail(&tmp_evict, &vm->userptr.repin_list);
 814
 815         return err;
 816 }
 817
 818 /**
 819  * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs
 820  * that need repinning.
 821  * @vm: The VM.
 822  *
 823  * This function does an advisory check for whether the VM has userptrs that
 824  * need repinning.
 825  *
 826  * Return: 0 if there are no indications of userptrs needing repinning,
 827  * -EAGAIN if there are.
 828  */
 829 int xe_vm_userptr_check_repin(struct xe_vm *vm)
 830 {
 831         return (list_empty_careful(&vm->userptr.repin_list) &&
 832                 list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
 833 }
 834
 835 static struct dma_fence *
 836 xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e,
 837                struct xe_sync_entry *syncs, u32 num_syncs,
 838                bool first_op, bool last_op);
 839
 840 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 841 {
 842         struct dma_fence *fence = NULL;
 843         struct xe_vma *vma, *next;
 844
 845         lockdep_assert_held(&vm->lock);
 846         if (xe_vm_no_dma_fences(vm) && !rebind_worker)
 847                 return NULL;
 848
 849         xe_vm_assert_held(vm);
 850         list_for_each_entry_safe(vma, next, &vm->rebind_list,
 851                                  combined_links.rebind) {
 852                 XE_WARN_ON(!vma->tile_present);
 853
 854                 list_del_init(&vma->combined_links.rebind);
 855                 dma_fence_put(fence);
 856                 if (rebind_worker)
 857                         trace_xe_vma_rebind_worker(vma);
 858                 else
 859                         trace_xe_vma_rebind_exec(vma);
 860                 fence = xe_vm_bind_vma(vma, NULL, NULL, 0, false, false);
 861                 if (IS_ERR(fence))
 862                         return fence;
 863         }
 864
 865         return fence;
 866 }
 867
 868 static struct xe_vma *xe_vma_create(struct xe_vm *vm,
 869                                     struct xe_bo *bo,
 870                                     u64 bo_offset_or_userptr,
 871                                     u64 start, u64 end,
 872                                     bool read_only,
 873                                     bool is_null,
 874                                     u8 tile_mask)
 875 {
 876         struct xe_vma *vma;
 877         struct xe_tile *tile;
 878         u8 id;
 879
 880         XE_BUG_ON(start >= end);
 881         XE_BUG_ON(end >= vm->size);
 882
 883         if (!bo && !is_null)    /* userptr */
 884                 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 885         else
 886                 vma = kzalloc(sizeof(*vma) - sizeof(struct xe_userptr),
 887                               GFP_KERNEL);
 888         if (!vma) {
 889                 vma = ERR_PTR(-ENOMEM);
 890                 return vma;
 891         }
 892
 893         INIT_LIST_HEAD(&vma->combined_links.rebind);
 894         INIT_LIST_HEAD(&vma->notifier.rebind_link);
 895         INIT_LIST_HEAD(&vma->extobj.link);
 896
 897         INIT_LIST_HEAD(&vma->gpuva.gem.entry);
 898         vma->gpuva.vm = &vm->gpuvm;
 899         vma->gpuva.va.addr = start;
 900         vma->gpuva.va.range = end - start + 1;
 901         if (read_only)
 902                 vma->gpuva.flags |= XE_VMA_READ_ONLY;
 903         if (is_null)
 904                 vma->gpuva.flags |= DRM_GPUVA_SPARSE;
 905
 906         if (tile_mask) {
 907                 vma->tile_mask = tile_mask;
 908         } else {
 909                 for_each_tile(tile, vm->xe, id)
 910                         vma->tile_mask |= 0x1 << id;
 911         }
 912
 913         if (vm->xe->info.platform == XE_PVC)
 914                 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT;
 915
 916         if (bo) {
 917                 struct drm_gpuvm_bo *vm_bo;
 918
 919                 xe_bo_assert_held(bo);
 920
 921                 vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base);
 922                 if (IS_ERR(vm_bo)) {
 923                         kfree(vma);
 924                         return ERR_CAST(vm_bo);
 925                 }
 926
 927                 drm_gem_object_get(&bo->ttm.base);
 928                 vma->gpuva.gem.obj = &bo->ttm.base;
 929                 vma->gpuva.gem.offset = bo_offset_or_userptr;
 930                 drm_gpuva_link(&vma->gpuva, vm_bo);
 931                 drm_gpuvm_bo_put(vm_bo);
 932         } else /* userptr or null */ {
 933                 if (!is_null) {
 934                         u64 size = end - start + 1;
 935                         int err;
 936
 937                         INIT_LIST_HEAD(&vma->userptr.invalidate_link);
 938                         vma->gpuva.gem.offset = bo_offset_or_userptr;
 939
 940                         err = mmu_interval_notifier_insert(&vma->userptr.notifier,
 941                                                            current->mm,
 942                                                            xe_vma_userptr(vma), size,
 943                                                            &vma_userptr_notifier_ops);
 944                         if (err) {
 945                                 kfree(vma);
 946                                 vma = ERR_PTR(err);
 947                                 return vma;
 948                         }
 949
 950                         vma->userptr.notifier_seq = LONG_MAX;
 951                 }
 952
 953                 xe_vm_get(vm);
 954         }
 955
 956         return vma;
 957 }
 958
 959 static bool vm_remove_extobj(struct xe_vma *vma)
 960 {
 961         if (!list_empty(&vma->extobj.link)) {
 962                 xe_vma_vm(vma)->extobj.entries--;
 963                 list_del_init(&vma->extobj.link);
 964                 return true;
 965         }
 966         return false;
 967 }
 968
 969 static void xe_vma_destroy_late(struct xe_vma *vma)
 970 {
 971         struct xe_vm *vm = xe_vma_vm(vma);
 972         struct xe_device *xe = vm->xe;
 973         bool read_only = xe_vma_read_only(vma);
 974
 975         if (xe_vma_is_userptr(vma)) {
 976                 if (vma->userptr.sg) {
 977                         dma_unmap_sgtable(xe->drm.dev,
 978                                           vma->userptr.sg,
 979                                           read_only ? DMA_TO_DEVICE :
 980                                           DMA_BIDIRECTIONAL, 0);
 981                         sg_free_table(vma->userptr.sg);
 982                         vma->userptr.sg = NULL;
 983                 }
 984
 985                 /*
 986                  * Since userptr pages are not pinned, we can't remove
 987                  * the notifer until we're sure the GPU is not accessing
 988                  * them anymore
 989                  */
 990                 mmu_interval_notifier_remove(&vma->userptr.notifier);
 991                 xe_vm_put(vm);
 992         } else if (xe_vma_is_null(vma)) {
 993                 xe_vm_put(vm);
 994         } else {
 995                 xe_bo_put(xe_vma_bo(vma));
 996         }
 997
 998         kfree(vma);
 999 }
1000
1001 static void vma_destroy_work_func(struct work_struct *w)
1002 {
1003         struct xe_vma *vma =
1004                 container_of(w, struct xe_vma, destroy_work);
1005
1006         xe_vma_destroy_late(vma);
1007 }
1008
1009 static struct xe_vma *
1010 bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm,
1011                             struct xe_vma *ignore)
1012 {
1013         struct drm_gpuvm_bo *vm_bo;
1014         struct drm_gpuva *va;
1015         struct drm_gem_object *obj = &bo->ttm.base;
1016
1017         xe_bo_assert_held(bo);
1018
1019         drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
1020                 drm_gpuvm_bo_for_each_va(va, vm_bo) {
1021                         struct xe_vma *vma = gpuva_to_vma(va);
1022
1023                         if (vma != ignore && xe_vma_vm(vma) == vm)
1024                                 return vma;
1025                 }
1026         }
1027
1028         return NULL;
1029 }
1030
1031 static bool bo_has_vm_references(struct xe_bo *bo, struct xe_vm *vm,
1032                                  struct xe_vma *ignore)
1033 {
1034         struct ww_acquire_ctx ww;
1035         bool ret;
1036
1037         xe_bo_lock(bo, &ww, 0, false);
1038         ret = !!bo_has_vm_references_locked(bo, vm, ignore);
1039         xe_bo_unlock(bo, &ww);
1040
1041         return ret;
1042 }
1043
1044 static void __vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
1045 {
1046         lockdep_assert_held_write(&vm->lock);
1047
1048         list_add(&vma->extobj.link, &vm->extobj.list);
1049         vm->extobj.entries++;
1050 }
1051
1052 static void vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma)
1053 {
1054         struct xe_bo *bo = xe_vma_bo(vma);
1055
1056         lockdep_assert_held_write(&vm->lock);
1057
1058         if (bo_has_vm_references(bo, vm, vma))
1059                 return;
1060
1061         __vm_insert_extobj(vm, vma);
1062 }
1063
1064 static void vma_destroy_cb(struct dma_fence *fence,
1065                            struct dma_fence_cb *cb)
1066 {
1067         struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb);
1068
1069         INIT_WORK(&vma->destroy_work, vma_destroy_work_func);
1070         queue_work(system_unbound_wq, &vma->destroy_work);
1071 }
1072
1073 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
1074 {
1075         struct xe_vm *vm = xe_vma_vm(vma);
1076
1077         lockdep_assert_held_write(&vm->lock);
1078         XE_BUG_ON(!list_empty(&vma->combined_links.destroy));
1079
1080         if (xe_vma_is_userptr(vma)) {
1081                 XE_WARN_ON(!(vma->gpuva.flags & XE_VMA_DESTROYED));
1082
1083                 spin_lock(&vm->userptr.invalidated_lock);
1084                 list_del(&vma->userptr.invalidate_link);
1085                 spin_unlock(&vm->userptr.invalidated_lock);
1086         } else if (!xe_vma_is_null(vma)) {
1087                 xe_bo_assert_held(xe_vma_bo(vma));
1088
1089                 spin_lock(&vm->notifier.list_lock);
1090                 list_del(&vma->notifier.rebind_link);
1091                 spin_unlock(&vm->notifier.list_lock);
1092
1093                 drm_gpuva_unlink(&vma->gpuva);
1094
1095                 if (!xe_vma_bo(vma)->vm && vm_remove_extobj(vma)) {
1096                         struct xe_vma *other;
1097
1098                         other = bo_has_vm_references_locked(xe_vma_bo(vma), vm, NULL);
1099
1100                         if (other)
1101                                 __vm_insert_extobj(vm, other);
1102                 }
1103         }
1104
1105         xe_vm_assert_held(vm);
1106         if (fence) {
1107                 int ret = dma_fence_add_callback(fence, &vma->destroy_cb,
1108                                                  vma_destroy_cb);
1109
1110                 if (ret) {
1111                         XE_WARN_ON(ret != -ENOENT);
1112                         xe_vma_destroy_late(vma);
1113                 }
1114         } else {
1115                 xe_vma_destroy_late(vma);
1116         }
1117 }
1118
1119 static void xe_vma_destroy_unlocked(struct xe_vma *vma)
1120 {
1121         struct ttm_validate_buffer tv[2];
1122         struct ww_acquire_ctx ww;
1123         struct xe_bo *bo = xe_vma_bo(vma);
1124         LIST_HEAD(objs);
1125         LIST_HEAD(dups);
1126         int err;
1127
1128         memset(tv, 0, sizeof(tv));
1129         tv[0].bo = xe_vm_ttm_bo(xe_vma_vm(vma));
1130         list_add(&tv[0].head, &objs);
1131
1132         if (bo) {
1133                 tv[1].bo = &xe_bo_get(bo)->ttm;
1134                 list_add(&tv[1].head, &objs);
1135         }
1136         err = ttm_eu_reserve_buffers(&ww, &objs, false, &dups);
1137         XE_WARN_ON(err);
1138
1139         xe_vma_destroy(vma, NULL);
1140
1141         ttm_eu_backoff_reservation(&ww, &objs);
1142         if (bo)
1143                 xe_bo_put(bo);
1144 }
1145
1146 struct xe_vma *
1147 xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range)
1148 {
1149         struct drm_gpuva *gpuva;
1150
1151         lockdep_assert_held(&vm->lock);
1152
1153         if (xe_vm_is_closed_or_banned(vm))
1154                 return NULL;
1155
1156         XE_BUG_ON(start + range > vm->size);
1157
1158         gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range);
1159
1160         return gpuva ? gpuva_to_vma(gpuva) : NULL;
1161 }
1162
1163 static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma)
1164 {
1165         int err;
1166
1167         XE_BUG_ON(xe_vma_vm(vma) != vm);
1168         lockdep_assert_held(&vm->lock);
1169
1170         err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva);
1171         XE_WARN_ON(err);        /* Shouldn't be possible */
1172
1173         return err;
1174 }
1175
1176 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma)
1177 {
1178         XE_BUG_ON(xe_vma_vm(vma) != vm);
1179         lockdep_assert_held(&vm->lock);
1180
1181         drm_gpuva_remove(&vma->gpuva);
1182         if (vm->usm.last_fault_vma == vma)
1183                 vm->usm.last_fault_vma = NULL;
1184 }
1185
1186 static struct drm_gpuva_op *xe_vm_op_alloc(void)
1187 {
1188         struct xe_vma_op *op;
1189
1190         op = kzalloc(sizeof(*op), GFP_KERNEL);
1191
1192         if (unlikely(!op))
1193                 return NULL;
1194
1195         return &op->base;
1196 }
1197
1198 static void xe_vm_free(struct drm_gpuvm *gpuvm);
1199
1200 static struct drm_gpuvm_ops gpuvm_ops = {
1201         .op_alloc = xe_vm_op_alloc,
1202         .vm_free = xe_vm_free,
1203 };
1204
1205 static void xe_vma_op_work_func(struct work_struct *w);
1206 static void vm_destroy_work_func(struct work_struct *w);
1207
1208 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
1209 {
1210         struct drm_gem_object *vm_resv_obj;
1211         struct xe_vm *vm;
1212         int err, number_tiles = 0;
1213         struct xe_tile *tile;
1214         u8 id;
1215
1216         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1217         if (!vm)
1218                 return ERR_PTR(-ENOMEM);
1219
1220         vm->xe = xe;
1221
1222         vm->size = 1ull << xe_pt_shift(xe->info.vm_max_level + 1);
1223
1224         vm->flags = flags;
1225
1226         init_rwsem(&vm->lock);
1227
1228         INIT_LIST_HEAD(&vm->rebind_list);
1229
1230         INIT_LIST_HEAD(&vm->userptr.repin_list);
1231         INIT_LIST_HEAD(&vm->userptr.invalidated);
1232         init_rwsem(&vm->userptr.notifier_lock);
1233         spin_lock_init(&vm->userptr.invalidated_lock);
1234
1235         INIT_LIST_HEAD(&vm->notifier.rebind_list);
1236         spin_lock_init(&vm->notifier.list_lock);
1237
1238         INIT_LIST_HEAD(&vm->async_ops.pending);
1239         INIT_WORK(&vm->async_ops.work, xe_vma_op_work_func);
1240         spin_lock_init(&vm->async_ops.lock);
1241
1242         INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
1243
1244         INIT_LIST_HEAD(&vm->preempt.engines);
1245         vm->preempt.min_run_period_ms = 10;     /* FIXME: Wire up to uAPI */
1246
1247         for_each_tile(tile, xe, id)
1248                 xe_range_fence_tree_init(&vm->rftree[id]);
1249
1250         INIT_LIST_HEAD(&vm->extobj.list);
1251
1252         if (!(flags & XE_VM_FLAG_MIGRATION))
1253                 xe_device_mem_access_get(xe);
1254
1255         vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
1256         if (!vm_resv_obj) {
1257                 err = -ENOMEM;
1258                 goto err_no_resv;
1259         }
1260
1261         drm_gpuvm_init(&vm->gpuvm, "Xe VM", 0, &xe->drm, vm_resv_obj,
1262                        0, vm->size, 0, 0, &gpuvm_ops);
1263
1264         drm_gem_object_put(vm_resv_obj);
1265
1266         err = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
1267         if (err)
1268                 goto err_close;
1269
1270         if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
1271                 vm->flags |= XE_VM_FLAG_64K;
1272
1273         for_each_tile(tile, xe, id) {
1274                 if (flags & XE_VM_FLAG_MIGRATION &&
1275                     tile->id != XE_VM_FLAG_TILE_ID(flags))
1276                         continue;
1277
1278                 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level);
1279                 if (IS_ERR(vm->pt_root[id])) {
1280                         err = PTR_ERR(vm->pt_root[id]);
1281                         vm->pt_root[id] = NULL;
1282                         goto err_unlock_close;
1283                 }
1284         }
1285
1286         if (flags & XE_VM_FLAG_SCRATCH_PAGE) {
1287                 for_each_tile(tile, xe, id) {
1288                         if (!vm->pt_root[id])
1289                                 continue;
1290
1291                         err = xe_pt_create_scratch(xe, tile, vm);
1292                         if (err)
1293                                 goto err_unlock_close;
1294                 }
1295                 vm->batch_invalidate_tlb = true;
1296         }
1297
1298         if (flags & XE_VM_FLAG_COMPUTE_MODE) {
1299                 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func);
1300                 vm->flags |= XE_VM_FLAG_COMPUTE_MODE;
1301                 vm->batch_invalidate_tlb = false;
1302         }
1303
1304         if (flags & XE_VM_FLAG_ASYNC_BIND_OPS) {
1305                 vm->async_ops.fence.context = dma_fence_context_alloc(1);
1306                 vm->flags |= XE_VM_FLAG_ASYNC_BIND_OPS;
1307         }
1308
1309         /* Fill pt_root after allocating scratch tables */
1310         for_each_tile(tile, xe, id) {
1311                 if (!vm->pt_root[id])
1312                         continue;
1313
1314                 xe_pt_populate_empty(tile, vm, vm->pt_root[id]);
1315         }
1316         dma_resv_unlock(xe_vm_resv(vm));
1317
1318         /* Kernel migration VM shouldn't have a circular loop.. */
1319         if (!(flags & XE_VM_FLAG_MIGRATION)) {
1320                 for_each_tile(tile, xe, id) {
1321                         struct xe_gt *gt = tile->primary_gt;
1322                         struct xe_vm *migrate_vm;
1323                         struct xe_engine *eng;
1324
1325                         if (!vm->pt_root[id])
1326                                 continue;
1327
1328                         migrate_vm = xe_migrate_get_vm(tile->migrate);
1329                         eng = xe_engine_create_class(xe, gt, migrate_vm,
1330                                                      XE_ENGINE_CLASS_COPY,
1331                                                      ENGINE_FLAG_VM);
1332                         xe_vm_put(migrate_vm);
1333                         if (IS_ERR(eng)) {
1334                                 err = PTR_ERR(eng);
1335                                 goto err_close;
1336                         }
1337                         vm->eng[id] = eng;
1338                         number_tiles++;
1339                 }
1340         }
1341
1342         if (number_tiles > 1)
1343                 vm->composite_fence_ctx = dma_fence_context_alloc(1);
1344
1345         mutex_lock(&xe->usm.lock);
1346         if (flags & XE_VM_FLAG_FAULT_MODE)
1347                 xe->usm.num_vm_in_fault_mode++;
1348         else if (!(flags & XE_VM_FLAG_MIGRATION))
1349                 xe->usm.num_vm_in_non_fault_mode++;
1350         mutex_unlock(&xe->usm.lock);
1351
1352         trace_xe_vm_create(vm);
1353
1354         return vm;
1355
1356 err_unlock_close:
1357         dma_resv_unlock(xe_vm_resv(vm));
1358 err_close:
1359         xe_vm_close_and_put(vm);
1360         return ERR_PTR(err);
1361
1362 err_no_resv:
1363         for_each_tile(tile, xe, id)
1364                 xe_range_fence_tree_fini(&vm->rftree[id]);
1365         kfree(vm);
1366         if (!(flags & XE_VM_FLAG_MIGRATION))
1367                 xe_device_mem_access_put(xe);
1368         return ERR_PTR(err);
1369 }
1370
1371 static void flush_async_ops(struct xe_vm *vm)
1372 {
1373         queue_work(system_unbound_wq, &vm->async_ops.work);
1374         flush_work(&vm->async_ops.work);
1375 }
1376
1377 static void vm_error_capture(struct xe_vm *vm, int err,
1378                              u32 op, u64 addr, u64 size)
1379 {
1380         struct drm_xe_vm_bind_op_error_capture capture;
1381         u64 __user *address =
1382                 u64_to_user_ptr(vm->async_ops.error_capture.addr);
1383         bool in_kthread = !current->mm;
1384
1385         capture.error = err;
1386         capture.op = op;
1387         capture.addr = addr;
1388         capture.size = size;
1389
1390         if (in_kthread) {
1391                 if (!mmget_not_zero(vm->async_ops.error_capture.mm))
1392                         goto mm_closed;
1393                 kthread_use_mm(vm->async_ops.error_capture.mm);
1394         }
1395
1396         if (copy_to_user(address, &capture, sizeof(capture)))
1397                 XE_WARN_ON("Copy to user failed");
1398
1399         if (in_kthread) {
1400                 kthread_unuse_mm(vm->async_ops.error_capture.mm);
1401                 mmput(vm->async_ops.error_capture.mm);
1402         }
1403
1404 mm_closed:
1405         wake_up_all(&vm->async_ops.error_capture.wq);
1406 }
1407
1408 static void xe_vm_close(struct xe_vm *vm)
1409 {
1410         down_write(&vm->lock);
1411         vm->size = 0;
1412         up_write(&vm->lock);
1413 }
1414
1415 void xe_vm_close_and_put(struct xe_vm *vm)
1416 {
1417         LIST_HEAD(contested);
1418         struct ww_acquire_ctx ww;
1419         struct xe_device *xe = vm->xe;
1420         struct xe_tile *tile;
1421         struct xe_vma *vma, *next_vma;
1422         struct drm_gpuva *gpuva, *next;
1423         u8 id;
1424
1425         XE_BUG_ON(vm->preempt.num_engines);
1426
1427         xe_vm_close(vm);
1428         flush_async_ops(vm);
1429         if (xe_vm_in_compute_mode(vm))
1430                 flush_work(&vm->preempt.rebind_work);
1431
1432         for_each_tile(tile, xe, id) {
1433                 if (vm->eng[id]) {
1434                         xe_engine_kill(vm->eng[id]);
1435                         xe_engine_put(vm->eng[id]);
1436                         vm->eng[id] = NULL;
1437                 }
1438         }
1439
1440         down_write(&vm->lock);
1441         xe_vm_lock(vm, &ww, 0, false);
1442         drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) {
1443                 vma = gpuva_to_vma(gpuva);
1444
1445                 if (xe_vma_has_no_bo(vma)) {
1446                         down_read(&vm->userptr.notifier_lock);
1447                         vma->gpuva.flags |= XE_VMA_DESTROYED;
1448                         up_read(&vm->userptr.notifier_lock);
1449                 }
1450
1451                 xe_vm_remove_vma(vm, vma);
1452
1453                 /* easy case, remove from VMA? */
1454                 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) {
1455                         list_del_init(&vma->combined_links.rebind);
1456                         xe_vma_destroy(vma, NULL);
1457                         continue;
1458                 }
1459
1460                 list_move_tail(&vma->combined_links.destroy, &contested);
1461         }
1462
1463         /*
1464          * All vm operations will add shared fences to resv.
1465          * The only exception is eviction for a shared object,
1466          * but even so, the unbind when evicted would still
1467          * install a fence to resv. Hence it's safe to
1468          * destroy the pagetables immediately.
1469          */
1470         for_each_tile(tile, xe, id) {
1471                 if (vm->scratch_bo[id]) {
1472                         u32 i;
1473
1474                         xe_bo_unpin(vm->scratch_bo[id]);
1475                         xe_bo_put(vm->scratch_bo[id]);
1476                         for (i = 0; i < vm->pt_root[id]->level; i++)
1477                                 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags,
1478                                               NULL);
1479                 }
1480                 if (vm->pt_root[id]) {
1481                         xe_pt_destroy(vm->pt_root[id], vm->flags, NULL);
1482                         vm->pt_root[id] = NULL;
1483                 }
1484         }
1485         xe_vm_unlock(vm, &ww);
1486
1487         /*
1488          * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL
1489          * Since we hold a refcount to the bo, we can remove and free
1490          * the members safely without locking.
1491          */
1492         list_for_each_entry_safe(vma, next_vma, &contested,
1493                                  combined_links.destroy) {
1494                 list_del_init(&vma->combined_links.destroy);
1495                 xe_vma_destroy_unlocked(vma);
1496         }
1497
1498         if (vm->async_ops.error_capture.addr)
1499                 wake_up_all(&vm->async_ops.error_capture.wq);
1500
1501         XE_WARN_ON(!list_empty(&vm->extobj.list));
1502         up_write(&vm->lock);
1503
1504         mutex_lock(&xe->usm.lock);
1505         if (vm->flags & XE_VM_FLAG_FAULT_MODE)
1506                 xe->usm.num_vm_in_fault_mode--;
1507         else if (!(vm->flags & XE_VM_FLAG_MIGRATION))
1508                 xe->usm.num_vm_in_non_fault_mode--;
1509         mutex_unlock(&xe->usm.lock);
1510
1511         for_each_tile(tile, xe, id)
1512                 xe_range_fence_tree_fini(&vm->rftree[id]);
1513
1514         xe_vm_put(vm);
1515 }
1516
1517 static void vm_destroy_work_func(struct work_struct *w)
1518 {
1519         struct xe_vm *vm =
1520                 container_of(w, struct xe_vm, destroy_work);
1521         struct xe_device *xe = vm->xe;
1522         struct xe_tile *tile;
1523         u8 id;
1524         void *lookup;
1525
1526         /* xe_vm_close_and_put was not called? */
1527         XE_WARN_ON(vm->size);
1528
1529         if (!(vm->flags & XE_VM_FLAG_MIGRATION)) {
1530                 xe_device_mem_access_put(xe);
1531
1532                 if (xe->info.has_asid) {
1533                         mutex_lock(&xe->usm.lock);
1534                         lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
1535                         XE_WARN_ON(lookup != vm);
1536                         mutex_unlock(&xe->usm.lock);
1537                 }
1538         }
1539
1540         for_each_tile(tile, xe, id)
1541                 XE_WARN_ON(vm->pt_root[id]);
1542
1543         trace_xe_vm_free(vm);
1544         dma_fence_put(vm->rebind_fence);
1545         kfree(vm);
1546 }
1547
1548 static void xe_vm_free(struct drm_gpuvm *gpuvm)
1549 {
1550         struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
1551
1552         /* To destroy the VM we need to be able to sleep */
1553         queue_work(system_unbound_wq, &vm->destroy_work);
1554 }
1555
1556 struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
1557 {
1558         struct xe_vm *vm;
1559
1560         mutex_lock(&xef->vm.lock);
1561         vm = xa_load(&xef->vm.xa, id);
1562         if (vm)
1563                 xe_vm_get(vm);
1564         mutex_unlock(&xef->vm.lock);
1565
1566         return vm;
1567 }
1568
1569 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile)
1570 {
1571         return xe_pde_encode(vm->pt_root[tile->id]->bo, 0,
1572                              XE_CACHE_WB);
1573 }
1574
1575 static struct dma_fence *
1576 xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e,
1577                  struct xe_sync_entry *syncs, u32 num_syncs,
1578                  bool first_op, bool last_op)
1579 {
1580         struct xe_tile *tile;
1581         struct dma_fence *fence = NULL;
1582         struct dma_fence **fences = NULL;
1583         struct dma_fence_array *cf = NULL;
1584         struct xe_vm *vm = xe_vma_vm(vma);
1585         int cur_fence = 0, i;
1586         int number_tiles = hweight8(vma->tile_present);
1587         int err;
1588         u8 id;
1589
1590         trace_xe_vma_unbind(vma);
1591
1592         if (number_tiles > 1) {
1593                 fences = kmalloc_array(number_tiles, sizeof(*fences),
1594                                        GFP_KERNEL);
1595                 if (!fences)
1596                         return ERR_PTR(-ENOMEM);
1597         }
1598
1599         for_each_tile(tile, vm->xe, id) {
1600                 if (!(vma->tile_present & BIT(id)))
1601                         goto next;
1602
1603                 fence = __xe_pt_unbind_vma(tile, vma, e, first_op ? syncs : NULL,
1604                                            first_op ? num_syncs : 0);
1605                 if (IS_ERR(fence)) {
1606                         err = PTR_ERR(fence);
1607                         goto err_fences;
1608                 }
1609
1610                 if (fences)
1611                         fences[cur_fence++] = fence;
1612
1613 next:
1614                 if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list))
1615                         e = list_next_entry(e, multi_gt_list);
1616         }
1617
1618         if (fences) {
1619                 cf = dma_fence_array_create(number_tiles, fences,
1620                                             vm->composite_fence_ctx,
1621                                             vm->composite_fence_seqno++,
1622                                             false);
1623                 if (!cf) {
1624                         --vm->composite_fence_seqno;
1625                         err = -ENOMEM;
1626                         goto err_fences;
1627                 }
1628         }
1629
1630         if (last_op) {
1631                 for (i = 0; i < num_syncs; i++)
1632                         xe_sync_entry_signal(&syncs[i], NULL,
1633                                              cf ? &cf->base : fence);
1634         }
1635
1636         return cf ? &cf->base : !fence ? dma_fence_get_stub() : fence;
1637
1638 err_fences:
1639         if (fences) {
1640                 while (cur_fence) {
1641                         /* FIXME: Rewind the previous binds? */
1642                         dma_fence_put(fences[--cur_fence]);
1643                 }
1644                 kfree(fences);
1645         }
1646
1647         return ERR_PTR(err);
1648 }
1649
1650 static struct dma_fence *
1651 xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e,
1652                struct xe_sync_entry *syncs, u32 num_syncs,
1653                bool first_op, bool last_op)
1654 {
1655         struct xe_tile *tile;
1656         struct dma_fence *fence;
1657         struct dma_fence **fences = NULL;
1658         struct dma_fence_array *cf = NULL;
1659         struct xe_vm *vm = xe_vma_vm(vma);
1660         int cur_fence = 0, i;
1661         int number_tiles = hweight8(vma->tile_mask);
1662         int err;
1663         u8 id;
1664
1665         trace_xe_vma_bind(vma);
1666
1667         if (number_tiles > 1) {
1668                 fences = kmalloc_array(number_tiles, sizeof(*fences),
1669                                        GFP_KERNEL);
1670                 if (!fences)
1671                         return ERR_PTR(-ENOMEM);
1672         }
1673
1674         for_each_tile(tile, vm->xe, id) {
1675                 if (!(vma->tile_mask & BIT(id)))
1676                         goto next;
1677
1678                 fence = __xe_pt_bind_vma(tile, vma, e, first_op ? syncs : NULL,
1679                                          first_op ? num_syncs : 0,
1680                                          vma->tile_present & BIT(id));
1681                 if (IS_ERR(fence)) {
1682                         err = PTR_ERR(fence);
1683                         goto err_fences;
1684                 }
1685
1686                 if (fences)
1687                         fences[cur_fence++] = fence;
1688
1689 next:
1690                 if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list))
1691                         e = list_next_entry(e, multi_gt_list);
1692         }
1693
1694         if (fences) {
1695                 cf = dma_fence_array_create(number_tiles, fences,
1696                                             vm->composite_fence_ctx,
1697                                             vm->composite_fence_seqno++,
1698                                             false);
1699                 if (!cf) {
1700                         --vm->composite_fence_seqno;
1701                         err = -ENOMEM;
1702                         goto err_fences;
1703                 }
1704         }
1705
1706         if (last_op) {
1707                 for (i = 0; i < num_syncs; i++)
1708                         xe_sync_entry_signal(&syncs[i], NULL,
1709                                              cf ? &cf->base : fence);
1710         }
1711
1712         return cf ? &cf->base : fence;
1713
1714 err_fences:
1715         if (fences) {
1716                 while (cur_fence) {
1717                         /* FIXME: Rewind the previous binds? */
1718                         dma_fence_put(fences[--cur_fence]);
1719                 }
1720                 kfree(fences);
1721         }
1722
1723         return ERR_PTR(err);
1724 }
1725
1726 struct async_op_fence {
1727         struct dma_fence fence;
1728         struct dma_fence *wait_fence;
1729         struct dma_fence_cb cb;
1730         struct xe_vm *vm;
1731         wait_queue_head_t wq;
1732         bool started;
1733 };
1734
1735 static const char *async_op_fence_get_driver_name(struct dma_fence *dma_fence)
1736 {
1737         return "xe";
1738 }
1739
1740 static const char *
1741 async_op_fence_get_timeline_name(struct dma_fence *dma_fence)
1742 {
1743         return "async_op_fence";
1744 }
1745
1746 static const struct dma_fence_ops async_op_fence_ops = {
1747         .get_driver_name = async_op_fence_get_driver_name,
1748         .get_timeline_name = async_op_fence_get_timeline_name,
1749 };
1750
1751 static void async_op_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
1752 {
1753         struct async_op_fence *afence =
1754                 container_of(cb, struct async_op_fence, cb);
1755
1756         afence->fence.error = afence->wait_fence->error;
1757         dma_fence_signal(&afence->fence);
1758         xe_vm_put(afence->vm);
1759         dma_fence_put(afence->wait_fence);
1760         dma_fence_put(&afence->fence);
1761 }
1762
1763 static void add_async_op_fence_cb(struct xe_vm *vm,
1764                                   struct dma_fence *fence,
1765                                   struct async_op_fence *afence)
1766 {
1767         int ret;
1768
1769         if (!xe_vm_no_dma_fences(vm)) {
1770                 afence->started = true;
1771                 smp_wmb();
1772                 wake_up_all(&afence->wq);
1773         }
1774
1775         afence->wait_fence = dma_fence_get(fence);
1776         afence->vm = xe_vm_get(vm);
1777         dma_fence_get(&afence->fence);
1778         ret = dma_fence_add_callback(fence, &afence->cb, async_op_fence_cb);
1779         if (ret == -ENOENT) {
1780                 afence->fence.error = afence->wait_fence->error;
1781                 dma_fence_signal(&afence->fence);
1782         }
1783         if (ret) {
1784                 xe_vm_put(vm);
1785                 dma_fence_put(afence->wait_fence);
1786                 dma_fence_put(&afence->fence);
1787         }
1788         XE_WARN_ON(ret && ret != -ENOENT);
1789 }
1790
1791 int xe_vm_async_fence_wait_start(struct dma_fence *fence)
1792 {
1793         if (fence->ops == &async_op_fence_ops) {
1794                 struct async_op_fence *afence =
1795                         container_of(fence, struct async_op_fence, fence);
1796
1797                 XE_BUG_ON(xe_vm_no_dma_fences(afence->vm));
1798
1799                 smp_rmb();
1800                 return wait_event_interruptible(afence->wq, afence->started);
1801         }
1802
1803         return 0;
1804 }
1805
1806 static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
1807                         struct xe_engine *e, struct xe_sync_entry *syncs,
1808                         u32 num_syncs, struct async_op_fence *afence,
1809                         bool immediate, bool first_op, bool last_op)
1810 {
1811         struct dma_fence *fence;
1812
1813         xe_vm_assert_held(vm);
1814
1815         if (immediate) {
1816                 fence = xe_vm_bind_vma(vma, e, syncs, num_syncs, first_op,
1817                                        last_op);
1818                 if (IS_ERR(fence))
1819                         return PTR_ERR(fence);
1820         } else {
1821                 int i;
1822
1823                 XE_BUG_ON(!xe_vm_in_fault_mode(vm));
1824
1825                 fence = dma_fence_get_stub();
1826                 if (last_op) {
1827                         for (i = 0; i < num_syncs; i++)
1828                                 xe_sync_entry_signal(&syncs[i], NULL, fence);
1829                 }
1830         }
1831         if (afence)
1832                 add_async_op_fence_cb(vm, fence, afence);
1833
1834         dma_fence_put(fence);
1835         return 0;
1836 }
1837
1838 static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_engine *e,
1839                       struct xe_bo *bo, struct xe_sync_entry *syncs,
1840                       u32 num_syncs, struct async_op_fence *afence,
1841                       bool immediate, bool first_op, bool last_op)
1842 {
1843         int err;
1844
1845         xe_vm_assert_held(vm);
1846         xe_bo_assert_held(bo);
1847
1848         if (bo && immediate) {
1849                 err = xe_bo_validate(bo, vm, true);
1850                 if (err)
1851                         return err;
1852         }
1853
1854         return __xe_vm_bind(vm, vma, e, syncs, num_syncs, afence, immediate,
1855                             first_op, last_op);
1856 }
1857
1858 static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
1859                         struct xe_engine *e, struct xe_sync_entry *syncs,
1860                         u32 num_syncs, struct async_op_fence *afence,
1861                         bool first_op, bool last_op)
1862 {
1863         struct dma_fence *fence;
1864
1865         xe_vm_assert_held(vm);
1866         xe_bo_assert_held(xe_vma_bo(vma));
1867
1868         fence = xe_vm_unbind_vma(vma, e, syncs, num_syncs, first_op, last_op);
1869         if (IS_ERR(fence))
1870                 return PTR_ERR(fence);
1871         if (afence)
1872                 add_async_op_fence_cb(vm, fence, afence);
1873
1874         xe_vma_destroy(vma, fence);
1875         dma_fence_put(fence);
1876
1877         return 0;
1878 }
1879
1880 static int vm_set_error_capture_address(struct xe_device *xe, struct xe_vm *vm,
1881                                         u64 value)
1882 {
1883         if (XE_IOCTL_DBG(xe, !value))
1884                 return -EINVAL;
1885
1886         if (XE_IOCTL_DBG(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS)))
1887                 return -EOPNOTSUPP;
1888
1889         if (XE_IOCTL_DBG(xe, vm->async_ops.error_capture.addr))
1890                 return -EOPNOTSUPP;
1891
1892         vm->async_ops.error_capture.mm = current->mm;
1893         vm->async_ops.error_capture.addr = value;
1894         init_waitqueue_head(&vm->async_ops.error_capture.wq);
1895
1896         return 0;
1897 }
1898
1899 typedef int (*xe_vm_set_property_fn)(struct xe_device *xe, struct xe_vm *vm,
1900                                      u64 value);
1901
1902 static const xe_vm_set_property_fn vm_set_property_funcs[] = {
1903         [XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS] =
1904                 vm_set_error_capture_address,
1905 };
1906
1907 static int vm_user_ext_set_property(struct xe_device *xe, struct xe_vm *vm,
1908                                     u64 extension)
1909 {
1910         u64 __user *address = u64_to_user_ptr(extension);
1911         struct drm_xe_ext_vm_set_property ext;
1912         int err;
1913
1914         err = __copy_from_user(&ext, address, sizeof(ext));
1915         if (XE_IOCTL_DBG(xe, err))
1916                 return -EFAULT;
1917
1918         if (XE_IOCTL_DBG(xe, ext.property >=
1919                          ARRAY_SIZE(vm_set_property_funcs)) ||
1920             XE_IOCTL_DBG(xe, ext.pad) ||
1921             XE_IOCTL_DBG(xe, ext.reserved[0] || ext.reserved[1]))
1922                 return -EINVAL;
1923
1924         return vm_set_property_funcs[ext.property](xe, vm, ext.value);
1925 }
1926
1927 typedef int (*xe_vm_user_extension_fn)(struct xe_device *xe, struct xe_vm *vm,
1928                                        u64 extension);
1929
1930 static const xe_vm_set_property_fn vm_user_extension_funcs[] = {
1931         [XE_VM_EXTENSION_SET_PROPERTY] = vm_user_ext_set_property,
1932 };
1933
1934 #define MAX_USER_EXTENSIONS     16
1935 static int vm_user_extensions(struct xe_device *xe, struct xe_vm *vm,
1936                               u64 extensions, int ext_number)
1937 {
1938         u64 __user *address = u64_to_user_ptr(extensions);
1939         struct xe_user_extension ext;
1940         int err;
1941
1942         if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
1943                 return -E2BIG;
1944
1945         err = __copy_from_user(&ext, address, sizeof(ext));
1946         if (XE_IOCTL_DBG(xe, err))
1947                 return -EFAULT;
1948
1949         if (XE_IOCTL_DBG(xe, ext.pad) ||
1950             XE_IOCTL_DBG(xe, ext.name >=
1951                          ARRAY_SIZE(vm_user_extension_funcs)))
1952                 return -EINVAL;
1953
1954         err = vm_user_extension_funcs[ext.name](xe, vm, extensions);
1955         if (XE_IOCTL_DBG(xe, err))
1956                 return err;
1957
1958         if (ext.next_extension)
1959                 return vm_user_extensions(xe, vm, ext.next_extension,
1960                                           ++ext_number);
1961
1962         return 0;
1963 }
1964
1965 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \
1966                                     DRM_XE_VM_CREATE_COMPUTE_MODE | \
1967                                     DRM_XE_VM_CREATE_ASYNC_BIND_OPS | \
1968                                     DRM_XE_VM_CREATE_FAULT_MODE)
1969
1970 int xe_vm_create_ioctl(struct drm_device *dev, void *data,
1971                        struct drm_file *file)
1972 {
1973         struct xe_device *xe = to_xe_device(dev);
1974         struct xe_file *xef = to_xe_file(file);
1975         struct drm_xe_vm_create *args = data;
1976         struct xe_vm *vm;
1977         u32 id, asid;
1978         int err;
1979         u32 flags = 0;
1980
1981         if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1982                 return -EINVAL;
1983
1984         if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS))
1985                 return -EINVAL;
1986
1987         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE &&
1988                          args->flags & DRM_XE_VM_CREATE_FAULT_MODE))
1989                 return -EINVAL;
1990
1991         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE &&
1992                          args->flags & DRM_XE_VM_CREATE_FAULT_MODE))
1993                 return -EINVAL;
1994
1995         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE &&
1996                          xe_device_in_non_fault_mode(xe)))
1997                 return -EINVAL;
1998
1999         if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FAULT_MODE) &&
2000                          xe_device_in_fault_mode(xe)))
2001                 return -EINVAL;
2002
2003         if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE &&
2004                          !xe->info.supports_usm))
2005                 return -EINVAL;
2006
2007         if (args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE)
2008                 flags |= XE_VM_FLAG_SCRATCH_PAGE;
2009         if (args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE)
2010                 flags |= XE_VM_FLAG_COMPUTE_MODE;
2011         if (args->flags & DRM_XE_VM_CREATE_ASYNC_BIND_OPS)
2012                 flags |= XE_VM_FLAG_ASYNC_BIND_OPS;
2013         if (args->flags & DRM_XE_VM_CREATE_FAULT_MODE)
2014                 flags |= XE_VM_FLAG_FAULT_MODE;
2015
2016         vm = xe_vm_create(xe, flags);
2017         if (IS_ERR(vm))
2018                 return PTR_ERR(vm);
2019
2020         if (args->extensions) {
2021                 err = vm_user_extensions(xe, vm, args->extensions, 0);
2022                 if (XE_IOCTL_DBG(xe, err)) {
2023                         xe_vm_close_and_put(vm);
2024                         return err;
2025                 }
2026         }
2027
2028         mutex_lock(&xef->vm.lock);
2029         err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL);
2030         mutex_unlock(&xef->vm.lock);
2031         if (err) {
2032                 xe_vm_close_and_put(vm);
2033                 return err;
2034         }
2035
2036         if (xe->info.has_asid) {
2037                 mutex_lock(&xe->usm.lock);
2038                 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm,
2039                                       XA_LIMIT(0, XE_MAX_ASID - 1),
2040                                       &xe->usm.next_asid, GFP_KERNEL);
2041                 mutex_unlock(&xe->usm.lock);
2042                 if (err) {
2043                         xe_vm_close_and_put(vm);
2044                         return err;
2045                 }
2046                 vm->usm.asid = asid;
2047         }
2048
2049         args->vm_id = id;
2050
2051 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
2052         /* Warning: Security issue - never enable by default */
2053         args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE);
2054 #endif
2055
2056         return 0;
2057 }
2058
2059 int xe_vm_destroy_ioctl(struct drm_device *dev, void *data,
2060                         struct drm_file *file)
2061 {
2062         struct xe_device *xe = to_xe_device(dev);
2063         struct xe_file *xef = to_xe_file(file);
2064         struct drm_xe_vm_destroy *args = data;
2065         struct xe_vm *vm;
2066         int err = 0;
2067
2068         if (XE_IOCTL_DBG(xe, args->pad) ||
2069             XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
2070                 return -EINVAL;
2071
2072         mutex_lock(&xef->vm.lock);
2073         vm = xa_load(&xef->vm.xa, args->vm_id);
2074         if (XE_IOCTL_DBG(xe, !vm))
2075                 err = -ENOENT;
2076         else if (XE_IOCTL_DBG(xe, vm->preempt.num_engines))
2077                 err = -EBUSY;
2078         else
2079                 xa_erase(&xef->vm.xa, args->vm_id);
2080         mutex_unlock(&xef->vm.lock);
2081
2082         if (!err)
2083                 xe_vm_close_and_put(vm);
2084
2085         return err;
2086 }
2087
2088 static const u32 region_to_mem_type[] = {
2089         XE_PL_TT,
2090         XE_PL_VRAM0,
2091         XE_PL_VRAM1,
2092 };
2093
2094 static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
2095                           struct xe_engine *e, u32 region,
2096                           struct xe_sync_entry *syncs, u32 num_syncs,
2097                           struct async_op_fence *afence, bool first_op,
2098                           bool last_op)
2099 {
2100         int err;
2101
2102         XE_BUG_ON(region > ARRAY_SIZE(region_to_mem_type));
2103
2104         if (!xe_vma_has_no_bo(vma)) {
2105                 err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
2106                 if (err)
2107                         return err;
2108         }
2109
2110         if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
2111                 return xe_vm_bind(vm, vma, e, xe_vma_bo(vma), syncs, num_syncs,
2112                                   afence, true, first_op, last_op);
2113         } else {
2114                 int i;
2115
2116                 /* Nothing to do, signal fences now */
2117                 if (last_op) {
2118                         for (i = 0; i < num_syncs; i++)
2119                                 xe_sync_entry_signal(&syncs[i], NULL,
2120                                                      dma_fence_get_stub());
2121                 }
2122                 if (afence)
2123                         dma_fence_signal(&afence->fence);
2124                 return 0;
2125         }
2126 }
2127
2128 #define VM_BIND_OP(op)  (op & 0xffff)
2129
2130 struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm)
2131 {
2132         int idx = vm->flags & XE_VM_FLAG_MIGRATION ?
2133                 XE_VM_FLAG_TILE_ID(vm->flags) : 0;
2134
2135         /* Safe to use index 0 as all BO in the VM share a single dma-resv lock */
2136         return &vm->pt_root[idx]->bo->ttm;
2137 }
2138
2139 static void xe_vm_tv_populate(struct xe_vm *vm, struct ttm_validate_buffer *tv)
2140 {
2141         tv->num_shared = 1;
2142         tv->bo = xe_vm_ttm_bo(vm);
2143 }
2144
2145 static void vm_set_async_error(struct xe_vm *vm, int err)
2146 {
2147         lockdep_assert_held(&vm->lock);
2148         vm->async_ops.error = err;
2149 }
2150
2151 static int vm_bind_ioctl_lookup_vma(struct xe_vm *vm, struct xe_bo *bo,
2152                                     u64 addr, u64 range, u32 op)
2153 {
2154         struct xe_device *xe = vm->xe;
2155         struct xe_vma *vma;
2156         bool async = !!(op & XE_VM_BIND_FLAG_ASYNC);
2157
2158         lockdep_assert_held(&vm->lock);
2159
2160         switch (VM_BIND_OP(op)) {
2161         case XE_VM_BIND_OP_MAP:
2162         case XE_VM_BIND_OP_MAP_USERPTR:
2163                 vma = xe_vm_find_overlapping_vma(vm, addr, range);
2164                 if (XE_IOCTL_DBG(xe, vma && !async))
2165                         return -EBUSY;
2166                 break;
2167         case XE_VM_BIND_OP_UNMAP:
2168         case XE_VM_BIND_OP_PREFETCH:
2169                 vma = xe_vm_find_overlapping_vma(vm, addr, range);
2170                 if (XE_IOCTL_DBG(xe, !vma))
2171                         /* Not an actual error, IOCTL cleans up returns and 0 */
2172                         return -ENODATA;
2173                 if (XE_IOCTL_DBG(xe, (xe_vma_start(vma) != addr ||
2174                                       xe_vma_end(vma) != addr + range) && !async))
2175                         return -EINVAL;
2176                 break;
2177         case XE_VM_BIND_OP_UNMAP_ALL:
2178                 if (XE_IOCTL_DBG(xe, list_empty(&bo->ttm.base.gpuva.list)))
2179                         /* Not an actual error, IOCTL cleans up returns and 0 */
2180                         return -ENODATA;
2181                 break;
2182         default:
2183                 XE_BUG_ON("NOT POSSIBLE");
2184                 return -EINVAL;
2185         }
2186
2187         return 0;
2188 }
2189
2190 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma,
2191                              bool post_commit)
2192 {
2193         down_read(&vm->userptr.notifier_lock);
2194         vma->gpuva.flags |= XE_VMA_DESTROYED;
2195         up_read(&vm->userptr.notifier_lock);
2196         if (post_commit)
2197                 xe_vm_remove_vma(vm, vma);
2198 }
2199
2200 #undef ULL
2201 #define ULL     unsigned long long
2202
2203 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
2204 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
2205 {
2206         struct xe_vma *vma;
2207
2208         switch (op->op) {
2209         case DRM_GPUVA_OP_MAP:
2210                 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx",
2211                        (ULL)op->map.va.addr, (ULL)op->map.va.range);
2212                 break;
2213         case DRM_GPUVA_OP_REMAP:
2214                 vma = gpuva_to_vma(op->remap.unmap->va);
2215                 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2216                        (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2217                        op->unmap.keep ? 1 : 0);
2218                 if (op->remap.prev)
2219                         vm_dbg(&xe->drm,
2220                                "REMAP:PREV: addr=0x%016llx, range=0x%016llx",
2221                                (ULL)op->remap.prev->va.addr,
2222                                (ULL)op->remap.prev->va.range);
2223                 if (op->remap.next)
2224                         vm_dbg(&xe->drm,
2225                                "REMAP:NEXT: addr=0x%016llx, range=0x%016llx",
2226                                (ULL)op->remap.next->va.addr,
2227                                (ULL)op->remap.next->va.range);
2228                 break;
2229         case DRM_GPUVA_OP_UNMAP:
2230                 vma = gpuva_to_vma(op->unmap.va);
2231                 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2232                        (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2233                        op->unmap.keep ? 1 : 0);
2234                 break;
2235         case DRM_GPUVA_OP_PREFETCH:
2236                 vma = gpuva_to_vma(op->prefetch.va);
2237                 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx",
2238                        (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma));
2239                 break;
2240         default:
2241                 XE_BUG_ON("NOT POSSIBLE");
2242         }
2243 }
2244 #else
2245 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
2246 {
2247 }
2248 #endif
2249
2250 /*
2251  * Create operations list from IOCTL arguments, setup operations fields so parse
2252  * and commit steps are decoupled from IOCTL arguments. This step can fail.
2253  */
2254 static struct drm_gpuva_ops *
2255 vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo,
2256                          u64 bo_offset_or_userptr, u64 addr, u64 range,
2257                          u32 operation, u8 tile_mask, u32 region)
2258 {
2259         struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL;
2260         struct ww_acquire_ctx ww;
2261         struct drm_gpuva_ops *ops;
2262         struct drm_gpuva_op *__op;
2263         struct xe_vma_op *op;
2264         struct drm_gpuvm_bo *vm_bo;
2265         int err;
2266
2267         lockdep_assert_held_write(&vm->lock);
2268
2269         vm_dbg(&vm->xe->drm,
2270                "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx",
2271                VM_BIND_OP(operation), (ULL)addr, (ULL)range,
2272                (ULL)bo_offset_or_userptr);
2273
2274         switch (VM_BIND_OP(operation)) {
2275         case XE_VM_BIND_OP_MAP:
2276         case XE_VM_BIND_OP_MAP_USERPTR:
2277                 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, addr, range,
2278                                                   obj, bo_offset_or_userptr);
2279                 if (IS_ERR(ops))
2280                         return ops;
2281
2282                 drm_gpuva_for_each_op(__op, ops) {
2283                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2284
2285                         op->tile_mask = tile_mask;
2286                         op->map.immediate =
2287                                 operation & XE_VM_BIND_FLAG_IMMEDIATE;
2288                         op->map.read_only =
2289                                 operation & XE_VM_BIND_FLAG_READONLY;
2290                         op->map.is_null = operation & XE_VM_BIND_FLAG_NULL;
2291                 }
2292                 break;
2293         case XE_VM_BIND_OP_UNMAP:
2294                 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range);
2295                 if (IS_ERR(ops))
2296                         return ops;
2297
2298                 drm_gpuva_for_each_op(__op, ops) {
2299                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2300
2301                         op->tile_mask = tile_mask;
2302                 }
2303                 break;
2304         case XE_VM_BIND_OP_PREFETCH:
2305                 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range);
2306                 if (IS_ERR(ops))
2307                         return ops;
2308
2309                 drm_gpuva_for_each_op(__op, ops) {
2310                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2311
2312                         op->tile_mask = tile_mask;
2313                         op->prefetch.region = region;
2314                 }
2315                 break;
2316         case XE_VM_BIND_OP_UNMAP_ALL:
2317                 XE_BUG_ON(!bo);
2318
2319                 err = xe_bo_lock(bo, &ww, 0, true);
2320                 if (err)
2321                         return ERR_PTR(err);
2322
2323                 vm_bo = drm_gpuvm_bo_find(&vm->gpuvm, obj);
2324                 if (!vm_bo)
2325                         break;
2326
2327                 ops = drm_gpuvm_bo_unmap_ops_create(vm_bo);
2328                 drm_gpuvm_bo_put(vm_bo);
2329                 xe_bo_unlock(bo, &ww);
2330                 if (IS_ERR(ops))
2331                         return ops;
2332
2333                 drm_gpuva_for_each_op(__op, ops) {
2334                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2335
2336                         op->tile_mask = tile_mask;
2337                 }
2338                 break;
2339         default:
2340                 XE_BUG_ON("NOT POSSIBLE");
2341                 ops = ERR_PTR(-EINVAL);
2342         }
2343
2344 #ifdef TEST_VM_ASYNC_OPS_ERROR
2345         if (operation & FORCE_ASYNC_OP_ERROR) {
2346                 op = list_first_entry_or_null(&ops->list, struct xe_vma_op,
2347                                               base.entry);
2348                 if (op)
2349                         op->inject_error = true;
2350         }
2351 #endif
2352
2353         if (!IS_ERR(ops))
2354                 drm_gpuva_for_each_op(__op, ops)
2355                         print_op(vm->xe, __op);
2356
2357         return ops;
2358 }
2359
2360 static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
2361                               u8 tile_mask, bool read_only, bool is_null)
2362 {
2363         struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
2364         struct xe_vma *vma;
2365         struct ww_acquire_ctx ww;
2366         int err;
2367
2368         lockdep_assert_held_write(&vm->lock);
2369
2370         if (bo) {
2371                 err = xe_bo_lock(bo, &ww, 0, true);
2372                 if (err)
2373                         return ERR_PTR(err);
2374         }
2375         vma = xe_vma_create(vm, bo, op->gem.offset,
2376                             op->va.addr, op->va.addr +
2377                             op->va.range - 1, read_only, is_null,
2378                             tile_mask);
2379         if (bo)
2380                 xe_bo_unlock(bo, &ww);
2381
2382         if (xe_vma_is_userptr(vma)) {
2383                 err = xe_vma_userptr_pin_pages(vma);
2384                 if (err) {
2385                         prep_vma_destroy(vm, vma, false);
2386                         xe_vma_destroy_unlocked(vma);
2387                         return ERR_PTR(err);
2388                 }
2389         } else if (!xe_vma_has_no_bo(vma) && !bo->vm) {
2390                 vm_insert_extobj(vm, vma);
2391                 err = add_preempt_fences(vm, bo);
2392                 if (err) {
2393                         prep_vma_destroy(vm, vma, false);
2394                         xe_vma_destroy_unlocked(vma);
2395                         return ERR_PTR(err);
2396                 }
2397         }
2398
2399         return vma;
2400 }
2401
2402 static u64 xe_vma_max_pte_size(struct xe_vma *vma)
2403 {
2404         if (vma->gpuva.flags & XE_VMA_PTE_1G)
2405                 return SZ_1G;
2406         else if (vma->gpuva.flags & XE_VMA_PTE_2M)
2407                 return SZ_2M;
2408
2409         return SZ_4K;
2410 }
2411
2412 /*
2413  * Parse operations list and create any resources needed for the operations
2414  * prior to fully committing to the operations. This setup can fail.
2415  */
2416 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_engine *e,
2417                                    struct drm_gpuva_ops **ops, int num_ops_list,
2418                                    struct xe_sync_entry *syncs, u32 num_syncs,
2419                                    struct list_head *ops_list, bool async)
2420 {
2421         struct xe_vma_op *last_op = NULL;
2422         struct list_head *async_list = NULL;
2423         struct async_op_fence *fence = NULL;
2424         int err, i;
2425
2426         lockdep_assert_held_write(&vm->lock);
2427         XE_BUG_ON(num_ops_list > 1 && !async);
2428
2429         if (num_syncs && async) {
2430                 u64 seqno;
2431
2432                 fence = kmalloc(sizeof(*fence), GFP_KERNEL);
2433                 if (!fence)
2434                         return -ENOMEM;
2435
2436                 seqno = e ? ++e->bind.fence_seqno : ++vm->async_ops.fence.seqno;
2437                 dma_fence_init(&fence->fence, &async_op_fence_ops,
2438                                &vm->async_ops.lock, e ? e->bind.fence_ctx :
2439                                vm->async_ops.fence.context, seqno);
2440
2441                 if (!xe_vm_no_dma_fences(vm)) {
2442                         fence->vm = vm;
2443                         fence->started = false;
2444                         init_waitqueue_head(&fence->wq);
2445                 }
2446         }
2447
2448         for (i = 0; i < num_ops_list; ++i) {
2449                 struct drm_gpuva_ops *__ops = ops[i];
2450                 struct drm_gpuva_op *__op;
2451
2452                 drm_gpuva_for_each_op(__op, __ops) {
2453                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2454                         bool first = !async_list;
2455
2456                         XE_BUG_ON(!first && !async);
2457
2458                         INIT_LIST_HEAD(&op->link);
2459                         if (first)
2460                                 async_list = ops_list;
2461                         list_add_tail(&op->link, async_list);
2462
2463                         if (first) {
2464                                 op->flags |= XE_VMA_OP_FIRST;
2465                                 op->num_syncs = num_syncs;
2466                                 op->syncs = syncs;
2467                         }
2468
2469                         op->engine = e;
2470
2471                         switch (op->base.op) {
2472                         case DRM_GPUVA_OP_MAP:
2473                         {
2474                                 struct xe_vma *vma;
2475
2476                                 vma = new_vma(vm, &op->base.map,
2477                                               op->tile_mask, op->map.read_only,
2478                                               op->map.is_null);
2479                                 if (IS_ERR(vma)) {
2480                                         err = PTR_ERR(vma);
2481                                         goto free_fence;
2482                                 }
2483
2484                                 op->map.vma = vma;
2485                                 break;
2486                         }
2487                         case DRM_GPUVA_OP_REMAP:
2488                         {
2489                                 struct xe_vma *old =
2490                                         gpuva_to_vma(op->base.remap.unmap->va);
2491
2492                                 op->remap.start = xe_vma_start(old);
2493                                 op->remap.range = xe_vma_size(old);
2494
2495                                 if (op->base.remap.prev) {
2496                                         struct xe_vma *vma;
2497                                         bool read_only =
2498                                                 op->base.remap.unmap->va->flags &
2499                                                 XE_VMA_READ_ONLY;
2500                                         bool is_null =
2501                                                 op->base.remap.unmap->va->flags &
2502                                                 DRM_GPUVA_SPARSE;
2503
2504                                         vma = new_vma(vm, op->base.remap.prev,
2505                                                       op->tile_mask, read_only,
2506                                                       is_null);
2507                                         if (IS_ERR(vma)) {
2508                                                 err = PTR_ERR(vma);
2509                                                 goto free_fence;
2510                                         }
2511
2512                                         op->remap.prev = vma;
2513
2514                                         /*
2515                                          * Userptr creates a new SG mapping so
2516                                          * we must also rebind.
2517                                          */
2518                                         op->remap.skip_prev = !xe_vma_is_userptr(old) &&
2519                                                 IS_ALIGNED(xe_vma_end(vma),
2520                                                            xe_vma_max_pte_size(old));
2521                                         if (op->remap.skip_prev) {
2522                                                 op->remap.range -=
2523                                                         xe_vma_end(vma) -
2524                                                         xe_vma_start(old);
2525                                                 op->remap.start = xe_vma_end(vma);
2526                                         }
2527                                 }
2528
2529                                 if (op->base.remap.next) {
2530                                         struct xe_vma *vma;
2531                                         bool read_only =
2532                                                 op->base.remap.unmap->va->flags &
2533                                                 XE_VMA_READ_ONLY;
2534
2535                                         bool is_null =
2536                                                 op->base.remap.unmap->va->flags &
2537                                                 DRM_GPUVA_SPARSE;
2538
2539                                         vma = new_vma(vm, op->base.remap.next,
2540                                                       op->tile_mask, read_only,
2541                                                       is_null);
2542                                         if (IS_ERR(vma)) {
2543                                                 err = PTR_ERR(vma);
2544                                                 goto free_fence;
2545                                         }
2546
2547                                         op->remap.next = vma;
2548
2549                                         /*
2550                                          * Userptr creates a new SG mapping so
2551                                          * we must also rebind.
2552                                          */
2553                                         op->remap.skip_next = !xe_vma_is_userptr(old) &&
2554                                                 IS_ALIGNED(xe_vma_start(vma),
2555                                                            xe_vma_max_pte_size(old));
2556                                         if (op->remap.skip_next)
2557                                                 op->remap.range -=
2558                                                         xe_vma_end(old) -
2559                                                         xe_vma_start(vma);
2560                                 }
2561                                 break;
2562                         }
2563                         case DRM_GPUVA_OP_UNMAP:
2564                         case DRM_GPUVA_OP_PREFETCH:
2565                                 /* Nothing to do */
2566                                 break;
2567                         default:
2568                                 XE_BUG_ON("NOT POSSIBLE");
2569                         }
2570
2571                         last_op = op;
2572                 }
2573
2574                 last_op->ops = __ops;
2575         }
2576
2577         if (!last_op)
2578                 return -ENODATA;
2579
2580         last_op->flags |= XE_VMA_OP_LAST;
2581         last_op->num_syncs = num_syncs;
2582         last_op->syncs = syncs;
2583         last_op->fence = fence;
2584
2585         return 0;
2586
2587 free_fence:
2588         kfree(fence);
2589         return err;
2590 }
2591
2592 static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
2593 {
2594         int err = 0;
2595
2596         lockdep_assert_held_write(&vm->lock);
2597
2598         switch (op->base.op) {
2599         case DRM_GPUVA_OP_MAP:
2600                 err |= xe_vm_insert_vma(vm, op->map.vma);
2601                 break;
2602         case DRM_GPUVA_OP_REMAP:
2603                 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va),
2604                                  true);
2605
2606                 if (op->remap.prev) {
2607                         err |= xe_vm_insert_vma(vm, op->remap.prev);
2608                         if (!err && op->remap.skip_prev)
2609                                 op->remap.prev = NULL;
2610                 }
2611                 if (op->remap.next) {
2612                         err |= xe_vm_insert_vma(vm, op->remap.next);
2613                         if (!err && op->remap.skip_next)
2614                                 op->remap.next = NULL;
2615                 }
2616
2617                 /* Adjust for partial unbind after removin VMA from VM */
2618                 if (!err) {
2619                         op->base.remap.unmap->va->va.addr = op->remap.start;
2620                         op->base.remap.unmap->va->va.range = op->remap.range;
2621                 }
2622                 break;
2623         case DRM_GPUVA_OP_UNMAP:
2624                 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true);
2625                 break;
2626         case DRM_GPUVA_OP_PREFETCH:
2627                 /* Nothing to do */
2628                 break;
2629         default:
2630                 XE_BUG_ON("NOT POSSIBLE");
2631         }
2632
2633         op->flags |= XE_VMA_OP_COMMITTED;
2634         return err;
2635 }
2636
2637 static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
2638                                struct xe_vma_op *op)
2639 {
2640         LIST_HEAD(objs);
2641         LIST_HEAD(dups);
2642         struct ttm_validate_buffer tv_bo, tv_vm;
2643         struct ww_acquire_ctx ww;
2644         struct xe_bo *vbo;
2645         int err;
2646
2647         lockdep_assert_held_write(&vm->lock);
2648
2649         xe_vm_tv_populate(vm, &tv_vm);
2650         list_add_tail(&tv_vm.head, &objs);
2651         vbo = xe_vma_bo(vma);
2652         if (vbo) {
2653                 /*
2654                  * An unbind can drop the last reference to the BO and
2655                  * the BO is needed for ttm_eu_backoff_reservation so
2656                  * take a reference here.
2657                  */
2658                 xe_bo_get(vbo);
2659
2660                 if (!vbo->vm) {
2661                         tv_bo.bo = &vbo->ttm;
2662                         tv_bo.num_shared = 1;
2663                         list_add(&tv_bo.head, &objs);
2664                 }
2665         }
2666
2667 again:
2668         err = ttm_eu_reserve_buffers(&ww, &objs, true, &dups);
2669         if (err) {
2670                 xe_bo_put(vbo);
2671                 return err;
2672         }
2673
2674         xe_vm_assert_held(vm);
2675         xe_bo_assert_held(xe_vma_bo(vma));
2676
2677         switch (op->base.op) {
2678         case DRM_GPUVA_OP_MAP:
2679                 err = xe_vm_bind(vm, vma, op->engine, xe_vma_bo(vma),
2680                                  op->syncs, op->num_syncs, op->fence,
2681                                  op->map.immediate || !xe_vm_in_fault_mode(vm),
2682                                  op->flags & XE_VMA_OP_FIRST,
2683                                  op->flags & XE_VMA_OP_LAST);
2684                 break;
2685         case DRM_GPUVA_OP_REMAP:
2686         {
2687                 bool prev = !!op->remap.prev;
2688                 bool next = !!op->remap.next;
2689
2690                 if (!op->remap.unmap_done) {
2691                         if (prev || next) {
2692                                 vm->async_ops.munmap_rebind_inflight = true;
2693                                 vma->gpuva.flags |= XE_VMA_FIRST_REBIND;
2694                         }
2695                         err = xe_vm_unbind(vm, vma, op->engine, op->syncs,
2696                                            op->num_syncs,
2697                                            !prev && !next ? op->fence : NULL,
2698                                            op->flags & XE_VMA_OP_FIRST,
2699                                            op->flags & XE_VMA_OP_LAST && !prev &&
2700                                            !next);
2701                         if (err)
2702                                 break;
2703                         op->remap.unmap_done = true;
2704                 }
2705
2706                 if (prev) {
2707                         op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND;
2708                         err = xe_vm_bind(vm, op->remap.prev, op->engine,
2709                                          xe_vma_bo(op->remap.prev), op->syncs,
2710                                          op->num_syncs,
2711                                          !next ? op->fence : NULL, true, false,
2712                                          op->flags & XE_VMA_OP_LAST && !next);
2713                         op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
2714                         if (err)
2715                                 break;
2716                         op->remap.prev = NULL;
2717                 }
2718
2719                 if (next) {
2720                         op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND;
2721                         err = xe_vm_bind(vm, op->remap.next, op->engine,
2722                                          xe_vma_bo(op->remap.next),
2723                                          op->syncs, op->num_syncs,
2724                                          op->fence, true, false,
2725                                          op->flags & XE_VMA_OP_LAST);
2726                         op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
2727                         if (err)
2728                                 break;
2729                         op->remap.next = NULL;
2730                 }
2731                 vm->async_ops.munmap_rebind_inflight = false;
2732
2733                 break;
2734         }
2735         case DRM_GPUVA_OP_UNMAP:
2736                 err = xe_vm_unbind(vm, vma, op->engine, op->syncs,
2737                                    op->num_syncs, op->fence,
2738                                    op->flags & XE_VMA_OP_FIRST,
2739                                    op->flags & XE_VMA_OP_LAST);
2740                 break;
2741         case DRM_GPUVA_OP_PREFETCH:
2742                 err = xe_vm_prefetch(vm, vma, op->engine, op->prefetch.region,
2743                                      op->syncs, op->num_syncs, op->fence,
2744                                      op->flags & XE_VMA_OP_FIRST,
2745                                      op->flags & XE_VMA_OP_LAST);
2746                 break;
2747         default:
2748                 XE_BUG_ON("NOT POSSIBLE");
2749         }
2750
2751         ttm_eu_backoff_reservation(&ww, &objs);
2752         if (err == -EAGAIN && xe_vma_is_userptr(vma)) {
2753                 lockdep_assert_held_write(&vm->lock);
2754                 err = xe_vma_userptr_pin_pages(vma);
2755                 if (!err)
2756                         goto again;
2757         }
2758         xe_bo_put(vbo);
2759
2760         if (err)
2761                 trace_xe_vma_fail(vma);
2762
2763         return err;
2764 }
2765
2766 static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
2767 {
2768         int ret = 0;
2769
2770         lockdep_assert_held_write(&vm->lock);
2771
2772 #ifdef TEST_VM_ASYNC_OPS_ERROR
2773         if (op->inject_error) {
2774                 op->inject_error = false;
2775                 return -ENOMEM;
2776         }
2777 #endif
2778
2779         switch (op->base.op) {
2780         case DRM_GPUVA_OP_MAP:
2781                 ret = __xe_vma_op_execute(vm, op->map.vma, op);
2782                 break;
2783         case DRM_GPUVA_OP_REMAP:
2784         {
2785                 struct xe_vma *vma;
2786
2787                 if (!op->remap.unmap_done)
2788                         vma = gpuva_to_vma(op->base.remap.unmap->va);
2789                 else if (op->remap.prev)
2790                         vma = op->remap.prev;
2791                 else
2792                         vma = op->remap.next;
2793
2794                 ret = __xe_vma_op_execute(vm, vma, op);
2795                 break;
2796         }
2797         case DRM_GPUVA_OP_UNMAP:
2798                 ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
2799                                           op);
2800                 break;
2801         case DRM_GPUVA_OP_PREFETCH:
2802                 ret = __xe_vma_op_execute(vm,
2803                                           gpuva_to_vma(op->base.prefetch.va),
2804                                           op);
2805                 break;
2806         default:
2807                 XE_BUG_ON("NOT POSSIBLE");
2808         }
2809
2810         return ret;
2811 }
2812
2813 static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
2814 {
2815         bool last = op->flags & XE_VMA_OP_LAST;
2816
2817         if (last) {
2818                 while (op->num_syncs--)
2819                         xe_sync_entry_cleanup(&op->syncs[op->num_syncs]);
2820                 kfree(op->syncs);
2821                 if (op->engine)
2822                         xe_engine_put(op->engine);
2823                 if (op->fence)
2824                         dma_fence_put(&op->fence->fence);
2825         }
2826         if (!list_empty(&op->link)) {
2827                 spin_lock_irq(&vm->async_ops.lock);
2828                 list_del(&op->link);
2829                 spin_unlock_irq(&vm->async_ops.lock);
2830         }
2831         if (op->ops)
2832                 drm_gpuva_ops_free(&vm->gpuvm, op->ops);
2833         if (last)
2834                 xe_vm_put(vm);
2835 }
2836
2837 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
2838                              bool post_commit)
2839 {
2840         lockdep_assert_held_write(&vm->lock);
2841
2842         switch (op->base.op) {
2843         case DRM_GPUVA_OP_MAP:
2844                 if (op->map.vma) {
2845                         prep_vma_destroy(vm, op->map.vma, post_commit);
2846                         xe_vma_destroy_unlocked(op->map.vma);
2847                 }
2848                 break;
2849         case DRM_GPUVA_OP_UNMAP:
2850         {
2851                 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
2852
2853                 down_read(&vm->userptr.notifier_lock);
2854                 vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2855                 up_read(&vm->userptr.notifier_lock);
2856                 if (post_commit)
2857                         xe_vm_insert_vma(vm, vma);
2858                 break;
2859         }
2860         case DRM_GPUVA_OP_REMAP:
2861         {
2862                 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va);
2863
2864                 if (op->remap.prev) {
2865                         prep_vma_destroy(vm, op->remap.prev, post_commit);
2866                         xe_vma_destroy_unlocked(op->remap.prev);
2867                 }
2868                 if (op->remap.next) {
2869                         prep_vma_destroy(vm, op->remap.next, post_commit);
2870                         xe_vma_destroy_unlocked(op->remap.next);
2871                 }
2872                 down_read(&vm->userptr.notifier_lock);
2873                 vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2874                 up_read(&vm->userptr.notifier_lock);
2875                 if (post_commit)
2876                         xe_vm_insert_vma(vm, vma);
2877                 break;
2878         }
2879         case DRM_GPUVA_OP_PREFETCH:
2880                 /* Nothing to do */
2881                 break;
2882         default:
2883                 XE_BUG_ON("NOT POSSIBLE");
2884         }
2885 }
2886
2887 static struct xe_vma_op *next_vma_op(struct xe_vm *vm)
2888 {
2889         return list_first_entry_or_null(&vm->async_ops.pending,
2890                                         struct xe_vma_op, link);
2891 }
2892
2893 static void xe_vma_op_work_func(struct work_struct *w)
2894 {
2895         struct xe_vm *vm = container_of(w, struct xe_vm, async_ops.work);
2896
2897         for (;;) {
2898                 struct xe_vma_op *op;
2899                 int err;
2900
2901                 if (vm->async_ops.error && !xe_vm_is_closed(vm))
2902                         break;
2903
2904                 spin_lock_irq(&vm->async_ops.lock);
2905                 op = next_vma_op(vm);
2906                 spin_unlock_irq(&vm->async_ops.lock);
2907
2908                 if (!op)
2909                         break;
2910
2911                 if (!xe_vm_is_closed(vm)) {
2912                         down_write(&vm->lock);
2913                         err = xe_vma_op_execute(vm, op);
2914                         if (err) {
2915                                 drm_warn(&vm->xe->drm,
2916                                          "Async VM op(%d) failed with %d",
2917                                          op->base.op, err);
2918                                 vm_set_async_error(vm, err);
2919                                 up_write(&vm->lock);
2920
2921                                 if (vm->async_ops.error_capture.addr)
2922                                         vm_error_capture(vm, err, 0, 0, 0);
2923                                 break;
2924                         }
2925                         up_write(&vm->lock);
2926                 } else {
2927                         struct xe_vma *vma;
2928
2929                         switch (op->base.op) {
2930                         case DRM_GPUVA_OP_REMAP:
2931                                 vma = gpuva_to_vma(op->base.remap.unmap->va);
2932                                 trace_xe_vma_flush(vma);
2933
2934                                 down_write(&vm->lock);
2935                                 xe_vma_destroy_unlocked(vma);
2936                                 up_write(&vm->lock);
2937                                 break;
2938                         case DRM_GPUVA_OP_UNMAP:
2939                                 vma = gpuva_to_vma(op->base.unmap.va);
2940                                 trace_xe_vma_flush(vma);
2941
2942                                 down_write(&vm->lock);
2943                                 xe_vma_destroy_unlocked(vma);
2944                                 up_write(&vm->lock);
2945                                 break;
2946                         default:
2947                                 /* Nothing to do */
2948                                 break;
2949                         }
2950
2951                         if (op->fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
2952                                                    &op->fence->fence.flags)) {
2953                                 if (!xe_vm_no_dma_fences(vm)) {
2954                                         op->fence->started = true;
2955                                         wake_up_all(&op->fence->wq);
2956                                 }
2957                                 dma_fence_signal(&op->fence->fence);
2958                         }
2959                 }
2960
2961                 xe_vma_op_cleanup(vm, op);
2962         }
2963 }
2964
2965 static int vm_bind_ioctl_ops_commit(struct xe_vm *vm,
2966                                     struct list_head *ops_list, bool async)
2967 {
2968         struct xe_vma_op *op, *last_op, *next;
2969         int err;
2970
2971         lockdep_assert_held_write(&vm->lock);
2972
2973         list_for_each_entry(op, ops_list, link) {
2974                 last_op = op;
2975                 err = xe_vma_op_commit(vm, op);
2976                 if (err)
2977                         goto unwind;
2978         }
2979
2980         if (!async) {
2981                 err = xe_vma_op_execute(vm, last_op);
2982                 if (err)
2983                         goto unwind;
2984                 xe_vma_op_cleanup(vm, last_op);
2985         } else {
2986                 int i;
2987                 bool installed = false;
2988
2989                 for (i = 0; i < last_op->num_syncs; i++)
2990                         installed |= xe_sync_entry_signal(&last_op->syncs[i],
2991                                                           NULL,
2992                                                           &last_op->fence->fence);
2993                 if (!installed && last_op->fence)
2994                         dma_fence_signal(&last_op->fence->fence);
2995
2996                 spin_lock_irq(&vm->async_ops.lock);
2997                 list_splice_tail(ops_list, &vm->async_ops.pending);
2998                 spin_unlock_irq(&vm->async_ops.lock);
2999
3000                 if (!vm->async_ops.error)
3001                         queue_work(system_unbound_wq, &vm->async_ops.work);
3002         }
3003
3004         return 0;
3005
3006 unwind:
3007         list_for_each_entry_reverse(op, ops_list, link)
3008                 xe_vma_op_unwind(vm, op, op->flags & XE_VMA_OP_COMMITTED);
3009         list_for_each_entry_safe(op, next, ops_list, link)
3010                 xe_vma_op_cleanup(vm, op);
3011
3012         return err;
3013 }
3014
3015 /*
3016  * Unwind operations list, called after a failure of vm_bind_ioctl_ops_create or
3017  * vm_bind_ioctl_ops_parse.
3018  */
3019 static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
3020                                      struct drm_gpuva_ops **ops,
3021                                      int num_ops_list)
3022 {
3023         int i;
3024
3025         for (i = 0; i < num_ops_list; ++i) {
3026                 struct drm_gpuva_ops *__ops = ops[i];
3027                 struct drm_gpuva_op *__op;
3028
3029                 if (!__ops)
3030                         continue;
3031
3032                 drm_gpuva_for_each_op(__op, __ops) {
3033                         struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
3034
3035                         xe_vma_op_unwind(vm, op, false);
3036                 }
3037         }
3038 }
3039
3040 #ifdef TEST_VM_ASYNC_OPS_ERROR
3041 #define SUPPORTED_FLAGS \
3042         (FORCE_ASYNC_OP_ERROR | XE_VM_BIND_FLAG_ASYNC | \
3043          XE_VM_BIND_FLAG_READONLY | XE_VM_BIND_FLAG_IMMEDIATE | \
3044          XE_VM_BIND_FLAG_NULL | 0xffff)
3045 #else
3046 #define SUPPORTED_FLAGS \
3047         (XE_VM_BIND_FLAG_ASYNC | XE_VM_BIND_FLAG_READONLY | \
3048          XE_VM_BIND_FLAG_IMMEDIATE | XE_VM_BIND_FLAG_NULL | 0xffff)
3049 #endif
3050 #define XE_64K_PAGE_MASK 0xffffull
3051
3052 #define MAX_BINDS       512     /* FIXME: Picking random upper limit */
3053
3054 static int vm_bind_ioctl_check_args(struct xe_device *xe,
3055                                     struct drm_xe_vm_bind *args,
3056                                     struct drm_xe_vm_bind_op **bind_ops,
3057                                     bool *async)
3058 {
3059         int err;
3060         int i;
3061
3062         if (XE_IOCTL_DBG(xe, args->extensions) ||
3063             XE_IOCTL_DBG(xe, !args->num_binds) ||
3064             XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS))
3065                 return -EINVAL;
3066
3067         if (args->num_binds > 1) {
3068                 u64 __user *bind_user =
3069                         u64_to_user_ptr(args->vector_of_binds);
3070
3071                 *bind_ops = kmalloc(sizeof(struct drm_xe_vm_bind_op) *
3072                                     args->num_binds, GFP_KERNEL);
3073                 if (!*bind_ops)
3074                         return -ENOMEM;
3075
3076                 err = __copy_from_user(*bind_ops, bind_user,
3077                                        sizeof(struct drm_xe_vm_bind_op) *
3078                                        args->num_binds);
3079                 if (XE_IOCTL_DBG(xe, err)) {
3080                         err = -EFAULT;
3081                         goto free_bind_ops;
3082                 }
3083         } else {
3084                 *bind_ops = &args->bind;
3085         }
3086
3087         for (i = 0; i < args->num_binds; ++i) {
3088                 u64 range = (*bind_ops)[i].range;
3089                 u64 addr = (*bind_ops)[i].addr;
3090                 u32 op = (*bind_ops)[i].op;
3091                 u32 obj = (*bind_ops)[i].obj;
3092                 u64 obj_offset = (*bind_ops)[i].obj_offset;
3093                 u32 region = (*bind_ops)[i].region;
3094                 bool is_null = op & XE_VM_BIND_FLAG_NULL;
3095
3096                 if (i == 0) {
3097                         *async = !!(op & XE_VM_BIND_FLAG_ASYNC);
3098                 } else if (XE_IOCTL_DBG(xe, !*async) ||
3099                            XE_IOCTL_DBG(xe, !(op & XE_VM_BIND_FLAG_ASYNC)) ||
3100                            XE_IOCTL_DBG(xe, VM_BIND_OP(op) ==
3101                                         XE_VM_BIND_OP_RESTART)) {
3102                         err = -EINVAL;
3103                         goto free_bind_ops;
3104                 }
3105
3106                 if (XE_IOCTL_DBG(xe, !*async &&
3107                                  VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL)) {
3108                         err = -EINVAL;
3109                         goto free_bind_ops;
3110                 }
3111
3112                 if (XE_IOCTL_DBG(xe, !*async &&
3113                                  VM_BIND_OP(op) == XE_VM_BIND_OP_PREFETCH)) {
3114                         err = -EINVAL;
3115                         goto free_bind_ops;
3116                 }
3117
3118                 if (XE_IOCTL_DBG(xe, VM_BIND_OP(op) >
3119                                  XE_VM_BIND_OP_PREFETCH) ||
3120                     XE_IOCTL_DBG(xe, op & ~SUPPORTED_FLAGS) ||
3121                     XE_IOCTL_DBG(xe, obj && is_null) ||
3122                     XE_IOCTL_DBG(xe, obj_offset && is_null) ||
3123                     XE_IOCTL_DBG(xe, VM_BIND_OP(op) != XE_VM_BIND_OP_MAP &&
3124                                  is_null) ||
3125                     XE_IOCTL_DBG(xe, !obj &&
3126                                  VM_BIND_OP(op) == XE_VM_BIND_OP_MAP &&
3127                                  !is_null) ||
3128                     XE_IOCTL_DBG(xe, !obj &&
3129                                  VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) ||
3130                     XE_IOCTL_DBG(xe, addr &&
3131                                  VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) ||
3132                     XE_IOCTL_DBG(xe, range &&
3133                                  VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) ||
3134                     XE_IOCTL_DBG(xe, obj &&
3135                                  VM_BIND_OP(op) == XE_VM_BIND_OP_MAP_USERPTR) ||
3136                     XE_IOCTL_DBG(xe, obj &&
3137                                  VM_BIND_OP(op) == XE_VM_BIND_OP_PREFETCH) ||
3138                     XE_IOCTL_DBG(xe, region &&
3139                                  VM_BIND_OP(op) != XE_VM_BIND_OP_PREFETCH) ||
3140                     XE_IOCTL_DBG(xe, !(BIT(region) &
3141                                        xe->info.mem_region_mask)) ||
3142                     XE_IOCTL_DBG(xe, obj &&
3143                                  VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP)) {
3144                         err = -EINVAL;
3145                         goto free_bind_ops;
3146                 }
3147
3148                 if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) ||
3149                     XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) ||
3150                     XE_IOCTL_DBG(xe, range & ~PAGE_MASK) ||
3151                     XE_IOCTL_DBG(xe, !range && VM_BIND_OP(op) !=
3152                                  XE_VM_BIND_OP_RESTART &&
3153                                  VM_BIND_OP(op) != XE_VM_BIND_OP_UNMAP_ALL)) {
3154                         err = -EINVAL;
3155                         goto free_bind_ops;
3156                 }
3157         }
3158
3159         return 0;
3160
3161 free_bind_ops:
3162         if (args->num_binds > 1)
3163                 kfree(*bind_ops);
3164         return err;
3165 }
3166
3167 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3168 {
3169         struct xe_device *xe = to_xe_device(dev);
3170         struct xe_file *xef = to_xe_file(file);
3171         struct drm_xe_vm_bind *args = data;
3172         struct drm_xe_sync __user *syncs_user;
3173         struct xe_bo **bos = NULL;
3174         struct drm_gpuva_ops **ops = NULL;
3175         struct xe_vm *vm;
3176         struct xe_engine *e = NULL;
3177         u32 num_syncs;
3178         struct xe_sync_entry *syncs = NULL;
3179         struct drm_xe_vm_bind_op *bind_ops;
3180         LIST_HEAD(ops_list);
3181         bool async;
3182         int err;
3183         int i;
3184
3185         err = vm_bind_ioctl_check_args(xe, args, &bind_ops, &async);
3186         if (err)
3187                 return err;
3188
3189         if (args->engine_id) {
3190                 e = xe_engine_lookup(xef, args->engine_id);
3191                 if (XE_IOCTL_DBG(xe, !e)) {
3192                         err = -ENOENT;
3193                         goto free_objs;
3194                 }
3195
3196                 if (XE_IOCTL_DBG(xe, !(e->flags & ENGINE_FLAG_VM))) {
3197                         err = -EINVAL;
3198                         goto put_engine;
3199                 }
3200         }
3201
3202         vm = xe_vm_lookup(xef, args->vm_id);
3203         if (XE_IOCTL_DBG(xe, !vm)) {
3204                 err = -EINVAL;
3205                 goto put_engine;
3206         }
3207
3208         err = down_write_killable(&vm->lock);
3209         if (err)
3210                 goto put_vm;
3211
3212         if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
3213                 err = -ENOENT;
3214                 goto release_vm_lock;
3215         }
3216
3217         if (VM_BIND_OP(bind_ops[0].op) == XE_VM_BIND_OP_RESTART) {
3218                 if (XE_IOCTL_DBG(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS)))
3219                         err = -EOPNOTSUPP;
3220                 if (XE_IOCTL_DBG(xe, !err && args->num_syncs))
3221                         err = EINVAL;
3222                 if (XE_IOCTL_DBG(xe, !err && !vm->async_ops.error))
3223                         err = -EPROTO;
3224
3225                 if (!err) {
3226                         trace_xe_vm_restart(vm);
3227                         vm_set_async_error(vm, 0);
3228
3229                         queue_work(system_unbound_wq, &vm->async_ops.work);
3230
3231                         /* Rebinds may have been blocked, give worker a kick */
3232                         if (xe_vm_in_compute_mode(vm))
3233                                 xe_vm_queue_rebind_worker(vm);
3234                 }
3235
3236                 goto release_vm_lock;
3237         }
3238
3239         if (XE_IOCTL_DBG(xe, !vm->async_ops.error &&
3240                          async != !!(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) {
3241                 err = -EOPNOTSUPP;
3242                 goto release_vm_lock;
3243         }
3244
3245         for (i = 0; i < args->num_binds; ++i) {
3246                 u64 range = bind_ops[i].range;
3247                 u64 addr = bind_ops[i].addr;
3248
3249                 if (XE_IOCTL_DBG(xe, range > vm->size) ||
3250                     XE_IOCTL_DBG(xe, addr > vm->size - range)) {
3251                         err = -EINVAL;
3252                         goto release_vm_lock;
3253                 }
3254
3255                 if (bind_ops[i].tile_mask) {
3256                         u64 valid_tiles = BIT(xe->info.tile_count) - 1;
3257
3258                         if (XE_IOCTL_DBG(xe, bind_ops[i].tile_mask &
3259                                          ~valid_tiles)) {
3260                                 err = -EINVAL;
3261                                 goto release_vm_lock;
3262                         }
3263                 }
3264         }
3265
3266         bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL);
3267         if (!bos) {
3268                 err = -ENOMEM;
3269                 goto release_vm_lock;
3270         }
3271
3272         ops = kzalloc(sizeof(*ops) * args->num_binds, GFP_KERNEL);
3273         if (!ops) {
3274                 err = -ENOMEM;
3275                 goto release_vm_lock;
3276         }
3277
3278         for (i = 0; i < args->num_binds; ++i) {
3279                 struct drm_gem_object *gem_obj;
3280                 u64 range = bind_ops[i].range;
3281                 u64 addr = bind_ops[i].addr;
3282                 u32 obj = bind_ops[i].obj;
3283                 u64 obj_offset = bind_ops[i].obj_offset;
3284
3285                 if (!obj)
3286                         continue;
3287
3288                 gem_obj = drm_gem_object_lookup(file, obj);
3289                 if (XE_IOCTL_DBG(xe, !gem_obj)) {
3290                         err = -ENOENT;
3291                         goto put_obj;
3292                 }
3293                 bos[i] = gem_to_xe_bo(gem_obj);
3294
3295                 if (XE_IOCTL_DBG(xe, range > bos[i]->size) ||
3296                     XE_IOCTL_DBG(xe, obj_offset >
3297                                  bos[i]->size - range)) {
3298                         err = -EINVAL;
3299                         goto put_obj;
3300                 }
3301
3302                 if (bos[i]->flags & XE_BO_INTERNAL_64K) {
3303                         if (XE_IOCTL_DBG(xe, obj_offset &
3304                                          XE_64K_PAGE_MASK) ||
3305                             XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) ||
3306                             XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) {
3307                                 err = -EINVAL;
3308                                 goto put_obj;
3309                         }
3310                 }
3311         }
3312
3313         if (args->num_syncs) {
3314                 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL);
3315                 if (!syncs) {
3316                         err = -ENOMEM;
3317                         goto put_obj;
3318                 }
3319         }
3320
3321         syncs_user = u64_to_user_ptr(args->syncs);
3322         for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
3323                 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
3324                                           &syncs_user[num_syncs], false,
3325                                           xe_vm_no_dma_fences(vm));
3326                 if (err)
3327                         goto free_syncs;
3328         }
3329
3330         /* Do some error checking first to make the unwind easier */
3331         for (i = 0; i < args->num_binds; ++i) {
3332                 u64 range = bind_ops[i].range;
3333                 u64 addr = bind_ops[i].addr;
3334                 u32 op = bind_ops[i].op;
3335
3336                 err = vm_bind_ioctl_lookup_vma(vm, bos[i], addr, range, op);
3337                 if (err)
3338                         goto free_syncs;
3339         }
3340
3341         for (i = 0; i < args->num_binds; ++i) {
3342                 u64 range = bind_ops[i].range;
3343                 u64 addr = bind_ops[i].addr;
3344                 u32 op = bind_ops[i].op;
3345                 u64 obj_offset = bind_ops[i].obj_offset;
3346                 u8 tile_mask = bind_ops[i].tile_mask;
3347                 u32 region = bind_ops[i].region;
3348
3349                 ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset,
3350                                                   addr, range, op, tile_mask,
3351                                                   region);
3352                 if (IS_ERR(ops[i])) {
3353                         err = PTR_ERR(ops[i]);
3354                         ops[i] = NULL;
3355                         goto unwind_ops;
3356                 }
3357         }
3358
3359         err = vm_bind_ioctl_ops_parse(vm, e, ops, args->num_binds,
3360                                       syncs, num_syncs, &ops_list, async);
3361         if (err)
3362                 goto unwind_ops;
3363
3364         err = vm_bind_ioctl_ops_commit(vm, &ops_list, async);
3365         up_write(&vm->lock);
3366
3367         for (i = 0; i < args->num_binds; ++i)
3368                 xe_bo_put(bos[i]);
3369
3370         kfree(bos);
3371         kfree(ops);
3372         if (args->num_binds > 1)
3373                 kfree(bind_ops);
3374
3375         return err;
3376
3377 unwind_ops:
3378         vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
3379 free_syncs:
3380         while (num_syncs--)
3381                 xe_sync_entry_cleanup(&syncs[num_syncs]);
3382
3383         kfree(syncs);
3384 put_obj:
3385         for (i = 0; i < args->num_binds; ++i)
3386                 xe_bo_put(bos[i]);
3387 release_vm_lock:
3388         up_write(&vm->lock);
3389 put_vm:
3390         xe_vm_put(vm);
3391 put_engine:
3392         if (e)
3393                 xe_engine_put(e);
3394 free_objs:
3395         kfree(bos);
3396         kfree(ops);
3397         if (args->num_binds > 1)
3398                 kfree(bind_ops);
3399         return err == -ENODATA ? 0 : err;
3400 }
3401
3402 /*
3403  * XXX: Using the TTM wrappers for now, likely can call into dma-resv code
3404  * directly to optimize. Also this likely should be an inline function.
3405  */
3406 int xe_vm_lock(struct xe_vm *vm, struct ww_acquire_ctx *ww,
3407                int num_resv, bool intr)
3408 {
3409         struct ttm_validate_buffer tv_vm;
3410         LIST_HEAD(objs);
3411         LIST_HEAD(dups);
3412
3413         XE_BUG_ON(!ww);
3414
3415         tv_vm.num_shared = num_resv;
3416         tv_vm.bo = xe_vm_ttm_bo(vm);
3417         list_add_tail(&tv_vm.head, &objs);
3418
3419         return ttm_eu_reserve_buffers(ww, &objs, intr, &dups);
3420 }
3421
3422 void xe_vm_unlock(struct xe_vm *vm, struct ww_acquire_ctx *ww)
3423 {
3424         dma_resv_unlock(xe_vm_resv(vm));
3425         ww_acquire_fini(ww);
3426 }
3427
3428 /**
3429  * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
3430  * @vma: VMA to invalidate
3431  *
3432  * Walks a list of page tables leaves which it memset the entries owned by this
3433  * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
3434  * complete.
3435  *
3436  * Returns 0 for success, negative error code otherwise.
3437  */
3438 int xe_vm_invalidate_vma(struct xe_vma *vma)
3439 {
3440         struct xe_device *xe = xe_vma_vm(vma)->xe;
3441         struct xe_tile *tile;
3442         u32 tile_needs_invalidate = 0;
3443         int seqno[XE_MAX_TILES_PER_DEVICE];
3444         u8 id;
3445         int ret;
3446
3447         XE_BUG_ON(!xe_vm_in_fault_mode(xe_vma_vm(vma)));
3448         XE_WARN_ON(xe_vma_is_null(vma));
3449         trace_xe_vma_usm_invalidate(vma);
3450
3451         /* Check that we don't race with page-table updates */
3452         if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
3453                 if (xe_vma_is_userptr(vma)) {
3454                         WARN_ON_ONCE(!mmu_interval_check_retry
3455                                      (&vma->userptr.notifier,
3456                                       vma->userptr.notifier_seq));
3457                         WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(xe_vma_vm(vma)),
3458                                                              DMA_RESV_USAGE_BOOKKEEP));
3459
3460                 } else {
3461                         xe_bo_assert_held(xe_vma_bo(vma));
3462                 }
3463         }
3464
3465         for_each_tile(tile, xe, id) {
3466                 if (xe_pt_zap_ptes(tile, vma)) {
3467                         tile_needs_invalidate |= BIT(id);
3468                         xe_device_wmb(xe);
3469                         /*
3470                          * FIXME: We potentially need to invalidate multiple
3471                          * GTs within the tile
3472                          */
3473                         seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma);
3474                         if (seqno[id] < 0)
3475                                 return seqno[id];
3476                 }
3477         }
3478
3479         for_each_tile(tile, xe, id) {
3480                 if (tile_needs_invalidate & BIT(id)) {
3481                         ret = xe_gt_tlb_invalidation_wait(tile->primary_gt, seqno[id]);
3482                         if (ret < 0)
3483                                 return ret;
3484                 }
3485         }
3486
3487         vma->usm.tile_invalidated = vma->tile_mask;
3488
3489         return 0;
3490 }
3491
3492 int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id)
3493 {
3494         struct drm_gpuva *gpuva;
3495         bool is_vram;
3496         uint64_t addr;
3497
3498         if (!down_read_trylock(&vm->lock)) {
3499                 drm_printf(p, " Failed to acquire VM lock to dump capture");
3500                 return 0;
3501         }
3502         if (vm->pt_root[gt_id]) {
3503                 addr = xe_bo_addr(vm->pt_root[gt_id]->bo, 0, XE_PAGE_SIZE,
3504                                   &is_vram);
3505                 drm_printf(p, " VM root: A:0x%llx %s\n", addr, is_vram ? "VRAM" : "SYS");
3506         }
3507
3508         drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
3509                 struct xe_vma *vma = gpuva_to_vma(gpuva);
3510                 bool is_userptr = xe_vma_is_userptr(vma);
3511                 bool is_null = xe_vma_is_null(vma);
3512
3513                 if (is_null) {
3514                         addr = 0;
3515                 } else if (is_userptr) {
3516                         struct xe_res_cursor cur;
3517
3518                         if (vma->userptr.sg) {
3519                                 xe_res_first_sg(vma->userptr.sg, 0, XE_PAGE_SIZE,
3520                                                 &cur);
3521                                 addr = xe_res_dma(&cur);
3522                         } else {
3523                                 addr = 0;
3524                         }
3525                 } else {
3526                         addr = __xe_bo_addr(xe_vma_bo(vma), 0, XE_PAGE_SIZE, &is_vram);
3527                 }
3528                 drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n",
3529                            xe_vma_start(vma), xe_vma_end(vma) - 1,
3530                            xe_vma_size(vma),
3531                            addr, is_null ? "NULL" : is_userptr ? "USR" :
3532                            is_vram ? "VRAM" : "SYS");
3533         }
3534         up_read(&vm->lock);
3535
3536         return 0;
3537 }