drivers/gpu/drm/xe/tests/xe_migrate.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020-2022 Intel Corporation
   4  */
   5
   6 #include <kunit/test.h>
   7 #include <kunit/visibility.h>
   8
   9 #include "tests/xe_migrate_test.h"
  10 #include "tests/xe_pci_test.h"
  11
  12 #include "xe_pci.h"
  13
  14 static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
  15                                 const char *str, struct kunit *test)
  16 {
  17         long ret;
  18
  19         if (IS_ERR(fence)) {
  20                 KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
  21                            PTR_ERR(fence));
  22                 return true;
  23         }
  24         if (!fence)
  25                 return true;
  26
  27         ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
  28         if (ret <= 0) {
  29                 KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
  30                 return true;
  31         }
  32
  33         return false;
  34 }
  35
  36 static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe,
  37                           struct xe_bb *bb, u32 second_idx, const char *str,
  38                           struct kunit *test)
  39 {
  40         u64 batch_base = xe_migrate_batch_base(m, xe->info.supports_usm);
  41         struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb,
  42                                                               batch_base,
  43                                                               second_idx);
  44         struct dma_fence *fence;
  45
  46         if (IS_ERR(job)) {
  47                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
  48                            PTR_ERR(job));
  49                 return PTR_ERR(job);
  50         }
  51
  52         xe_sched_job_arm(job);
  53         fence = dma_fence_get(&job->drm.s_fence->finished);
  54         xe_sched_job_push(job);
  55
  56         if (sanity_fence_failed(xe, fence, str, test))
  57                 return -ETIMEDOUT;
  58
  59         dma_fence_put(fence);
  60         kunit_info(test, "%s: Job completed\n", str);
  61         return 0;
  62 }
  63
  64 static void
  65 sanity_populate_cb(struct xe_migrate_pt_update *pt_update,
  66                    struct xe_gt *gt, struct iosys_map *map, void *dst,
  67                    u32 qword_ofs, u32 num_qwords,
  68                    const struct xe_vm_pgtable_update *update)
  69 {
  70         struct migrate_test_params *p =
  71                 to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));
  72         int i;
  73         u64 *ptr = dst;
  74         u64 value;
  75
  76         for (i = 0; i < num_qwords; i++) {
  77                 value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL;
  78                 if (map)
  79                         xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) *
  80                                   sizeof(u64), u64, value);
  81                 else
  82                         ptr[i] = value;
  83         }
  84
  85         kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU");
  86         if (p->force_gpu && map)
  87                 KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n");
  88 }
  89
  90 static const struct xe_migrate_pt_update_ops sanity_ops = {
  91         .populate = sanity_populate_cb,
  92 };
  93
  94 #define check(_retval, _expected, str, _test)                           \
  95         do { if ((_retval) != (_expected)) {                            \
  96                         KUNIT_FAIL(_test, "Sanity check failed: " str   \
  97                                    " expected %llx, got %llx\n",        \
  98                                    (u64)(_expected), (u64)(_retval));   \
  99                 } } while (0)
 100
 101 static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
 102                       struct kunit *test)
 103 {
 104         struct xe_device *xe = gt_to_xe(m->gt);
 105         u64 retval, expected = 0;
 106         bool big = bo->size >= SZ_2M;
 107         struct dma_fence *fence;
 108         const char *str = big ? "Copying big bo" : "Copying small bo";
 109         int err;
 110
 111         struct xe_bo *sysmem = xe_bo_create_locked(xe, m->gt, NULL,
 112                                                    bo->size,
 113                                                    ttm_bo_type_kernel,
 114                                                    XE_BO_CREATE_SYSTEM_BIT);
 115         if (IS_ERR(sysmem)) {
 116                 KUNIT_FAIL(test, "Failed to allocate sysmem bo for %s: %li\n",
 117                            str, PTR_ERR(sysmem));
 118                 return;
 119         }
 120
 121         err = xe_bo_validate(sysmem, NULL, false);
 122         if (err) {
 123                 KUNIT_FAIL(test, "Failed to validate system bo for %s: %li\n",
 124                            str, err);
 125                 goto out_unlock;
 126         }
 127
 128         err = xe_bo_vmap(sysmem);
 129         if (err) {
 130                 KUNIT_FAIL(test, "Failed to vmap system bo for %s: %li\n",
 131                            str, err);
 132                 goto out_unlock;
 133         }
 134
 135         xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
 136         fence = xe_migrate_clear(m, sysmem, sysmem->ttm.resource);
 137         if (!sanity_fence_failed(xe, fence, big ? "Clearing sysmem big bo" :
 138                                  "Clearing sysmem small bo", test)) {
 139                 retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
 140                 check(retval, expected, "sysmem first offset should be cleared",
 141                       test);
 142                 retval = xe_map_rd(xe, &sysmem->vmap, sysmem->size - 8, u64);
 143                 check(retval, expected, "sysmem last offset should be cleared",
 144                       test);
 145         }
 146         dma_fence_put(fence);
 147
 148         /* Try to copy 0xc0 from sysmem to vram with 2MB or 64KiB/4KiB pages */
 149         xe_map_memset(xe, &sysmem->vmap, 0, 0xc0, sysmem->size);
 150         xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size);
 151
 152         expected = 0xc0c0c0c0c0c0c0c0;
 153         fence = xe_migrate_copy(m, sysmem, sysmem->ttm.resource,
 154                                 bo->ttm.resource);
 155         if (!sanity_fence_failed(xe, fence, big ? "Copying big bo sysmem -> vram" :
 156                                  "Copying small bo sysmem -> vram", test)) {
 157                 retval = xe_map_rd(xe, &bo->vmap, 0, u64);
 158                 check(retval, expected,
 159                       "sysmem -> vram bo first offset should be copied", test);
 160                 retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64);
 161                 check(retval, expected,
 162                       "sysmem -> vram bo offset should be copied", test);
 163         }
 164         dma_fence_put(fence);
 165
 166         /* And other way around.. slightly hacky.. */
 167         xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
 168         xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size);
 169
 170         fence = xe_migrate_copy(m, sysmem, bo->ttm.resource,
 171                                 sysmem->ttm.resource);
 172         if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> sysmem" :
 173                                  "Copying small bo vram -> sysmem", test)) {
 174                 retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
 175                 check(retval, expected,
 176                       "vram -> sysmem bo first offset should be copied", test);
 177                 retval = xe_map_rd(xe, &sysmem->vmap, bo->size - 8, u64);
 178                 check(retval, expected,
 179                       "vram -> sysmem bo last offset should be copied", test);
 180         }
 181         dma_fence_put(fence);
 182
 183         xe_bo_vunmap(sysmem);
 184 out_unlock:
 185         xe_bo_unlock_no_vm(sysmem);
 186         xe_bo_put(sysmem);
 187 }
 188
 189 static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt,
 190                            struct kunit *test, bool force_gpu)
 191 {
 192         struct xe_device *xe = gt_to_xe(m->gt);
 193         struct dma_fence *fence;
 194         u64 retval, expected;
 195         ktime_t then, now;
 196         int i;
 197
 198         struct xe_vm_pgtable_update update = {
 199                 .ofs = 1,
 200                 .qwords = 0x10,
 201                 .pt_bo = pt,
 202         };
 203         struct xe_migrate_pt_update pt_update = {
 204                 .ops = &sanity_ops,
 205         };
 206         struct migrate_test_params p = {
 207                 .base.id = XE_TEST_LIVE_MIGRATE,
 208                 .force_gpu = force_gpu,
 209         };
 210
 211         test->priv = &p;
 212         /* Test xe_migrate_update_pgtables() updates the pagetable as expected */
 213         expected = 0xf0f0f0f0f0f0f0f0ULL;
 214         xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size);
 215
 216         then = ktime_get();
 217         fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1,
 218                                            NULL, 0, &pt_update);
 219         now = ktime_get();
 220         if (sanity_fence_failed(xe, fence, "Migration pagetable update", test))
 221                 return;
 222
 223         kunit_info(test, "Updating without syncing took %llu us,\n",
 224                    (unsigned long long)ktime_to_us(ktime_sub(now, then)));
 225
 226         dma_fence_put(fence);
 227         retval = xe_map_rd(xe, &pt->vmap, 0, u64);
 228         check(retval, expected, "PTE[0] must stay untouched", test);
 229
 230         for (i = 0; i < update.qwords; i++) {
 231                 retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64);
 232                 check(retval, i * 0x1111111111111111ULL, "PTE update", test);
 233         }
 234
 235         retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords),
 236                            u64);
 237         check(retval, expected, "PTE[0x11] must stay untouched", test);
 238 }
 239
 240 static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
 241 {
 242         struct xe_gt *gt = m->gt;
 243         struct xe_device *xe = gt_to_xe(gt);
 244         struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny;
 245         struct xe_res_cursor src_it;
 246         struct dma_fence *fence;
 247         u64 retval, expected;
 248         struct xe_bb *bb;
 249         int err;
 250         u8 id = gt->info.id;
 251
 252         err = xe_bo_vmap(bo);
 253         if (err) {
 254                 KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
 255                            PTR_ERR(bo));
 256                 return;
 257         }
 258
 259         big = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, SZ_4M,
 260                                    ttm_bo_type_kernel,
 261                                    XE_BO_CREATE_VRAM_IF_DGFX(m->gt) |
 262                                    XE_BO_CREATE_PINNED_BIT);
 263         if (IS_ERR(big)) {
 264                 KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
 265                 goto vunmap;
 266         }
 267
 268         pt = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, XE_PAGE_SIZE,
 269                                   ttm_bo_type_kernel,
 270                                   XE_BO_CREATE_VRAM_IF_DGFX(m->gt) |
 271                                   XE_BO_CREATE_PINNED_BIT);
 272         if (IS_ERR(pt)) {
 273                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 274                            PTR_ERR(pt));
 275                 goto free_big;
 276         }
 277
 278         tiny = xe_bo_create_pin_map(xe, m->gt, m->eng->vm,
 279                                     2 * SZ_4K,
 280                                     ttm_bo_type_kernel,
 281                                     XE_BO_CREATE_VRAM_IF_DGFX(m->gt) |
 282                                     XE_BO_CREATE_PINNED_BIT);
 283         if (IS_ERR(tiny)) {
 284                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 285                            PTR_ERR(pt));
 286                 goto free_pt;
 287         }
 288
 289         bb = xe_bb_new(m->gt, 32, xe->info.supports_usm);
 290         if (IS_ERR(bb)) {
 291                 KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
 292                            PTR_ERR(bb));
 293                 goto free_tiny;
 294         }
 295
 296         kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
 297                    (unsigned long)xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, XE_PAGE_SIZE),
 298                    (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE));
 299
 300         /* First part of the test, are we updating our pagetable bo with a new entry? */
 301         xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
 302                   0xdeaddeadbeefbeef);
 303         expected = gen8_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0, 0);
 304         if (m->eng->vm->flags & XE_VM_FLAGS_64K)
 305                 expected |= XE_PTE_PS64;
 306         if (xe_bo_is_vram(pt))
 307                 xe_res_first(pt->ttm.resource, 0, pt->size, &src_it);
 308         else
 309                 xe_res_first_sg(xe_bo_get_sg(pt), 0, pt->size, &src_it);
 310
 311         emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt),
 312                  &src_it, XE_PAGE_SIZE, pt);
 313
 314         run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test);
 315
 316         retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
 317                            u64);
 318         check(retval, expected, "PTE entry write", test);
 319
 320         /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
 321         bb->len = 0;
 322         bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
 323         xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
 324         expected = 0;
 325
 326         emit_clear(m->gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
 327                    IS_DGFX(xe));
 328         run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable",
 329                        test);
 330
 331         retval = xe_map_rd(xe, &pt->vmap, 0, u32);
 332         check(retval, expected, "Write to PT after adding PTE", test);
 333
 334         /* Sanity checks passed, try the full ones! */
 335
 336         /* Clear a small bo */
 337         kunit_info(test, "Clearing small buffer object\n");
 338         xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size);
 339         expected = 0;
 340         fence = xe_migrate_clear(m, tiny, tiny->ttm.resource);
 341         if (sanity_fence_failed(xe, fence, "Clearing small bo", test))
 342                 goto out;
 343
 344         dma_fence_put(fence);
 345         retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
 346         check(retval, expected, "Command clear small first value", test);
 347         retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32);
 348         check(retval, expected, "Command clear small last value", test);
 349
 350         if (IS_DGFX(xe)) {
 351                 kunit_info(test, "Copying small buffer object to system\n");
 352                 test_copy(m, tiny, test);
 353         }
 354
 355         /* Clear a big bo */
 356         kunit_info(test, "Clearing big buffer object\n");
 357         xe_map_memset(xe, &big->vmap, 0, 0x11, big->size);
 358         expected = 0;
 359         fence = xe_migrate_clear(m, big, big->ttm.resource);
 360         if (sanity_fence_failed(xe, fence, "Clearing big bo", test))
 361                 goto out;
 362
 363         dma_fence_put(fence);
 364         retval = xe_map_rd(xe, &big->vmap, 0, u32);
 365         check(retval, expected, "Command clear big first value", test);
 366         retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32);
 367         check(retval, expected, "Command clear big last value", test);
 368
 369         if (IS_DGFX(xe)) {
 370                 kunit_info(test, "Copying big buffer object to system\n");
 371                 test_copy(m, big, test);
 372         }
 373
 374         kunit_info(test, "Testing page table update using CPU if GPU idle.\n");
 375         test_pt_update(m, pt, test, false);
 376         kunit_info(test, "Testing page table update using GPU\n");
 377         test_pt_update(m, pt, test, true);
 378
 379 out:
 380         xe_bb_free(bb, NULL);
 381 free_tiny:
 382         xe_bo_unpin(tiny);
 383         xe_bo_put(tiny);
 384 free_pt:
 385         xe_bo_unpin(pt);
 386         xe_bo_put(pt);
 387 free_big:
 388         xe_bo_unpin(big);
 389         xe_bo_put(big);
 390 vunmap:
 391         xe_bo_vunmap(m->pt_bo);
 392 }
 393
 394 static int migrate_test_run_device(struct xe_device *xe)
 395 {
 396         struct kunit *test = xe_cur_kunit();
 397         struct xe_gt *gt;
 398         int id;
 399
 400         for_each_gt(gt, xe, id) {
 401                 struct xe_migrate *m = gt->migrate;
 402                 struct ww_acquire_ctx ww;
 403
 404                 kunit_info(test, "Testing gt id %d.\n", id);
 405                 xe_vm_lock(m->eng->vm, &ww, 0, true);
 406                 xe_device_mem_access_get(xe);
 407                 xe_migrate_sanity_test(m, test);
 408                 xe_device_mem_access_put(xe);
 409                 xe_vm_unlock(m->eng->vm, &ww);
 410         }
 411
 412         return 0;
 413 }
 414
 415 void xe_migrate_sanity_kunit(struct kunit *test)
 416 {
 417         xe_call_for_each_device(migrate_test_run_device);
 418 }
 419 EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit);