drivers/gpu/drm/xe/tests/xe_migrate.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020-2022 Intel Corporation
   4  */
   5
   6 #include <kunit/test.h>
   7
   8 #include "tests/xe_migrate_test.h"
   9
  10 #include "xe_pci.h"
  11
  12 static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
  13                                 const char *str, struct kunit *test)
  14 {
  15         long ret;
  16
  17         if (IS_ERR(fence)) {
  18                 KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
  19                            PTR_ERR(fence));
  20                 return true;
  21         }
  22         if (!fence)
  23                 return true;
  24
  25         ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
  26         if (ret <= 0) {
  27                 KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
  28                 return true;
  29         }
  30
  31         return false;
  32 }
  33
  34 static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe,
  35                           struct xe_bb *bb, u32 second_idx, const char *str,
  36                           struct kunit *test)
  37 {
  38         struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb,
  39                                                               m->batch_base_ofs,
  40                                                               second_idx);
  41         struct dma_fence *fence;
  42
  43         if (IS_ERR(job)) {
  44                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
  45                            PTR_ERR(job));
  46                 return PTR_ERR(job);
  47         }
  48
  49         xe_sched_job_arm(job);
  50         fence = dma_fence_get(&job->drm.s_fence->finished);
  51         xe_sched_job_push(job);
  52
  53         if (sanity_fence_failed(xe, fence, str, test))
  54                 return -ETIMEDOUT;
  55
  56         dma_fence_put(fence);
  57         kunit_info(test, "%s: Job completed\n", str);
  58         return 0;
  59 }
  60
  61 static void
  62 sanity_populate_cb(struct xe_migrate_pt_update *pt_update,
  63                    struct xe_gt *gt, struct iosys_map *map, void *dst,
  64                    u32 qword_ofs, u32 num_qwords,
  65                    const struct xe_vm_pgtable_update *update)
  66 {
  67         struct migrate_test_params *p =
  68                 to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));
  69         int i;
  70         u64 *ptr = dst;
  71         u64 value;
  72
  73         for (i = 0; i < num_qwords; i++) {
  74                 value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL;
  75                 if (map)
  76                         xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) *
  77                                   sizeof(u64), u64, value);
  78                 else
  79                         ptr[i] = value;
  80         }
  81
  82         kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU");
  83         if (p->force_gpu && map)
  84                 KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n");
  85 }
  86
  87 static const struct xe_migrate_pt_update_ops sanity_ops = {
  88         .populate = sanity_populate_cb,
  89 };
  90
  91 #define check(_retval, _expected, str, _test)                           \
  92         do { if ((_retval) != (_expected)) {                            \
  93                         KUNIT_FAIL(_test, "Sanity check failed: " str   \
  94                                    " expected %llx, got %llx\n",        \
  95                                    (u64)(_expected), (u64)(_retval));   \
  96                 } } while (0)
  97
  98 static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
  99                       struct kunit *test)
 100 {
 101         struct xe_device *xe = gt_to_xe(m->gt);
 102         u64 retval, expected = 0xc0c0c0c0c0c0c0c0ULL;
 103         bool big = bo->size >= SZ_2M;
 104         struct dma_fence *fence;
 105         const char *str = big ? "Copying big bo" : "Copying small bo";
 106         int err;
 107
 108         struct xe_bo *sysmem = xe_bo_create_locked(xe, m->gt, NULL,
 109                                                    bo->size,
 110                                                    ttm_bo_type_kernel,
 111                                                    XE_BO_CREATE_SYSTEM_BIT);
 112         if (IS_ERR(sysmem)) {
 113                 KUNIT_FAIL(test, "Failed to allocate sysmem bo for %s: %li\n",
 114                            str, PTR_ERR(sysmem));
 115                 return;
 116         }
 117
 118         err = xe_bo_validate(sysmem, NULL, false);
 119         if (err) {
 120                 KUNIT_FAIL(test, "Failed to validate system bo for %s: %li\n",
 121                            str, err);
 122                 goto out_unlock;
 123         }
 124
 125         err = xe_bo_vmap(sysmem);
 126         if (err) {
 127                 KUNIT_FAIL(test, "Failed to vmap system bo for %s: %li\n",
 128                            str, err);
 129                 goto out_unlock;
 130         }
 131
 132         xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
 133         fence = xe_migrate_clear(m, sysmem, sysmem->ttm.resource, 0xc0c0c0c0);
 134         if (!sanity_fence_failed(xe, fence, big ? "Clearing sysmem big bo" :
 135                                  "Clearing sysmem small bo", test)) {
 136                 retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
 137                 check(retval, expected, "sysmem first offset should be cleared",
 138                       test);
 139                 retval = xe_map_rd(xe, &sysmem->vmap, sysmem->size - 8, u64);
 140                 check(retval, expected, "sysmem last offset should be cleared",
 141                       test);
 142         }
 143         dma_fence_put(fence);
 144
 145         /* Try to copy 0xc0 from sysmem to vram with 2MB or 64KiB/4KiB pages */
 146         xe_map_memset(xe, &sysmem->vmap, 0, 0xc0, sysmem->size);
 147         xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size);
 148
 149         fence = xe_migrate_copy(m, sysmem, sysmem->ttm.resource,
 150                                 bo->ttm.resource);
 151         if (!sanity_fence_failed(xe, fence, big ? "Copying big bo sysmem -> vram" :
 152                                  "Copying small bo sysmem -> vram", test)) {
 153                 retval = xe_map_rd(xe, &bo->vmap, 0, u64);
 154                 check(retval, expected,
 155                       "sysmem -> vram bo first offset should be copied", test);
 156                 retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64);
 157                 check(retval, expected,
 158                       "sysmem -> vram bo offset should be copied", test);
 159         }
 160         dma_fence_put(fence);
 161
 162         /* And other way around.. slightly hacky.. */
 163         xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
 164         xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size);
 165
 166         fence = xe_migrate_copy(m, sysmem, bo->ttm.resource,
 167                                 sysmem->ttm.resource);
 168         if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> sysmem" :
 169                                  "Copying small bo vram -> sysmem", test)) {
 170                 retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
 171                 check(retval, expected,
 172                       "vram -> sysmem bo first offset should be copied", test);
 173                 retval = xe_map_rd(xe, &sysmem->vmap, bo->size - 8, u64);
 174                 check(retval, expected,
 175                       "vram -> sysmem bo last offset should be copied", test);
 176         }
 177         dma_fence_put(fence);
 178
 179         xe_bo_vunmap(sysmem);
 180 out_unlock:
 181         xe_bo_unlock_no_vm(sysmem);
 182         xe_bo_put(sysmem);
 183 }
 184
 185 static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt,
 186                            struct kunit *test, bool force_gpu)
 187 {
 188         struct xe_device *xe = gt_to_xe(m->gt);
 189         struct dma_fence *fence;
 190         u64 retval, expected;
 191         ktime_t then, now;
 192         int i;
 193
 194         struct xe_vm_pgtable_update update = {
 195                 .ofs = 1,
 196                 .qwords = 0x10,
 197                 .pt_bo = pt,
 198         };
 199         struct xe_migrate_pt_update pt_update = {
 200                 .ops = &sanity_ops,
 201         };
 202         struct migrate_test_params p = {
 203                 .base.id = XE_TEST_LIVE_MIGRATE,
 204                 .force_gpu = force_gpu,
 205         };
 206
 207         test->priv = &p;
 208         /* Test xe_migrate_update_pgtables() updates the pagetable as expected */
 209         expected = 0xf0f0f0f0f0f0f0f0ULL;
 210         xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size);
 211
 212         then = ktime_get();
 213         fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1,
 214                                            NULL, 0, &pt_update);
 215         now = ktime_get();
 216         if (sanity_fence_failed(xe, fence, "Migration pagetable update", test))
 217                 return;
 218
 219         kunit_info(test, "Updating without syncing took %llu us,\n",
 220                    (unsigned long long)ktime_to_us(ktime_sub(now, then)));
 221
 222         dma_fence_put(fence);
 223         retval = xe_map_rd(xe, &pt->vmap, 0, u64);
 224         check(retval, expected, "PTE[0] must stay untouched", test);
 225
 226         for (i = 0; i < update.qwords; i++) {
 227                 retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64);
 228                 check(retval, i * 0x1111111111111111ULL, "PTE update", test);
 229         }
 230
 231         retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords),
 232                            u64);
 233         check(retval, expected, "PTE[0x11] must stay untouched", test);
 234 }
 235
 236 static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
 237 {
 238         struct xe_gt *gt = m->gt;
 239         struct xe_device *xe = gt_to_xe(gt);
 240         struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny;
 241         struct xe_res_cursor src_it;
 242         struct dma_fence *fence;
 243         u64 retval, expected;
 244         struct xe_bb *bb;
 245         int err;
 246         u8 id = gt->info.id;
 247
 248         err = xe_bo_vmap(bo);
 249         if (err) {
 250                 KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
 251                            PTR_ERR(bo));
 252                 return;
 253         }
 254
 255         big = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, SZ_4M,
 256                                    ttm_bo_type_kernel,
 257                                    XE_BO_CREATE_VRAM_IF_DGFX(m->gt) |
 258                                    XE_BO_CREATE_PINNED_BIT);
 259         if (IS_ERR(big)) {
 260                 KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
 261                 goto vunmap;
 262         }
 263
 264         pt = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, GEN8_PAGE_SIZE,
 265                                   ttm_bo_type_kernel,
 266                                   XE_BO_CREATE_VRAM_IF_DGFX(m->gt) |
 267                                   XE_BO_CREATE_PINNED_BIT);
 268         if (IS_ERR(pt)) {
 269                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 270                            PTR_ERR(pt));
 271                 goto free_big;
 272         }
 273
 274         tiny = xe_bo_create_pin_map(xe, m->gt, m->eng->vm,
 275                                     2 * SZ_4K,
 276                                     ttm_bo_type_kernel,
 277                                     XE_BO_CREATE_VRAM_IF_DGFX(m->gt) |
 278                                     XE_BO_CREATE_PINNED_BIT);
 279         if (IS_ERR(tiny)) {
 280                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 281                            PTR_ERR(pt));
 282                 goto free_pt;
 283         }
 284
 285         bb = xe_bb_new(m->gt, 32, xe->info.supports_usm);
 286         if (IS_ERR(bb)) {
 287                 KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
 288                            PTR_ERR(bb));
 289                 goto free_tiny;
 290         }
 291
 292         kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
 293                    (unsigned long)xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, GEN8_PAGE_SIZE),
 294                    (unsigned long)xe_bo_main_addr(m->pt_bo, GEN8_PAGE_SIZE));
 295
 296         /* First part of the test, are we updating our pagetable bo with a new entry? */
 297         xe_map_wr(xe, &bo->vmap, GEN8_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 0xdeaddeadbeefbeef);
 298         expected = gen8_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0, 0);
 299         if (m->eng->vm->flags & XE_VM_FLAGS_64K)
 300                 expected |= GEN12_PTE_PS64;
 301         xe_res_first(pt->ttm.resource, 0, pt->size, &src_it);
 302         emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt),
 303                  &src_it, GEN8_PAGE_SIZE, pt);
 304         run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test);
 305
 306         retval = xe_map_rd(xe, &bo->vmap, GEN8_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
 307                            u64);
 308         check(retval, expected, "PTE entry write", test);
 309
 310         /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
 311         bb->len = 0;
 312         bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
 313         xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
 314         expected = 0x12345678U;
 315
 316         emit_clear(m->gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
 317                    expected, IS_DGFX(xe));
 318         run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable",
 319                        test);
 320
 321         retval = xe_map_rd(xe, &pt->vmap, 0, u32);
 322         check(retval, expected, "Write to PT after adding PTE", test);
 323
 324         /* Sanity checks passed, try the full ones! */
 325
 326         /* Clear a small bo */
 327         kunit_info(test, "Clearing small buffer object\n");
 328         xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size);
 329         expected = 0x224488ff;
 330         fence = xe_migrate_clear(m, tiny, tiny->ttm.resource, expected);
 331         if (sanity_fence_failed(xe, fence, "Clearing small bo", test))
 332                 goto out;
 333
 334         dma_fence_put(fence);
 335         retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
 336         check(retval, expected, "Command clear small first value", test);
 337         retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32);
 338         check(retval, expected, "Command clear small last value", test);
 339
 340         if (IS_DGFX(xe)) {
 341                 kunit_info(test, "Copying small buffer object to system\n");
 342                 test_copy(m, tiny, test);
 343         }
 344
 345         /* Clear a big bo with a fixed value */
 346         kunit_info(test, "Clearing big buffer object\n");
 347         xe_map_memset(xe, &big->vmap, 0, 0x11, big->size);
 348         expected = 0x11223344U;
 349         fence = xe_migrate_clear(m, big, big->ttm.resource, expected);
 350         if (sanity_fence_failed(xe, fence, "Clearing big bo", test))
 351                 goto out;
 352
 353         dma_fence_put(fence);
 354         retval = xe_map_rd(xe, &big->vmap, 0, u32);
 355         check(retval, expected, "Command clear big first value", test);
 356         retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32);
 357         check(retval, expected, "Command clear big last value", test);
 358
 359         if (IS_DGFX(xe)) {
 360                 kunit_info(test, "Copying big buffer object to system\n");
 361                 test_copy(m, big, test);
 362         }
 363
 364         kunit_info(test, "Testing page table update using CPU if GPU idle.\n");
 365         test_pt_update(m, pt, test, false);
 366         kunit_info(test, "Testing page table update using GPU\n");
 367         test_pt_update(m, pt, test, true);
 368
 369 out:
 370         xe_bb_free(bb, NULL);
 371 free_tiny:
 372         xe_bo_unpin(tiny);
 373         xe_bo_put(tiny);
 374 free_pt:
 375         xe_bo_unpin(pt);
 376         xe_bo_put(pt);
 377 free_big:
 378         xe_bo_unpin(big);
 379         xe_bo_put(big);
 380 vunmap:
 381         xe_bo_vunmap(m->pt_bo);
 382 }
 383
 384 static int migrate_test_run_device(struct xe_device *xe)
 385 {
 386         struct kunit *test = xe_cur_kunit();
 387         struct xe_gt *gt;
 388         int id;
 389
 390         for_each_gt(gt, xe, id) {
 391                 struct xe_migrate *m = gt->migrate;
 392                 struct ww_acquire_ctx ww;
 393
 394                 kunit_info(test, "Testing gt id %d.\n", id);
 395                 xe_vm_lock(m->eng->vm, &ww, 0, true);
 396                 xe_device_mem_access_get(xe);
 397                 xe_migrate_sanity_test(m, test);
 398                 xe_device_mem_access_put(xe);
 399                 xe_vm_unlock(m->eng->vm, &ww);
 400         }
 401
 402         return 0;
 403 }
 404
 405 void xe_migrate_sanity_kunit(struct kunit *test)
 406 {
 407         xe_call_for_each_device(migrate_test_run_device);
 408 }
 409 EXPORT_SYMBOL(xe_migrate_sanity_kunit);