drivers/gpu/drm/i915/selftests/i915_gem_context.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/prime_numbers.h>
  26
  27 #include "gt/intel_reset.h"
  28 #include "i915_selftest.h"
  29
  30 #include "i915_random.h"
  31 #include "igt_flush_test.h"
  32 #include "igt_gem_utils.h"
  33 #include "igt_live_test.h"
  34 #include "igt_reset.h"
  35 #include "igt_spinner.h"
  36
  37 #include "mock_drm.h"
  38 #include "mock_gem_device.h"
  39 #include "huge_gem_object.h"
  40
  41 #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
  42
  43 static int live_nop_switch(void *arg)
  44 {
  45         const unsigned int nctx = 1024;
  46         struct drm_i915_private *i915 = arg;
  47         struct intel_engine_cs *engine;
  48         struct i915_gem_context **ctx;
  49         enum intel_engine_id id;
  50         intel_wakeref_t wakeref;
  51         struct igt_live_test t;
  52         struct drm_file *file;
  53         unsigned long n;
  54         int err = -ENODEV;
  55
  56         /*
  57          * Create as many contexts as we can feasibly get away with
  58          * and check we can switch between them rapidly.
  59          *
  60          * Serves as very simple stress test for submission and HW switching
  61          * between contexts.
  62          */
  63
  64         if (!DRIVER_CAPS(i915)->has_logical_contexts)
  65                 return 0;
  66
  67         file = mock_file(i915);
  68         if (IS_ERR(file))
  69                 return PTR_ERR(file);
  70
  71         mutex_lock(&i915->drm.struct_mutex);
  72         wakeref = intel_runtime_pm_get(i915);
  73
  74         ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL);
  75         if (!ctx) {
  76                 err = -ENOMEM;
  77                 goto out_unlock;
  78         }
  79
  80         for (n = 0; n < nctx; n++) {
  81                 ctx[n] = live_context(i915, file);
  82                 if (IS_ERR(ctx[n])) {
  83                         err = PTR_ERR(ctx[n]);
  84                         goto out_unlock;
  85                 }
  86         }
  87
  88         for_each_engine(engine, i915, id) {
  89                 struct i915_request *rq;
  90                 unsigned long end_time, prime;
  91                 ktime_t times[2] = {};
  92
  93                 times[0] = ktime_get_raw();
  94                 for (n = 0; n < nctx; n++) {
  95                         rq = igt_request_alloc(ctx[n], engine);
  96                         if (IS_ERR(rq)) {
  97                                 err = PTR_ERR(rq);
  98                                 goto out_unlock;
  99                         }
 100                         i915_request_add(rq);
 101                 }
 102                 if (i915_request_wait(rq,
 103                                       I915_WAIT_LOCKED,
 104                                       HZ / 5) < 0) {
 105                         pr_err("Failed to populated %d contexts\n", nctx);
 106                         i915_gem_set_wedged(i915);
 107                         err = -EIO;
 108                         goto out_unlock;
 109                 }
 110
 111                 times[1] = ktime_get_raw();
 112
 113                 pr_info("Populated %d contexts on %s in %lluns\n",
 114                         nctx, engine->name, ktime_to_ns(times[1] - times[0]));
 115
 116                 err = igt_live_test_begin(&t, i915, __func__, engine->name);
 117                 if (err)
 118                         goto out_unlock;
 119
 120                 end_time = jiffies + i915_selftest.timeout_jiffies;
 121                 for_each_prime_number_from(prime, 2, 8192) {
 122                         times[1] = ktime_get_raw();
 123
 124                         for (n = 0; n < prime; n++) {
 125                                 rq = igt_request_alloc(ctx[n % nctx], engine);
 126                                 if (IS_ERR(rq)) {
 127                                         err = PTR_ERR(rq);
 128                                         goto out_unlock;
 129                                 }
 130
 131                                 /*
 132                                  * This space is left intentionally blank.
 133                                  *
 134                                  * We do not actually want to perform any
 135                                  * action with this request, we just want
 136                                  * to measure the latency in allocation
 137                                  * and submission of our breadcrumbs -
 138                                  * ensuring that the bare request is sufficient
 139                                  * for the system to work (i.e. proper HEAD
 140                                  * tracking of the rings, interrupt handling,
 141                                  * etc). It also gives us the lowest bounds
 142                                  * for latency.
 143                                  */
 144
 145                                 i915_request_add(rq);
 146                         }
 147                         if (i915_request_wait(rq,
 148                                               I915_WAIT_LOCKED,
 149                                               HZ / 5) < 0) {
 150                                 pr_err("Switching between %ld contexts timed out\n",
 151                                        prime);
 152                                 i915_gem_set_wedged(i915);
 153                                 break;
 154                         }
 155
 156                         times[1] = ktime_sub(ktime_get_raw(), times[1]);
 157                         if (prime == 2)
 158                                 times[0] = times[1];
 159
 160                         if (__igt_timeout(end_time, NULL))
 161                                 break;
 162                 }
 163
 164                 err = igt_live_test_end(&t);
 165                 if (err)
 166                         goto out_unlock;
 167
 168                 pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n",
 169                         engine->name,
 170                         ktime_to_ns(times[0]),
 171                         prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1));
 172         }
 173
 174 out_unlock:
 175         intel_runtime_pm_put(i915, wakeref);
 176         mutex_unlock(&i915->drm.struct_mutex);
 177         mock_file_free(i915, file);
 178         return err;
 179 }
 180
 181 static struct i915_vma *
 182 gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
 183 {
 184         struct drm_i915_gem_object *obj;
 185         const int gen = INTEL_GEN(vma->vm->i915);
 186         unsigned long n, size;
 187         u32 *cmd;
 188         int err;
 189
 190         size = (4 * count + 1) * sizeof(u32);
 191         size = round_up(size, PAGE_SIZE);
 192         obj = i915_gem_object_create_internal(vma->vm->i915, size);
 193         if (IS_ERR(obj))
 194                 return ERR_CAST(obj);
 195
 196         cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
 197         if (IS_ERR(cmd)) {
 198                 err = PTR_ERR(cmd);
 199                 goto err;
 200         }
 201
 202         GEM_BUG_ON(offset + (count - 1) * PAGE_SIZE > vma->node.size);
 203         offset += vma->node.start;
 204
 205         for (n = 0; n < count; n++) {
 206                 if (gen >= 8) {
 207                         *cmd++ = MI_STORE_DWORD_IMM_GEN4;
 208                         *cmd++ = lower_32_bits(offset);
 209                         *cmd++ = upper_32_bits(offset);
 210                         *cmd++ = value;
 211                 } else if (gen >= 4) {
 212                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
 213                                 (gen < 6 ? MI_USE_GGTT : 0);
 214                         *cmd++ = 0;
 215                         *cmd++ = offset;
 216                         *cmd++ = value;
 217                 } else {
 218                         *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 219                         *cmd++ = offset;
 220                         *cmd++ = value;
 221                 }
 222                 offset += PAGE_SIZE;
 223         }
 224         *cmd = MI_BATCH_BUFFER_END;
 225         i915_gem_object_flush_map(obj);
 226         i915_gem_object_unpin_map(obj);
 227
 228         err = i915_gem_object_set_to_gtt_domain(obj, false);
 229         if (err)
 230                 goto err;
 231
 232         vma = i915_vma_instance(obj, vma->vm, NULL);
 233         if (IS_ERR(vma)) {
 234                 err = PTR_ERR(vma);
 235                 goto err;
 236         }
 237
 238         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 239         if (err)
 240                 goto err;
 241
 242         return vma;
 243
 244 err:
 245         i915_gem_object_put(obj);
 246         return ERR_PTR(err);
 247 }
 248
 249 static unsigned long real_page_count(struct drm_i915_gem_object *obj)
 250 {
 251         return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
 252 }
 253
 254 static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
 255 {
 256         return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
 257 }
 258
 259 static int gpu_fill(struct drm_i915_gem_object *obj,
 260                     struct i915_gem_context *ctx,
 261                     struct intel_engine_cs *engine,
 262                     unsigned int dw)
 263 {
 264         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 265         struct i915_address_space *vm =
 266                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
 267         struct i915_request *rq;
 268         struct i915_vma *vma;
 269         struct i915_vma *batch;
 270         unsigned int flags;
 271         int err;
 272
 273         GEM_BUG_ON(obj->base.size > vm->total);
 274         GEM_BUG_ON(!intel_engine_can_store_dword(engine));
 275
 276         vma = i915_vma_instance(obj, vm, NULL);
 277         if (IS_ERR(vma))
 278                 return PTR_ERR(vma);
 279
 280         err = i915_gem_object_set_to_gtt_domain(obj, false);
 281         if (err)
 282                 return err;
 283
 284         err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
 285         if (err)
 286                 return err;
 287
 288         /* Within the GTT the huge objects maps every page onto
 289          * its 1024 real pages (using phys_pfn = dma_pfn % 1024).
 290          * We set the nth dword within the page using the nth
 291          * mapping via the GTT - this should exercise the GTT mapping
 292          * whilst checking that each context provides a unique view
 293          * into the object.
 294          */
 295         batch = gpu_fill_dw(vma,
 296                             (dw * real_page_count(obj)) << PAGE_SHIFT |
 297                             (dw * sizeof(u32)),
 298                             real_page_count(obj),
 299                             dw);
 300         if (IS_ERR(batch)) {
 301                 err = PTR_ERR(batch);
 302                 goto err_vma;
 303         }
 304
 305         rq = igt_request_alloc(ctx, engine);
 306         if (IS_ERR(rq)) {
 307                 err = PTR_ERR(rq);
 308                 goto err_batch;
 309         }
 310
 311         flags = 0;
 312         if (INTEL_GEN(vm->i915) <= 5)
 313                 flags |= I915_DISPATCH_SECURE;
 314
 315         err = engine->emit_bb_start(rq,
 316                                     batch->node.start, batch->node.size,
 317                                     flags);
 318         if (err)
 319                 goto err_request;
 320
 321         err = i915_vma_move_to_active(batch, rq, 0);
 322         if (err)
 323                 goto skip_request;
 324
 325         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 326         if (err)
 327                 goto skip_request;
 328
 329         i915_gem_object_set_active_reference(batch->obj);
 330         i915_vma_unpin(batch);
 331         i915_vma_close(batch);
 332
 333         i915_vma_unpin(vma);
 334
 335         i915_request_add(rq);
 336
 337         return 0;
 338
 339 skip_request:
 340         i915_request_skip(rq, err);
 341 err_request:
 342         i915_request_add(rq);
 343 err_batch:
 344         i915_vma_unpin(batch);
 345         i915_vma_put(batch);
 346 err_vma:
 347         i915_vma_unpin(vma);
 348         return err;
 349 }
 350
 351 static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
 352 {
 353         const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
 354         unsigned int n, m, need_flush;
 355         int err;
 356
 357         err = i915_gem_obj_prepare_shmem_write(obj, &need_flush);
 358         if (err)
 359                 return err;
 360
 361         for (n = 0; n < real_page_count(obj); n++) {
 362                 u32 *map;
 363
 364                 map = kmap_atomic(i915_gem_object_get_page(obj, n));
 365                 for (m = 0; m < DW_PER_PAGE; m++)
 366                         map[m] = value;
 367                 if (!has_llc)
 368                         drm_clflush_virt_range(map, PAGE_SIZE);
 369                 kunmap_atomic(map);
 370         }
 371
 372         i915_gem_obj_finish_shmem_access(obj);
 373         obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
 374         obj->write_domain = 0;
 375         return 0;
 376 }
 377
 378 static noinline int cpu_check(struct drm_i915_gem_object *obj,
 379                               unsigned int idx, unsigned int max)
 380 {
 381         unsigned int n, m, needs_flush;
 382         int err;
 383
 384         err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
 385         if (err)
 386                 return err;
 387
 388         for (n = 0; n < real_page_count(obj); n++) {
 389                 u32 *map;
 390
 391                 map = kmap_atomic(i915_gem_object_get_page(obj, n));
 392                 if (needs_flush & CLFLUSH_BEFORE)
 393                         drm_clflush_virt_range(map, PAGE_SIZE);
 394
 395                 for (m = 0; m < max; m++) {
 396                         if (map[m] != m) {
 397                                 pr_err("%pS: Invalid value at object %d page %d/%ld, offset %d/%d: found %x expected %x\n",
 398                                        __builtin_return_address(0), idx,
 399                                        n, real_page_count(obj), m, max,
 400                                        map[m], m);
 401                                 err = -EINVAL;
 402                                 goto out_unmap;
 403                         }
 404                 }
 405
 406                 for (; m < DW_PER_PAGE; m++) {
 407                         if (map[m] != STACK_MAGIC) {
 408                                 pr_err("%pS: Invalid value at object %d page %d, offset %d: found %x expected %x (uninitialised)\n",
 409                                        __builtin_return_address(0), idx, n, m,
 410                                        map[m], STACK_MAGIC);
 411                                 err = -EINVAL;
 412                                 goto out_unmap;
 413                         }
 414                 }
 415
 416 out_unmap:
 417                 kunmap_atomic(map);
 418                 if (err)
 419                         break;
 420         }
 421
 422         i915_gem_obj_finish_shmem_access(obj);
 423         return err;
 424 }
 425
 426 static int file_add_object(struct drm_file *file,
 427                             struct drm_i915_gem_object *obj)
 428 {
 429         int err;
 430
 431         GEM_BUG_ON(obj->base.handle_count);
 432
 433         /* tie the object to the drm_file for easy reaping */
 434         err = idr_alloc(&file->object_idr, &obj->base, 1, 0, GFP_KERNEL);
 435         if (err < 0)
 436                 return  err;
 437
 438         i915_gem_object_get(obj);
 439         obj->base.handle_count++;
 440         return 0;
 441 }
 442
 443 static struct drm_i915_gem_object *
 444 create_test_object(struct i915_gem_context *ctx,
 445                    struct drm_file *file,
 446                    struct list_head *objects)
 447 {
 448         struct drm_i915_gem_object *obj;
 449         struct i915_address_space *vm =
 450                 ctx->ppgtt ? &ctx->ppgtt->vm : &ctx->i915->ggtt.vm;
 451         u64 size;
 452         int err;
 453
 454         size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
 455         size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
 456
 457         obj = huge_gem_object(ctx->i915, DW_PER_PAGE * PAGE_SIZE, size);
 458         if (IS_ERR(obj))
 459                 return obj;
 460
 461         err = file_add_object(file, obj);
 462         i915_gem_object_put(obj);
 463         if (err)
 464                 return ERR_PTR(err);
 465
 466         err = cpu_fill(obj, STACK_MAGIC);
 467         if (err) {
 468                 pr_err("Failed to fill object with cpu, err=%d\n",
 469                        err);
 470                 return ERR_PTR(err);
 471         }
 472
 473         list_add_tail(&obj->st_link, objects);
 474         return obj;
 475 }
 476
 477 static unsigned long max_dwords(struct drm_i915_gem_object *obj)
 478 {
 479         unsigned long npages = fake_page_count(obj);
 480
 481         GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
 482         return npages / DW_PER_PAGE;
 483 }
 484
 485 static int igt_ctx_exec(void *arg)
 486 {
 487         struct drm_i915_private *i915 = arg;
 488         struct intel_engine_cs *engine;
 489         enum intel_engine_id id;
 490         int err = -ENODEV;
 491
 492         /*
 493          * Create a few different contexts (with different mm) and write
 494          * through each ctx/mm using the GPU making sure those writes end
 495          * up in the expected pages of our obj.
 496          */
 497
 498         if (!DRIVER_CAPS(i915)->has_logical_contexts)
 499                 return 0;
 500
 501         for_each_engine(engine, i915, id) {
 502                 struct drm_i915_gem_object *obj = NULL;
 503                 unsigned long ncontexts, ndwords, dw;
 504                 struct igt_live_test t;
 505                 struct drm_file *file;
 506                 IGT_TIMEOUT(end_time);
 507                 LIST_HEAD(objects);
 508
 509                 if (!intel_engine_can_store_dword(engine))
 510                         continue;
 511
 512                 if (!engine->context_size)
 513                         continue; /* No logical context support in HW */
 514
 515                 file = mock_file(i915);
 516                 if (IS_ERR(file))
 517                         return PTR_ERR(file);
 518
 519                 mutex_lock(&i915->drm.struct_mutex);
 520
 521                 err = igt_live_test_begin(&t, i915, __func__, engine->name);
 522                 if (err)
 523                         goto out_unlock;
 524
 525                 ncontexts = 0;
 526                 ndwords = 0;
 527                 dw = 0;
 528                 while (!time_after(jiffies, end_time)) {
 529                         struct i915_gem_context *ctx;
 530                         intel_wakeref_t wakeref;
 531
 532                         ctx = live_context(i915, file);
 533                         if (IS_ERR(ctx)) {
 534                                 err = PTR_ERR(ctx);
 535                                 goto out_unlock;
 536                         }
 537
 538                         if (!obj) {
 539                                 obj = create_test_object(ctx, file, &objects);
 540                                 if (IS_ERR(obj)) {
 541                                         err = PTR_ERR(obj);
 542                                         goto out_unlock;
 543                                 }
 544                         }
 545
 546                         with_intel_runtime_pm(i915, wakeref)
 547                                 err = gpu_fill(obj, ctx, engine, dw);
 548                         if (err) {
 549                                 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
 550                                        ndwords, dw, max_dwords(obj),
 551                                        engine->name, ctx->hw_id,
 552                                        yesno(!!ctx->ppgtt), err);
 553                                 goto out_unlock;
 554                         }
 555
 556                         if (++dw == max_dwords(obj)) {
 557                                 obj = NULL;
 558                                 dw = 0;
 559                         }
 560
 561                         ndwords++;
 562                         ncontexts++;
 563                 }
 564
 565                 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
 566                         ncontexts, engine->name, ndwords);
 567
 568                 ncontexts = dw = 0;
 569                 list_for_each_entry(obj, &objects, st_link) {
 570                         unsigned int rem =
 571                                 min_t(unsigned int, ndwords - dw, max_dwords(obj));
 572
 573                         err = cpu_check(obj, ncontexts++, rem);
 574                         if (err)
 575                                 break;
 576
 577                         dw += rem;
 578                 }
 579
 580 out_unlock:
 581                 if (igt_live_test_end(&t))
 582                         err = -EIO;
 583                 mutex_unlock(&i915->drm.struct_mutex);
 584
 585                 mock_file_free(i915, file);
 586                 if (err)
 587                         return err;
 588         }
 589
 590         return 0;
 591 }
 592
 593 static int igt_shared_ctx_exec(void *arg)
 594 {
 595         struct drm_i915_private *i915 = arg;
 596         struct i915_gem_context *parent;
 597         struct intel_engine_cs *engine;
 598         enum intel_engine_id id;
 599         struct igt_live_test t;
 600         struct drm_file *file;
 601         int err = 0;
 602
 603         /*
 604          * Create a few different contexts with the same mm and write
 605          * through each ctx using the GPU making sure those writes end
 606          * up in the expected pages of our obj.
 607          */
 608         if (!DRIVER_CAPS(i915)->has_logical_contexts)
 609                 return 0;
 610
 611         file = mock_file(i915);
 612         if (IS_ERR(file))
 613                 return PTR_ERR(file);
 614
 615         mutex_lock(&i915->drm.struct_mutex);
 616
 617         parent = live_context(i915, file);
 618         if (IS_ERR(parent)) {
 619                 err = PTR_ERR(parent);
 620                 goto out_unlock;
 621         }
 622
 623         if (!parent->ppgtt) { /* not full-ppgtt; nothing to share */
 624                 err = 0;
 625                 goto out_unlock;
 626         }
 627
 628         err = igt_live_test_begin(&t, i915, __func__, "");
 629         if (err)
 630                 goto out_unlock;
 631
 632         for_each_engine(engine, i915, id) {
 633                 unsigned long ncontexts, ndwords, dw;
 634                 struct drm_i915_gem_object *obj = NULL;
 635                 IGT_TIMEOUT(end_time);
 636                 LIST_HEAD(objects);
 637
 638                 if (!intel_engine_can_store_dword(engine))
 639                         continue;
 640
 641                 dw = 0;
 642                 ndwords = 0;
 643                 ncontexts = 0;
 644                 while (!time_after(jiffies, end_time)) {
 645                         struct i915_gem_context *ctx;
 646                         intel_wakeref_t wakeref;
 647
 648                         ctx = kernel_context(i915);
 649                         if (IS_ERR(ctx)) {
 650                                 err = PTR_ERR(ctx);
 651                                 goto out_test;
 652                         }
 653
 654                         __assign_ppgtt(ctx, parent->ppgtt);
 655
 656                         if (!obj) {
 657                                 obj = create_test_object(parent, file, &objects);
 658                                 if (IS_ERR(obj)) {
 659                                         err = PTR_ERR(obj);
 660                                         kernel_context_close(ctx);
 661                                         goto out_test;
 662                                 }
 663                         }
 664
 665                         err = 0;
 666                         with_intel_runtime_pm(i915, wakeref)
 667                                 err = gpu_fill(obj, ctx, engine, dw);
 668                         if (err) {
 669                                 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
 670                                        ndwords, dw, max_dwords(obj),
 671                                        engine->name, ctx->hw_id,
 672                                        yesno(!!ctx->ppgtt), err);
 673                                 kernel_context_close(ctx);
 674                                 goto out_test;
 675                         }
 676
 677                         if (++dw == max_dwords(obj)) {
 678                                 obj = NULL;
 679                                 dw = 0;
 680                         }
 681
 682                         ndwords++;
 683                         ncontexts++;
 684
 685                         kernel_context_close(ctx);
 686                 }
 687                 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
 688                         ncontexts, engine->name, ndwords);
 689
 690                 ncontexts = dw = 0;
 691                 list_for_each_entry(obj, &objects, st_link) {
 692                         unsigned int rem =
 693                                 min_t(unsigned int, ndwords - dw, max_dwords(obj));
 694
 695                         err = cpu_check(obj, ncontexts++, rem);
 696                         if (err)
 697                                 goto out_test;
 698
 699                         dw += rem;
 700                 }
 701         }
 702 out_test:
 703         if (igt_live_test_end(&t))
 704                 err = -EIO;
 705 out_unlock:
 706         mutex_unlock(&i915->drm.struct_mutex);
 707
 708         mock_file_free(i915, file);
 709         return err;
 710 }
 711
 712 static struct i915_vma *rpcs_query_batch(struct i915_vma *vma)
 713 {
 714         struct drm_i915_gem_object *obj;
 715         u32 *cmd;
 716         int err;
 717
 718         if (INTEL_GEN(vma->vm->i915) < 8)
 719                 return ERR_PTR(-EINVAL);
 720
 721         obj = i915_gem_object_create_internal(vma->vm->i915, PAGE_SIZE);
 722         if (IS_ERR(obj))
 723                 return ERR_CAST(obj);
 724
 725         cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
 726         if (IS_ERR(cmd)) {
 727                 err = PTR_ERR(cmd);
 728                 goto err;
 729         }
 730
 731         *cmd++ = MI_STORE_REGISTER_MEM_GEN8;
 732         *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE);
 733         *cmd++ = lower_32_bits(vma->node.start);
 734         *cmd++ = upper_32_bits(vma->node.start);
 735         *cmd = MI_BATCH_BUFFER_END;
 736
 737         __i915_gem_object_flush_map(obj, 0, 64);
 738         i915_gem_object_unpin_map(obj);
 739
 740         vma = i915_vma_instance(obj, vma->vm, NULL);
 741         if (IS_ERR(vma)) {
 742                 err = PTR_ERR(vma);
 743                 goto err;
 744         }
 745
 746         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 747         if (err)
 748                 goto err;
 749
 750         return vma;
 751
 752 err:
 753         i915_gem_object_put(obj);
 754         return ERR_PTR(err);
 755 }
 756
 757 static int
 758 emit_rpcs_query(struct drm_i915_gem_object *obj,
 759                 struct intel_context *ce,
 760                 struct i915_request **rq_out)
 761 {
 762         struct i915_request *rq;
 763         struct i915_vma *batch;
 764         struct i915_vma *vma;
 765         int err;
 766
 767         GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
 768
 769         vma = i915_vma_instance(obj, &ce->gem_context->ppgtt->vm, NULL);
 770         if (IS_ERR(vma))
 771                 return PTR_ERR(vma);
 772
 773         err = i915_gem_object_set_to_gtt_domain(obj, false);
 774         if (err)
 775                 return err;
 776
 777         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 778         if (err)
 779                 return err;
 780
 781         batch = rpcs_query_batch(vma);
 782         if (IS_ERR(batch)) {
 783                 err = PTR_ERR(batch);
 784                 goto err_vma;
 785         }
 786
 787         rq = i915_request_create(ce);
 788         if (IS_ERR(rq)) {
 789                 err = PTR_ERR(rq);
 790                 goto err_batch;
 791         }
 792
 793         err = rq->engine->emit_bb_start(rq,
 794                                         batch->node.start, batch->node.size,
 795                                         0);
 796         if (err)
 797                 goto err_request;
 798
 799         err = i915_vma_move_to_active(batch, rq, 0);
 800         if (err)
 801                 goto skip_request;
 802
 803         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 804         if (err)
 805                 goto skip_request;
 806
 807         i915_gem_object_set_active_reference(batch->obj);
 808         i915_vma_unpin(batch);
 809         i915_vma_close(batch);
 810
 811         i915_vma_unpin(vma);
 812
 813         *rq_out = i915_request_get(rq);
 814
 815         i915_request_add(rq);
 816
 817         return 0;
 818
 819 skip_request:
 820         i915_request_skip(rq, err);
 821 err_request:
 822         i915_request_add(rq);
 823 err_batch:
 824         i915_vma_unpin(batch);
 825 err_vma:
 826         i915_vma_unpin(vma);
 827
 828         return err;
 829 }
 830
 831 #define TEST_IDLE       BIT(0)
 832 #define TEST_BUSY       BIT(1)
 833 #define TEST_RESET      BIT(2)
 834
 835 static int
 836 __sseu_prepare(struct drm_i915_private *i915,
 837                const char *name,
 838                unsigned int flags,
 839                struct intel_context *ce,
 840                struct igt_spinner **spin)
 841 {
 842         struct i915_request *rq;
 843         int ret;
 844
 845         *spin = NULL;
 846         if (!(flags & (TEST_BUSY | TEST_RESET)))
 847                 return 0;
 848
 849         *spin = kzalloc(sizeof(**spin), GFP_KERNEL);
 850         if (!*spin)
 851                 return -ENOMEM;
 852
 853         ret = igt_spinner_init(*spin, i915);
 854         if (ret)
 855                 goto err_free;
 856
 857         rq = igt_spinner_create_request(*spin,
 858                                         ce->gem_context,
 859                                         ce->engine,
 860                                         MI_NOOP);
 861         if (IS_ERR(rq)) {
 862                 ret = PTR_ERR(rq);
 863                 goto err_fini;
 864         }
 865
 866         i915_request_add(rq);
 867
 868         if (!igt_wait_for_spinner(*spin, rq)) {
 869                 pr_err("%s: Spinner failed to start!\n", name);
 870                 ret = -ETIMEDOUT;
 871                 goto err_end;
 872         }
 873
 874         return 0;
 875
 876 err_end:
 877         igt_spinner_end(*spin);
 878 err_fini:
 879         igt_spinner_fini(*spin);
 880 err_free:
 881         kfree(fetch_and_zero(spin));
 882         return ret;
 883 }
 884
 885 static int
 886 __read_slice_count(struct drm_i915_private *i915,
 887                    struct intel_context *ce,
 888                    struct drm_i915_gem_object *obj,
 889                    struct igt_spinner *spin,
 890                    u32 *rpcs)
 891 {
 892         struct i915_request *rq = NULL;
 893         u32 s_mask, s_shift;
 894         unsigned int cnt;
 895         u32 *buf, val;
 896         long ret;
 897
 898         ret = emit_rpcs_query(obj, ce, &rq);
 899         if (ret)
 900                 return ret;
 901
 902         if (spin)
 903                 igt_spinner_end(spin);
 904
 905         ret = i915_request_wait(rq, I915_WAIT_LOCKED, MAX_SCHEDULE_TIMEOUT);
 906         i915_request_put(rq);
 907         if (ret < 0)
 908                 return ret;
 909
 910         buf = i915_gem_object_pin_map(obj, I915_MAP_WB);
 911         if (IS_ERR(buf)) {
 912                 ret = PTR_ERR(buf);
 913                 return ret;
 914         }
 915
 916         if (INTEL_GEN(i915) >= 11) {
 917                 s_mask = GEN11_RPCS_S_CNT_MASK;
 918                 s_shift = GEN11_RPCS_S_CNT_SHIFT;
 919         } else {
 920                 s_mask = GEN8_RPCS_S_CNT_MASK;
 921                 s_shift = GEN8_RPCS_S_CNT_SHIFT;
 922         }
 923
 924         val = *buf;
 925         cnt = (val & s_mask) >> s_shift;
 926         *rpcs = val;
 927
 928         i915_gem_object_unpin_map(obj);
 929
 930         return cnt;
 931 }
 932
 933 static int
 934 __check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected,
 935              const char *prefix, const char *suffix)
 936 {
 937         if (slices == expected)
 938                 return 0;
 939
 940         if (slices < 0) {
 941                 pr_err("%s: %s read slice count failed with %d%s\n",
 942                        name, prefix, slices, suffix);
 943                 return slices;
 944         }
 945
 946         pr_err("%s: %s slice count %d is not %u%s\n",
 947                name, prefix, slices, expected, suffix);
 948
 949         pr_info("RPCS=0x%x; %u%sx%u%s\n",
 950                 rpcs, slices,
 951                 (rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "",
 952                 (rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT,
 953                 (rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : "");
 954
 955         return -EINVAL;
 956 }
 957
 958 static int
 959 __sseu_finish(struct drm_i915_private *i915,
 960               const char *name,
 961               unsigned int flags,
 962               struct intel_context *ce,
 963               struct drm_i915_gem_object *obj,
 964               unsigned int expected,
 965               struct igt_spinner *spin)
 966 {
 967         unsigned int slices = hweight32(ce->engine->sseu.slice_mask);
 968         u32 rpcs = 0;
 969         int ret = 0;
 970
 971         if (flags & TEST_RESET) {
 972                 ret = i915_reset_engine(ce->engine, "sseu");
 973                 if (ret)
 974                         goto out;
 975         }
 976
 977         ret = __read_slice_count(i915, ce, obj,
 978                                  flags & TEST_RESET ? NULL : spin, &rpcs);
 979         ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!");
 980         if (ret)
 981                 goto out;
 982
 983         ret = __read_slice_count(i915, ce->engine->kernel_context, obj,
 984                                  NULL, &rpcs);
 985         ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!");
 986
 987 out:
 988         if (spin)
 989                 igt_spinner_end(spin);
 990
 991         if ((flags & TEST_IDLE) && ret == 0) {
 992                 ret = i915_gem_wait_for_idle(i915,
 993                                              I915_WAIT_LOCKED,
 994                                              MAX_SCHEDULE_TIMEOUT);
 995                 if (ret)
 996                         return ret;
 997
 998                 ret = __read_slice_count(i915, ce, obj, NULL, &rpcs);
 999                 ret = __check_rpcs(name, rpcs, ret, expected,
1000                                    "Context", " after idle!");
1001         }
1002
1003         return ret;
1004 }
1005
1006 static int
1007 __sseu_test(struct drm_i915_private *i915,
1008             const char *name,
1009             unsigned int flags,
1010             struct intel_context *ce,
1011             struct drm_i915_gem_object *obj,
1012             struct intel_sseu sseu)
1013 {
1014         struct igt_spinner *spin = NULL;
1015         int ret;
1016
1017         ret = __sseu_prepare(i915, name, flags, ce, &spin);
1018         if (ret)
1019                 return ret;
1020
1021         ret = __intel_context_reconfigure_sseu(ce, sseu);
1022         if (ret)
1023                 goto out_spin;
1024
1025         ret = __sseu_finish(i915, name, flags, ce, obj,
1026                             hweight32(sseu.slice_mask), spin);
1027
1028 out_spin:
1029         if (spin) {
1030                 igt_spinner_end(spin);
1031                 igt_spinner_fini(spin);
1032                 kfree(spin);
1033         }
1034         return ret;
1035 }
1036
1037 static int
1038 __igt_ctx_sseu(struct drm_i915_private *i915,
1039                const char *name,
1040                unsigned int flags)
1041 {
1042         struct intel_engine_cs *engine = i915->engine[RCS0];
1043         struct intel_sseu default_sseu = engine->sseu;
1044         struct drm_i915_gem_object *obj;
1045         struct i915_gem_context *ctx;
1046         struct intel_context *ce;
1047         struct intel_sseu pg_sseu;
1048         intel_wakeref_t wakeref;
1049         struct drm_file *file;
1050         int ret;
1051
1052         if (INTEL_GEN(i915) < 9)
1053                 return 0;
1054
1055         if (!RUNTIME_INFO(i915)->sseu.has_slice_pg)
1056                 return 0;
1057
1058         if (hweight32(default_sseu.slice_mask) < 2)
1059                 return 0;
1060
1061         /*
1062          * Gen11 VME friendly power-gated configuration with half enabled
1063          * sub-slices.
1064          */
1065         pg_sseu = default_sseu;
1066         pg_sseu.slice_mask = 1;
1067         pg_sseu.subslice_mask =
1068                 ~(~0 << (hweight32(default_sseu.subslice_mask) / 2));
1069
1070         pr_info("SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n",
1071                 name, flags, hweight32(default_sseu.slice_mask),
1072                 hweight32(pg_sseu.slice_mask));
1073
1074         file = mock_file(i915);
1075         if (IS_ERR(file))
1076                 return PTR_ERR(file);
1077
1078         if (flags & TEST_RESET)
1079                 igt_global_reset_lock(i915);
1080
1081         mutex_lock(&i915->drm.struct_mutex);
1082
1083         ctx = live_context(i915, file);
1084         if (IS_ERR(ctx)) {
1085                 ret = PTR_ERR(ctx);
1086                 goto out_unlock;
1087         }
1088         i915_gem_context_clear_bannable(ctx); /* to reset and beyond! */
1089
1090         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1091         if (IS_ERR(obj)) {
1092                 ret = PTR_ERR(obj);
1093                 goto out_unlock;
1094         }
1095
1096         wakeref = intel_runtime_pm_get(i915);
1097
1098         ce = i915_gem_context_get_engine(ctx, RCS0);
1099         if (IS_ERR(ce)) {
1100                 ret = PTR_ERR(ce);
1101                 goto out_rpm;
1102         }
1103
1104         ret = intel_context_pin(ce);
1105         if (ret)
1106                 goto out_context;
1107
1108         /* First set the default mask. */
1109         ret = __sseu_test(i915, name, flags, ce, obj, default_sseu);
1110         if (ret)
1111                 goto out_fail;
1112
1113         /* Then set a power-gated configuration. */
1114         ret = __sseu_test(i915, name, flags, ce, obj, pg_sseu);
1115         if (ret)
1116                 goto out_fail;
1117
1118         /* Back to defaults. */
1119         ret = __sseu_test(i915, name, flags, ce, obj, default_sseu);
1120         if (ret)
1121                 goto out_fail;
1122
1123         /* One last power-gated configuration for the road. */
1124         ret = __sseu_test(i915, name, flags, ce, obj, pg_sseu);
1125         if (ret)
1126                 goto out_fail;
1127
1128 out_fail:
1129         if (igt_flush_test(i915, I915_WAIT_LOCKED))
1130                 ret = -EIO;
1131
1132         intel_context_unpin(ce);
1133 out_context:
1134         intel_context_put(ce);
1135 out_rpm:
1136         intel_runtime_pm_put(i915, wakeref);
1137         i915_gem_object_put(obj);
1138
1139 out_unlock:
1140         mutex_unlock(&i915->drm.struct_mutex);
1141
1142         if (flags & TEST_RESET)
1143                 igt_global_reset_unlock(i915);
1144
1145         mock_file_free(i915, file);
1146
1147         if (ret)
1148                 pr_err("%s: Failed with %d!\n", name, ret);
1149
1150         return ret;
1151 }
1152
1153 static int igt_ctx_sseu(void *arg)
1154 {
1155         struct {
1156                 const char *name;
1157                 unsigned int flags;
1158         } *phase, phases[] = {
1159                 { .name = "basic", .flags = 0 },
1160                 { .name = "idle", .flags = TEST_IDLE },
1161                 { .name = "busy", .flags = TEST_BUSY },
1162                 { .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET },
1163                 { .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE },
1164                 { .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE },
1165         };
1166         unsigned int i;
1167         int ret = 0;
1168
1169         for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases);
1170              i++, phase++)
1171                 ret = __igt_ctx_sseu(arg, phase->name, phase->flags);
1172
1173         return ret;
1174 }
1175
1176 static int igt_ctx_readonly(void *arg)
1177 {
1178         struct drm_i915_private *i915 = arg;
1179         struct drm_i915_gem_object *obj = NULL;
1180         struct i915_gem_context *ctx;
1181         struct i915_hw_ppgtt *ppgtt;
1182         unsigned long idx, ndwords, dw;
1183         struct igt_live_test t;
1184         struct drm_file *file;
1185         I915_RND_STATE(prng);
1186         IGT_TIMEOUT(end_time);
1187         LIST_HEAD(objects);
1188         int err = -ENODEV;
1189
1190         /*
1191          * Create a few read-only objects (with the occasional writable object)
1192          * and try to write into these object checking that the GPU discards
1193          * any write to a read-only object.
1194          */
1195
1196         file = mock_file(i915);
1197         if (IS_ERR(file))
1198                 return PTR_ERR(file);
1199
1200         mutex_lock(&i915->drm.struct_mutex);
1201
1202         err = igt_live_test_begin(&t, i915, __func__, "");
1203         if (err)
1204                 goto out_unlock;
1205
1206         ctx = live_context(i915, file);
1207         if (IS_ERR(ctx)) {
1208                 err = PTR_ERR(ctx);
1209                 goto out_unlock;
1210         }
1211
1212         ppgtt = ctx->ppgtt ?: i915->mm.aliasing_ppgtt;
1213         if (!ppgtt || !ppgtt->vm.has_read_only) {
1214                 err = 0;
1215                 goto out_unlock;
1216         }
1217
1218         ndwords = 0;
1219         dw = 0;
1220         while (!time_after(jiffies, end_time)) {
1221                 struct intel_engine_cs *engine;
1222                 unsigned int id;
1223
1224                 for_each_engine(engine, i915, id) {
1225                         intel_wakeref_t wakeref;
1226
1227                         if (!intel_engine_can_store_dword(engine))
1228                                 continue;
1229
1230                         if (!obj) {
1231                                 obj = create_test_object(ctx, file, &objects);
1232                                 if (IS_ERR(obj)) {
1233                                         err = PTR_ERR(obj);
1234                                         goto out_unlock;
1235                                 }
1236
1237                                 if (prandom_u32_state(&prng) & 1)
1238                                         i915_gem_object_set_readonly(obj);
1239                         }
1240
1241                         err = 0;
1242                         with_intel_runtime_pm(i915, wakeref)
1243                                 err = gpu_fill(obj, ctx, engine, dw);
1244                         if (err) {
1245                                 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
1246                                        ndwords, dw, max_dwords(obj),
1247                                        engine->name, ctx->hw_id,
1248                                        yesno(!!ctx->ppgtt), err);
1249                                 goto out_unlock;
1250                         }
1251
1252                         if (++dw == max_dwords(obj)) {
1253                                 obj = NULL;
1254                                 dw = 0;
1255                         }
1256                         ndwords++;
1257                 }
1258         }
1259         pr_info("Submitted %lu dwords (across %u engines)\n",
1260                 ndwords, RUNTIME_INFO(i915)->num_engines);
1261
1262         dw = 0;
1263         idx = 0;
1264         list_for_each_entry(obj, &objects, st_link) {
1265                 unsigned int rem =
1266                         min_t(unsigned int, ndwords - dw, max_dwords(obj));
1267                 unsigned int num_writes;
1268
1269                 num_writes = rem;
1270                 if (i915_gem_object_is_readonly(obj))
1271                         num_writes = 0;
1272
1273                 err = cpu_check(obj, idx++, num_writes);
1274                 if (err)
1275                         break;
1276
1277                 dw += rem;
1278         }
1279
1280 out_unlock:
1281         if (igt_live_test_end(&t))
1282                 err = -EIO;
1283         mutex_unlock(&i915->drm.struct_mutex);
1284
1285         mock_file_free(i915, file);
1286         return err;
1287 }
1288
1289 static int check_scratch(struct i915_gem_context *ctx, u64 offset)
1290 {
1291         struct drm_mm_node *node =
1292                 __drm_mm_interval_first(&ctx->ppgtt->vm.mm,
1293                                         offset, offset + sizeof(u32) - 1);
1294         if (!node || node->start > offset)
1295                 return 0;
1296
1297         GEM_BUG_ON(offset >= node->start + node->size);
1298
1299         pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n",
1300                upper_32_bits(offset), lower_32_bits(offset));
1301         return -EINVAL;
1302 }
1303
1304 static int write_to_scratch(struct i915_gem_context *ctx,
1305                             struct intel_engine_cs *engine,
1306                             u64 offset, u32 value)
1307 {
1308         struct drm_i915_private *i915 = ctx->i915;
1309         struct drm_i915_gem_object *obj;
1310         struct i915_request *rq;
1311         struct i915_vma *vma;
1312         u32 *cmd;
1313         int err;
1314
1315         GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1316
1317         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1318         if (IS_ERR(obj))
1319                 return PTR_ERR(obj);
1320
1321         cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1322         if (IS_ERR(cmd)) {
1323                 err = PTR_ERR(cmd);
1324                 goto err;
1325         }
1326
1327         *cmd++ = MI_STORE_DWORD_IMM_GEN4;
1328         if (INTEL_GEN(i915) >= 8) {
1329                 *cmd++ = lower_32_bits(offset);
1330                 *cmd++ = upper_32_bits(offset);
1331         } else {
1332                 *cmd++ = 0;
1333                 *cmd++ = offset;
1334         }
1335         *cmd++ = value;
1336         *cmd = MI_BATCH_BUFFER_END;
1337         __i915_gem_object_flush_map(obj, 0, 64);
1338         i915_gem_object_unpin_map(obj);
1339
1340         vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL);
1341         if (IS_ERR(vma)) {
1342                 err = PTR_ERR(vma);
1343                 goto err;
1344         }
1345
1346         err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1347         if (err)
1348                 goto err;
1349
1350         err = check_scratch(ctx, offset);
1351         if (err)
1352                 goto err_unpin;
1353
1354         rq = igt_request_alloc(ctx, engine);
1355         if (IS_ERR(rq)) {
1356                 err = PTR_ERR(rq);
1357                 goto err_unpin;
1358         }
1359
1360         err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0);
1361         if (err)
1362                 goto err_request;
1363
1364         err = i915_vma_move_to_active(vma, rq, 0);
1365         if (err)
1366                 goto skip_request;
1367
1368         i915_gem_object_set_active_reference(obj);
1369         i915_vma_unpin(vma);
1370         i915_vma_close(vma);
1371
1372         i915_request_add(rq);
1373
1374         return 0;
1375
1376 skip_request:
1377         i915_request_skip(rq, err);
1378 err_request:
1379         i915_request_add(rq);
1380 err_unpin:
1381         i915_vma_unpin(vma);
1382 err:
1383         i915_gem_object_put(obj);
1384         return err;
1385 }
1386
1387 static int read_from_scratch(struct i915_gem_context *ctx,
1388                              struct intel_engine_cs *engine,
1389                              u64 offset, u32 *value)
1390 {
1391         struct drm_i915_private *i915 = ctx->i915;
1392         struct drm_i915_gem_object *obj;
1393         const u32 RCS_GPR0 = 0x2600; /* not all engines have their own GPR! */
1394         const u32 result = 0x100;
1395         struct i915_request *rq;
1396         struct i915_vma *vma;
1397         u32 *cmd;
1398         int err;
1399
1400         GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1401
1402         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1403         if (IS_ERR(obj))
1404                 return PTR_ERR(obj);
1405
1406         cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1407         if (IS_ERR(cmd)) {
1408                 err = PTR_ERR(cmd);
1409                 goto err;
1410         }
1411
1412         memset(cmd, POISON_INUSE, PAGE_SIZE);
1413         if (INTEL_GEN(i915) >= 8) {
1414                 *cmd++ = MI_LOAD_REGISTER_MEM_GEN8;
1415                 *cmd++ = RCS_GPR0;
1416                 *cmd++ = lower_32_bits(offset);
1417                 *cmd++ = upper_32_bits(offset);
1418                 *cmd++ = MI_STORE_REGISTER_MEM_GEN8;
1419                 *cmd++ = RCS_GPR0;
1420                 *cmd++ = result;
1421                 *cmd++ = 0;
1422         } else {
1423                 *cmd++ = MI_LOAD_REGISTER_MEM;
1424                 *cmd++ = RCS_GPR0;
1425                 *cmd++ = offset;
1426                 *cmd++ = MI_STORE_REGISTER_MEM;
1427                 *cmd++ = RCS_GPR0;
1428                 *cmd++ = result;
1429         }
1430         *cmd = MI_BATCH_BUFFER_END;
1431
1432         i915_gem_object_flush_map(obj);
1433         i915_gem_object_unpin_map(obj);
1434
1435         vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL);
1436         if (IS_ERR(vma)) {
1437                 err = PTR_ERR(vma);
1438                 goto err;
1439         }
1440
1441         err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1442         if (err)
1443                 goto err;
1444
1445         err = check_scratch(ctx, offset);
1446         if (err)
1447                 goto err_unpin;
1448
1449         rq = igt_request_alloc(ctx, engine);
1450         if (IS_ERR(rq)) {
1451                 err = PTR_ERR(rq);
1452                 goto err_unpin;
1453         }
1454
1455         err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0);
1456         if (err)
1457                 goto err_request;
1458
1459         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1460         if (err)
1461                 goto skip_request;
1462
1463         i915_vma_unpin(vma);
1464         i915_vma_close(vma);
1465
1466         i915_request_add(rq);
1467
1468         err = i915_gem_object_set_to_cpu_domain(obj, false);
1469         if (err)
1470                 goto err;
1471
1472         cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1473         if (IS_ERR(cmd)) {
1474                 err = PTR_ERR(cmd);
1475                 goto err;
1476         }
1477
1478         *value = cmd[result / sizeof(*cmd)];
1479         i915_gem_object_unpin_map(obj);
1480         i915_gem_object_put(obj);
1481
1482         return 0;
1483
1484 skip_request:
1485         i915_request_skip(rq, err);
1486 err_request:
1487         i915_request_add(rq);
1488 err_unpin:
1489         i915_vma_unpin(vma);
1490 err:
1491         i915_gem_object_put(obj);
1492         return err;
1493 }
1494
1495 static int igt_vm_isolation(void *arg)
1496 {
1497         struct drm_i915_private *i915 = arg;
1498         struct i915_gem_context *ctx_a, *ctx_b;
1499         struct intel_engine_cs *engine;
1500         intel_wakeref_t wakeref;
1501         struct igt_live_test t;
1502         struct drm_file *file;
1503         I915_RND_STATE(prng);
1504         unsigned long count;
1505         unsigned int id;
1506         u64 vm_total;
1507         int err;
1508
1509         if (INTEL_GEN(i915) < 7)
1510                 return 0;
1511
1512         /*
1513          * The simple goal here is that a write into one context is not
1514          * observed in a second (separate page tables and scratch).
1515          */
1516
1517         file = mock_file(i915);
1518         if (IS_ERR(file))
1519                 return PTR_ERR(file);
1520
1521         mutex_lock(&i915->drm.struct_mutex);
1522
1523         err = igt_live_test_begin(&t, i915, __func__, "");
1524         if (err)
1525                 goto out_unlock;
1526
1527         ctx_a = live_context(i915, file);
1528         if (IS_ERR(ctx_a)) {
1529                 err = PTR_ERR(ctx_a);
1530                 goto out_unlock;
1531         }
1532
1533         ctx_b = live_context(i915, file);
1534         if (IS_ERR(ctx_b)) {
1535                 err = PTR_ERR(ctx_b);
1536                 goto out_unlock;
1537         }
1538
1539         /* We can only test vm isolation, if the vm are distinct */
1540         if (ctx_a->ppgtt == ctx_b->ppgtt)
1541                 goto out_unlock;
1542
1543         vm_total = ctx_a->ppgtt->vm.total;
1544         GEM_BUG_ON(ctx_b->ppgtt->vm.total != vm_total);
1545         vm_total -= I915_GTT_PAGE_SIZE;
1546
1547         wakeref = intel_runtime_pm_get(i915);
1548
1549         count = 0;
1550         for_each_engine(engine, i915, id) {
1551                 IGT_TIMEOUT(end_time);
1552                 unsigned long this = 0;
1553
1554                 if (!intel_engine_can_store_dword(engine))
1555                         continue;
1556
1557                 while (!__igt_timeout(end_time, NULL)) {
1558                         u32 value = 0xc5c5c5c5;
1559                         u64 offset;
1560
1561                         div64_u64_rem(i915_prandom_u64_state(&prng),
1562                                       vm_total, &offset);
1563                         offset &= -sizeof(u32);
1564                         offset += I915_GTT_PAGE_SIZE;
1565
1566                         err = write_to_scratch(ctx_a, engine,
1567                                                offset, 0xdeadbeef);
1568                         if (err == 0)
1569                                 err = read_from_scratch(ctx_b, engine,
1570                                                         offset, &value);
1571                         if (err)
1572                                 goto out_rpm;
1573
1574                         if (value) {
1575                                 pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n",
1576                                        engine->name, value,
1577                                        upper_32_bits(offset),
1578                                        lower_32_bits(offset),
1579                                        this);
1580                                 err = -EINVAL;
1581                                 goto out_rpm;
1582                         }
1583
1584                         this++;
1585                 }
1586                 count += this;
1587         }
1588         pr_info("Checked %lu scratch offsets across %d engines\n",
1589                 count, RUNTIME_INFO(i915)->num_engines);
1590
1591 out_rpm:
1592         intel_runtime_pm_put(i915, wakeref);
1593 out_unlock:
1594         if (igt_live_test_end(&t))
1595                 err = -EIO;
1596         mutex_unlock(&i915->drm.struct_mutex);
1597
1598         mock_file_free(i915, file);
1599         return err;
1600 }
1601
1602 static __maybe_unused const char *
1603 __engine_name(struct drm_i915_private *i915, intel_engine_mask_t engines)
1604 {
1605         struct intel_engine_cs *engine;
1606         intel_engine_mask_t tmp;
1607
1608         if (engines == ALL_ENGINES)
1609                 return "all";
1610
1611         for_each_engine_masked(engine, i915, engines, tmp)
1612                 return engine->name;
1613
1614         return "none";
1615 }
1616
1617 static void mock_barrier_task(void *data)
1618 {
1619         unsigned int *counter = data;
1620
1621         ++*counter;
1622 }
1623
1624 static int mock_context_barrier(void *arg)
1625 {
1626 #undef pr_fmt
1627 #define pr_fmt(x) "context_barrier_task():" # x
1628         struct drm_i915_private *i915 = arg;
1629         struct i915_gem_context *ctx;
1630         struct i915_request *rq;
1631         unsigned int counter;
1632         int err;
1633
1634         /*
1635          * The context barrier provides us with a callback after it emits
1636          * a request; useful for retiring old state after loading new.
1637          */
1638
1639         mutex_lock(&i915->drm.struct_mutex);
1640
1641         ctx = mock_context(i915, "mock");
1642         if (!ctx) {
1643                 err = -ENOMEM;
1644                 goto unlock;
1645         }
1646
1647         counter = 0;
1648         err = context_barrier_task(ctx, 0,
1649                                    NULL, mock_barrier_task, &counter);
1650         if (err) {
1651                 pr_err("Failed at line %d, err=%d\n", __LINE__, err);
1652                 goto out;
1653         }
1654         if (counter == 0) {
1655                 pr_err("Did not retire immediately with 0 engines\n");
1656                 err = -EINVAL;
1657                 goto out;
1658         }
1659
1660         counter = 0;
1661         err = context_barrier_task(ctx, ALL_ENGINES,
1662                                    NULL, mock_barrier_task, &counter);
1663         if (err) {
1664                 pr_err("Failed at line %d, err=%d\n", __LINE__, err);
1665                 goto out;
1666         }
1667         if (counter == 0) {
1668                 pr_err("Did not retire immediately for all unused engines\n");
1669                 err = -EINVAL;
1670                 goto out;
1671         }
1672
1673         rq = igt_request_alloc(ctx, i915->engine[RCS0]);
1674         if (IS_ERR(rq)) {
1675                 pr_err("Request allocation failed!\n");
1676                 goto out;
1677         }
1678         i915_request_add(rq);
1679
1680         counter = 0;
1681         context_barrier_inject_fault = BIT(RCS0);
1682         err = context_barrier_task(ctx, ALL_ENGINES,
1683                                    NULL, mock_barrier_task, &counter);
1684         context_barrier_inject_fault = 0;
1685         if (err == -ENXIO)
1686                 err = 0;
1687         else
1688                 pr_err("Did not hit fault injection!\n");
1689         if (counter != 0) {
1690                 pr_err("Invoked callback on error!\n");
1691                 err = -EIO;
1692         }
1693         if (err)
1694                 goto out;
1695
1696         counter = 0;
1697         err = context_barrier_task(ctx, ALL_ENGINES,
1698                                    NULL, mock_barrier_task, &counter);
1699         if (err) {
1700                 pr_err("Failed at line %d, err=%d\n", __LINE__, err);
1701                 goto out;
1702         }
1703         mock_device_flush(i915);
1704         if (counter == 0) {
1705                 pr_err("Did not retire on each active engines\n");
1706                 err = -EINVAL;
1707                 goto out;
1708         }
1709
1710 out:
1711         mock_context_close(ctx);
1712 unlock:
1713         mutex_unlock(&i915->drm.struct_mutex);
1714         return err;
1715 #undef pr_fmt
1716 #define pr_fmt(x) x
1717 }
1718
1719 int i915_gem_context_mock_selftests(void)
1720 {
1721         static const struct i915_subtest tests[] = {
1722                 SUBTEST(mock_context_barrier),
1723         };
1724         struct drm_i915_private *i915;
1725         int err;
1726
1727         i915 = mock_gem_device();
1728         if (!i915)
1729                 return -ENOMEM;
1730
1731         err = i915_subtests(tests, i915);
1732
1733         drm_dev_put(&i915->drm);
1734         return err;
1735 }
1736
1737 int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv)
1738 {
1739         static const struct i915_subtest tests[] = {
1740                 SUBTEST(live_nop_switch),
1741                 SUBTEST(igt_ctx_exec),
1742                 SUBTEST(igt_ctx_readonly),
1743                 SUBTEST(igt_ctx_sseu),
1744                 SUBTEST(igt_shared_ctx_exec),
1745                 SUBTEST(igt_vm_isolation),
1746         };
1747
1748         if (i915_terminally_wedged(dev_priv))
1749                 return 0;
1750
1751         return i915_subtests(tests, dev_priv);
1752 }