drivers/gpu/drm/i915/selftests/i915_request.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/prime_numbers.h>
  26 #include <linux/pm_qos.h>
  27 #include <linux/sort.h>
  28
  29 #include "gem/i915_gem_pm.h"
  30 #include "gem/selftests/mock_context.h"
  31
  32 #include "gt/intel_engine_heartbeat.h"
  33 #include "gt/intel_engine_pm.h"
  34 #include "gt/intel_engine_user.h"
  35 #include "gt/intel_gt.h"
  36 #include "gt/intel_gt_clock_utils.h"
  37 #include "gt/intel_gt_requests.h"
  38 #include "gt/selftest_engine_heartbeat.h"
  39
  40 #include "i915_random.h"
  41 #include "i915_selftest.h"
  42 #include "igt_flush_test.h"
  43 #include "igt_live_test.h"
  44 #include "igt_spinner.h"
  45 #include "lib_sw_fence.h"
  46
  47 #include "mock_drm.h"
  48 #include "mock_gem_device.h"
  49
  50 static unsigned int num_uabi_engines(struct drm_i915_private *i915)
  51 {
  52         struct intel_engine_cs *engine;
  53         unsigned int count;
  54
  55         count = 0;
  56         for_each_uabi_engine(engine, i915)
  57                 count++;
  58
  59         return count;
  60 }
  61
  62 static struct intel_engine_cs *rcs0(struct drm_i915_private *i915)
  63 {
  64         return intel_engine_lookup_user(i915, I915_ENGINE_CLASS_RENDER, 0);
  65 }
  66
  67 static int igt_add_request(void *arg)
  68 {
  69         struct drm_i915_private *i915 = arg;
  70         struct i915_request *request;
  71
  72         /* Basic preliminary test to create a request and let it loose! */
  73
  74         request = mock_request(rcs0(i915)->kernel_context, HZ / 10);
  75         if (!request)
  76                 return -ENOMEM;
  77
  78         i915_request_add(request);
  79
  80         return 0;
  81 }
  82
  83 static int igt_wait_request(void *arg)
  84 {
  85         const long T = HZ / 4;
  86         struct drm_i915_private *i915 = arg;
  87         struct i915_request *request;
  88         int err = -EINVAL;
  89
  90         /* Submit a request, then wait upon it */
  91
  92         request = mock_request(rcs0(i915)->kernel_context, T);
  93         if (!request)
  94                 return -ENOMEM;
  95
  96         i915_request_get(request);
  97
  98         if (i915_request_wait(request, 0, 0) != -ETIME) {
  99                 pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n");
 100                 goto out_request;
 101         }
 102
 103         if (i915_request_wait(request, 0, T) != -ETIME) {
 104                 pr_err("request wait succeeded (expected timeout before submit!)\n");
 105                 goto out_request;
 106         }
 107
 108         if (i915_request_completed(request)) {
 109                 pr_err("request completed before submit!!\n");
 110                 goto out_request;
 111         }
 112
 113         i915_request_add(request);
 114
 115         if (i915_request_wait(request, 0, 0) != -ETIME) {
 116                 pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n");
 117                 goto out_request;
 118         }
 119
 120         if (i915_request_completed(request)) {
 121                 pr_err("request completed immediately!\n");
 122                 goto out_request;
 123         }
 124
 125         if (i915_request_wait(request, 0, T / 2) != -ETIME) {
 126                 pr_err("request wait succeeded (expected timeout!)\n");
 127                 goto out_request;
 128         }
 129
 130         if (i915_request_wait(request, 0, T) == -ETIME) {
 131                 pr_err("request wait timed out!\n");
 132                 goto out_request;
 133         }
 134
 135         if (!i915_request_completed(request)) {
 136                 pr_err("request not complete after waiting!\n");
 137                 goto out_request;
 138         }
 139
 140         if (i915_request_wait(request, 0, T) == -ETIME) {
 141                 pr_err("request wait timed out when already complete!\n");
 142                 goto out_request;
 143         }
 144
 145         err = 0;
 146 out_request:
 147         i915_request_put(request);
 148         mock_device_flush(i915);
 149         return err;
 150 }
 151
 152 static int igt_fence_wait(void *arg)
 153 {
 154         const long T = HZ / 4;
 155         struct drm_i915_private *i915 = arg;
 156         struct i915_request *request;
 157         int err = -EINVAL;
 158
 159         /* Submit a request, treat it as a fence and wait upon it */
 160
 161         request = mock_request(rcs0(i915)->kernel_context, T);
 162         if (!request)
 163                 return -ENOMEM;
 164
 165         if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) {
 166                 pr_err("fence wait success before submit (expected timeout)!\n");
 167                 goto out;
 168         }
 169
 170         i915_request_add(request);
 171
 172         if (dma_fence_is_signaled(&request->fence)) {
 173                 pr_err("fence signaled immediately!\n");
 174                 goto out;
 175         }
 176
 177         if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) {
 178                 pr_err("fence wait success after submit (expected timeout)!\n");
 179                 goto out;
 180         }
 181
 182         if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
 183                 pr_err("fence wait timed out (expected success)!\n");
 184                 goto out;
 185         }
 186
 187         if (!dma_fence_is_signaled(&request->fence)) {
 188                 pr_err("fence unsignaled after waiting!\n");
 189                 goto out;
 190         }
 191
 192         if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
 193                 pr_err("fence wait timed out when complete (expected success)!\n");
 194                 goto out;
 195         }
 196
 197         err = 0;
 198 out:
 199         mock_device_flush(i915);
 200         return err;
 201 }
 202
 203 static int igt_request_rewind(void *arg)
 204 {
 205         struct drm_i915_private *i915 = arg;
 206         struct i915_request *request, *vip;
 207         struct i915_gem_context *ctx[2];
 208         struct intel_context *ce;
 209         int err = -EINVAL;
 210
 211         ctx[0] = mock_context(i915, "A");
 212
 213         ce = i915_gem_context_get_engine(ctx[0], RCS0);
 214         GEM_BUG_ON(IS_ERR(ce));
 215         request = mock_request(ce, 2 * HZ);
 216         intel_context_put(ce);
 217         if (!request) {
 218                 err = -ENOMEM;
 219                 goto err_context_0;
 220         }
 221
 222         i915_request_get(request);
 223         i915_request_add(request);
 224
 225         ctx[1] = mock_context(i915, "B");
 226
 227         ce = i915_gem_context_get_engine(ctx[1], RCS0);
 228         GEM_BUG_ON(IS_ERR(ce));
 229         vip = mock_request(ce, 0);
 230         intel_context_put(ce);
 231         if (!vip) {
 232                 err = -ENOMEM;
 233                 goto err_context_1;
 234         }
 235
 236         /* Simulate preemption by manual reordering */
 237         if (!mock_cancel_request(request)) {
 238                 pr_err("failed to cancel request (already executed)!\n");
 239                 i915_request_add(vip);
 240                 goto err_context_1;
 241         }
 242         i915_request_get(vip);
 243         i915_request_add(vip);
 244         rcu_read_lock();
 245         request->engine->submit_request(request);
 246         rcu_read_unlock();
 247
 248
 249         if (i915_request_wait(vip, 0, HZ) == -ETIME) {
 250                 pr_err("timed out waiting for high priority request\n");
 251                 goto err;
 252         }
 253
 254         if (i915_request_completed(request)) {
 255                 pr_err("low priority request already completed\n");
 256                 goto err;
 257         }
 258
 259         err = 0;
 260 err:
 261         i915_request_put(vip);
 262 err_context_1:
 263         mock_context_close(ctx[1]);
 264         i915_request_put(request);
 265 err_context_0:
 266         mock_context_close(ctx[0]);
 267         mock_device_flush(i915);
 268         return err;
 269 }
 270
 271 struct smoketest {
 272         struct intel_engine_cs *engine;
 273         struct i915_gem_context **contexts;
 274         atomic_long_t num_waits, num_fences;
 275         int ncontexts, max_batch;
 276         struct i915_request *(*request_alloc)(struct intel_context *ce);
 277 };
 278
 279 static struct i915_request *
 280 __mock_request_alloc(struct intel_context *ce)
 281 {
 282         return mock_request(ce, 0);
 283 }
 284
 285 static struct i915_request *
 286 __live_request_alloc(struct intel_context *ce)
 287 {
 288         return intel_context_create_request(ce);
 289 }
 290
 291 static int __igt_breadcrumbs_smoketest(void *arg)
 292 {
 293         struct smoketest *t = arg;
 294         const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1;
 295         const unsigned int total = 4 * t->ncontexts + 1;
 296         unsigned int num_waits = 0, num_fences = 0;
 297         struct i915_request **requests;
 298         I915_RND_STATE(prng);
 299         unsigned int *order;
 300         int err = 0;
 301
 302         /*
 303          * A very simple test to catch the most egregious of list handling bugs.
 304          *
 305          * At its heart, we simply create oodles of requests running across
 306          * multiple kthreads and enable signaling on them, for the sole purpose
 307          * of stressing our breadcrumb handling. The only inspection we do is
 308          * that the fences were marked as signaled.
 309          */
 310
 311         requests = kcalloc(total, sizeof(*requests), GFP_KERNEL);
 312         if (!requests)
 313                 return -ENOMEM;
 314
 315         order = i915_random_order(total, &prng);
 316         if (!order) {
 317                 err = -ENOMEM;
 318                 goto out_requests;
 319         }
 320
 321         while (!kthread_should_stop()) {
 322                 struct i915_sw_fence *submit, *wait;
 323                 unsigned int n, count;
 324
 325                 submit = heap_fence_create(GFP_KERNEL);
 326                 if (!submit) {
 327                         err = -ENOMEM;
 328                         break;
 329                 }
 330
 331                 wait = heap_fence_create(GFP_KERNEL);
 332                 if (!wait) {
 333                         i915_sw_fence_commit(submit);
 334                         heap_fence_put(submit);
 335                         err = -ENOMEM;
 336                         break;
 337                 }
 338
 339                 i915_random_reorder(order, total, &prng);
 340                 count = 1 + i915_prandom_u32_max_state(max_batch, &prng);
 341
 342                 for (n = 0; n < count; n++) {
 343                         struct i915_gem_context *ctx =
 344                                 t->contexts[order[n] % t->ncontexts];
 345                         struct i915_request *rq;
 346                         struct intel_context *ce;
 347
 348                         ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx);
 349                         GEM_BUG_ON(IS_ERR(ce));
 350                         rq = t->request_alloc(ce);
 351                         intel_context_put(ce);
 352                         if (IS_ERR(rq)) {
 353                                 err = PTR_ERR(rq);
 354                                 count = n;
 355                                 break;
 356                         }
 357
 358                         err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
 359                                                                submit,
 360                                                                GFP_KERNEL);
 361
 362                         requests[n] = i915_request_get(rq);
 363                         i915_request_add(rq);
 364
 365                         if (err >= 0)
 366                                 err = i915_sw_fence_await_dma_fence(wait,
 367                                                                     &rq->fence,
 368                                                                     0,
 369                                                                     GFP_KERNEL);
 370
 371                         if (err < 0) {
 372                                 i915_request_put(rq);
 373                                 count = n;
 374                                 break;
 375                         }
 376                 }
 377
 378                 i915_sw_fence_commit(submit);
 379                 i915_sw_fence_commit(wait);
 380
 381                 if (!wait_event_timeout(wait->wait,
 382                                         i915_sw_fence_done(wait),
 383                                         5 * HZ)) {
 384                         struct i915_request *rq = requests[count - 1];
 385
 386                         pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n",
 387                                atomic_read(&wait->pending), count,
 388                                rq->fence.context, rq->fence.seqno,
 389                                t->engine->name);
 390                         GEM_TRACE_DUMP();
 391
 392                         intel_gt_set_wedged(t->engine->gt);
 393                         GEM_BUG_ON(!i915_request_completed(rq));
 394                         i915_sw_fence_wait(wait);
 395                         err = -EIO;
 396                 }
 397
 398                 for (n = 0; n < count; n++) {
 399                         struct i915_request *rq = requests[n];
 400
 401                         if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 402                                       &rq->fence.flags)) {
 403                                 pr_err("%llu:%llu was not signaled!\n",
 404                                        rq->fence.context, rq->fence.seqno);
 405                                 err = -EINVAL;
 406                         }
 407
 408                         i915_request_put(rq);
 409                 }
 410
 411                 heap_fence_put(wait);
 412                 heap_fence_put(submit);
 413
 414                 if (err < 0)
 415                         break;
 416
 417                 num_fences += count;
 418                 num_waits++;
 419
 420                 cond_resched();
 421         }
 422
 423         atomic_long_add(num_fences, &t->num_fences);
 424         atomic_long_add(num_waits, &t->num_waits);
 425
 426         kfree(order);
 427 out_requests:
 428         kfree(requests);
 429         return err;
 430 }
 431
 432 static int mock_breadcrumbs_smoketest(void *arg)
 433 {
 434         struct drm_i915_private *i915 = arg;
 435         struct smoketest t = {
 436                 .engine = rcs0(i915),
 437                 .ncontexts = 1024,
 438                 .max_batch = 1024,
 439                 .request_alloc = __mock_request_alloc
 440         };
 441         unsigned int ncpus = num_online_cpus();
 442         struct task_struct **threads;
 443         unsigned int n;
 444         int ret = 0;
 445
 446         /*
 447          * Smoketest our breadcrumb/signal handling for requests across multiple
 448          * threads. A very simple test to only catch the most egregious of bugs.
 449          * See __igt_breadcrumbs_smoketest();
 450          */
 451
 452         threads = kcalloc(ncpus, sizeof(*threads), GFP_KERNEL);
 453         if (!threads)
 454                 return -ENOMEM;
 455
 456         t.contexts = kcalloc(t.ncontexts, sizeof(*t.contexts), GFP_KERNEL);
 457         if (!t.contexts) {
 458                 ret = -ENOMEM;
 459                 goto out_threads;
 460         }
 461
 462         for (n = 0; n < t.ncontexts; n++) {
 463                 t.contexts[n] = mock_context(t.engine->i915, "mock");
 464                 if (!t.contexts[n]) {
 465                         ret = -ENOMEM;
 466                         goto out_contexts;
 467                 }
 468         }
 469
 470         for (n = 0; n < ncpus; n++) {
 471                 threads[n] = kthread_run(__igt_breadcrumbs_smoketest,
 472                                          &t, "igt/%d", n);
 473                 if (IS_ERR(threads[n])) {
 474                         ret = PTR_ERR(threads[n]);
 475                         ncpus = n;
 476                         break;
 477                 }
 478
 479                 get_task_struct(threads[n]);
 480         }
 481
 482         yield(); /* start all threads before we begin */
 483         msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
 484
 485         for (n = 0; n < ncpus; n++) {
 486                 int err;
 487
 488                 err = kthread_stop(threads[n]);
 489                 if (err < 0 && !ret)
 490                         ret = err;
 491
 492                 put_task_struct(threads[n]);
 493         }
 494         pr_info("Completed %lu waits for %lu fence across %d cpus\n",
 495                 atomic_long_read(&t.num_waits),
 496                 atomic_long_read(&t.num_fences),
 497                 ncpus);
 498
 499 out_contexts:
 500         for (n = 0; n < t.ncontexts; n++) {
 501                 if (!t.contexts[n])
 502                         break;
 503                 mock_context_close(t.contexts[n]);
 504         }
 505         kfree(t.contexts);
 506 out_threads:
 507         kfree(threads);
 508         return ret;
 509 }
 510
 511 int i915_request_mock_selftests(void)
 512 {
 513         static const struct i915_subtest tests[] = {
 514                 SUBTEST(igt_add_request),
 515                 SUBTEST(igt_wait_request),
 516                 SUBTEST(igt_fence_wait),
 517                 SUBTEST(igt_request_rewind),
 518                 SUBTEST(mock_breadcrumbs_smoketest),
 519         };
 520         struct drm_i915_private *i915;
 521         intel_wakeref_t wakeref;
 522         int err = 0;
 523
 524         i915 = mock_gem_device();
 525         if (!i915)
 526                 return -ENOMEM;
 527
 528         with_intel_runtime_pm(&i915->runtime_pm, wakeref)
 529                 err = i915_subtests(tests, i915);
 530
 531         mock_destroy_device(i915);
 532
 533         return err;
 534 }
 535
 536 static int live_nop_request(void *arg)
 537 {
 538         struct drm_i915_private *i915 = arg;
 539         struct intel_engine_cs *engine;
 540         struct igt_live_test t;
 541         int err = -ENODEV;
 542
 543         /*
 544          * Submit various sized batches of empty requests, to each engine
 545          * (individually), and wait for the batch to complete. We can check
 546          * the overhead of submitting requests to the hardware.
 547          */
 548
 549         for_each_uabi_engine(engine, i915) {
 550                 unsigned long n, prime;
 551                 IGT_TIMEOUT(end_time);
 552                 ktime_t times[2] = {};
 553
 554                 err = igt_live_test_begin(&t, i915, __func__, engine->name);
 555                 if (err)
 556                         return err;
 557
 558                 intel_engine_pm_get(engine);
 559                 for_each_prime_number_from(prime, 1, 8192) {
 560                         struct i915_request *request = NULL;
 561
 562                         times[1] = ktime_get_raw();
 563
 564                         for (n = 0; n < prime; n++) {
 565                                 i915_request_put(request);
 566                                 request = i915_request_create(engine->kernel_context);
 567                                 if (IS_ERR(request))
 568                                         return PTR_ERR(request);
 569
 570                                 /*
 571                                  * This space is left intentionally blank.
 572                                  *
 573                                  * We do not actually want to perform any
 574                                  * action with this request, we just want
 575                                  * to measure the latency in allocation
 576                                  * and submission of our breadcrumbs -
 577                                  * ensuring that the bare request is sufficient
 578                                  * for the system to work (i.e. proper HEAD
 579                                  * tracking of the rings, interrupt handling,
 580                                  * etc). It also gives us the lowest bounds
 581                                  * for latency.
 582                                  */
 583
 584                                 i915_request_get(request);
 585                                 i915_request_add(request);
 586                         }
 587                         i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
 588                         i915_request_put(request);
 589
 590                         times[1] = ktime_sub(ktime_get_raw(), times[1]);
 591                         if (prime == 1)
 592                                 times[0] = times[1];
 593
 594                         if (__igt_timeout(end_time, NULL))
 595                                 break;
 596                 }
 597                 intel_engine_pm_put(engine);
 598
 599                 err = igt_live_test_end(&t);
 600                 if (err)
 601                         return err;
 602
 603                 pr_info("Request latencies on %s: 1 = %lluns, %lu = %lluns\n",
 604                         engine->name,
 605                         ktime_to_ns(times[0]),
 606                         prime, div64_u64(ktime_to_ns(times[1]), prime));
 607         }
 608
 609         return err;
 610 }
 611
 612 static int __cancel_inactive(struct intel_engine_cs *engine)
 613 {
 614         struct intel_context *ce;
 615         struct igt_spinner spin;
 616         struct i915_request *rq;
 617         int err = 0;
 618
 619         if (igt_spinner_init(&spin, engine->gt))
 620                 return -ENOMEM;
 621
 622         ce = intel_context_create(engine);
 623         if (IS_ERR(ce)) {
 624                 err = PTR_ERR(ce);
 625                 goto out_spin;
 626         }
 627
 628         rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
 629         if (IS_ERR(rq)) {
 630                 err = PTR_ERR(rq);
 631                 goto out_ce;
 632         }
 633
 634         pr_debug("%s: Cancelling inactive request\n", engine->name);
 635         i915_request_cancel(rq, -EINTR);
 636         i915_request_get(rq);
 637         i915_request_add(rq);
 638
 639         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 640                 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
 641
 642                 pr_err("%s: Failed to cancel inactive request\n", engine->name);
 643                 intel_engine_dump(engine, &p, "%s\n", engine->name);
 644                 err = -ETIME;
 645                 goto out_rq;
 646         }
 647
 648         if (rq->fence.error != -EINTR) {
 649                 pr_err("%s: fence not cancelled (%u)\n",
 650                        engine->name, rq->fence.error);
 651                 err = -EINVAL;
 652         }
 653
 654 out_rq:
 655         i915_request_put(rq);
 656 out_ce:
 657         intel_context_put(ce);
 658 out_spin:
 659         igt_spinner_fini(&spin);
 660         if (err)
 661                 pr_err("%s: %s error %d\n", __func__, engine->name, err);
 662         return err;
 663 }
 664
 665 static int __cancel_active(struct intel_engine_cs *engine)
 666 {
 667         struct intel_context *ce;
 668         struct igt_spinner spin;
 669         struct i915_request *rq;
 670         int err = 0;
 671
 672         if (igt_spinner_init(&spin, engine->gt))
 673                 return -ENOMEM;
 674
 675         ce = intel_context_create(engine);
 676         if (IS_ERR(ce)) {
 677                 err = PTR_ERR(ce);
 678                 goto out_spin;
 679         }
 680
 681         rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
 682         if (IS_ERR(rq)) {
 683                 err = PTR_ERR(rq);
 684                 goto out_ce;
 685         }
 686
 687         pr_debug("%s: Cancelling active request\n", engine->name);
 688         i915_request_get(rq);
 689         i915_request_add(rq);
 690         if (!igt_wait_for_spinner(&spin, rq)) {
 691                 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
 692
 693                 pr_err("Failed to start spinner on %s\n", engine->name);
 694                 intel_engine_dump(engine, &p, "%s\n", engine->name);
 695                 err = -ETIME;
 696                 goto out_rq;
 697         }
 698         i915_request_cancel(rq, -EINTR);
 699
 700         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 701                 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
 702
 703                 pr_err("%s: Failed to cancel active request\n", engine->name);
 704                 intel_engine_dump(engine, &p, "%s\n", engine->name);
 705                 err = -ETIME;
 706                 goto out_rq;
 707         }
 708
 709         if (rq->fence.error != -EINTR) {
 710                 pr_err("%s: fence not cancelled (%u)\n",
 711                        engine->name, rq->fence.error);
 712                 err = -EINVAL;
 713         }
 714
 715 out_rq:
 716         i915_request_put(rq);
 717 out_ce:
 718         intel_context_put(ce);
 719 out_spin:
 720         igt_spinner_fini(&spin);
 721         if (err)
 722                 pr_err("%s: %s error %d\n", __func__, engine->name, err);
 723         return err;
 724 }
 725
 726 static int __cancel_completed(struct intel_engine_cs *engine)
 727 {
 728         struct intel_context *ce;
 729         struct igt_spinner spin;
 730         struct i915_request *rq;
 731         int err = 0;
 732
 733         if (igt_spinner_init(&spin, engine->gt))
 734                 return -ENOMEM;
 735
 736         ce = intel_context_create(engine);
 737         if (IS_ERR(ce)) {
 738                 err = PTR_ERR(ce);
 739                 goto out_spin;
 740         }
 741
 742         rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
 743         if (IS_ERR(rq)) {
 744                 err = PTR_ERR(rq);
 745                 goto out_ce;
 746         }
 747         igt_spinner_end(&spin);
 748         i915_request_get(rq);
 749         i915_request_add(rq);
 750
 751         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
 752                 err = -ETIME;
 753                 goto out_rq;
 754         }
 755
 756         pr_debug("%s: Cancelling completed request\n", engine->name);
 757         i915_request_cancel(rq, -EINTR);
 758         if (rq->fence.error) {
 759                 pr_err("%s: fence not cancelled (%u)\n",
 760                        engine->name, rq->fence.error);
 761                 err = -EINVAL;
 762         }
 763
 764 out_rq:
 765         i915_request_put(rq);
 766 out_ce:
 767         intel_context_put(ce);
 768 out_spin:
 769         igt_spinner_fini(&spin);
 770         if (err)
 771                 pr_err("%s: %s error %d\n", __func__, engine->name, err);
 772         return err;
 773 }
 774
 775 static int live_cancel_request(void *arg)
 776 {
 777         struct drm_i915_private *i915 = arg;
 778         struct intel_engine_cs *engine;
 779
 780         /*
 781          * Check cancellation of requests. We expect to be able to immediately
 782          * cancel active requests, even if they are currently on the GPU.
 783          */
 784
 785         for_each_uabi_engine(engine, i915) {
 786                 struct igt_live_test t;
 787                 int err, err2;
 788
 789                 if (!intel_engine_has_preemption(engine))
 790                         continue;
 791
 792                 err = igt_live_test_begin(&t, i915, __func__, engine->name);
 793                 if (err)
 794                         return err;
 795
 796                 err = __cancel_inactive(engine);
 797                 if (err == 0)
 798                         err = __cancel_active(engine);
 799                 if (err == 0)
 800                         err = __cancel_completed(engine);
 801
 802                 err2 = igt_live_test_end(&t);
 803                 if (err)
 804                         return err;
 805                 if (err2)
 806                         return err2;
 807         }
 808
 809         return 0;
 810 }
 811
 812 static struct i915_vma *empty_batch(struct drm_i915_private *i915)
 813 {
 814         struct drm_i915_gem_object *obj;
 815         struct i915_vma *vma;
 816         u32 *cmd;
 817         int err;
 818
 819         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 820         if (IS_ERR(obj))
 821                 return ERR_CAST(obj);
 822
 823         cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
 824         if (IS_ERR(cmd)) {
 825                 err = PTR_ERR(cmd);
 826                 goto err;
 827         }
 828
 829         *cmd = MI_BATCH_BUFFER_END;
 830
 831         __i915_gem_object_flush_map(obj, 0, 64);
 832         i915_gem_object_unpin_map(obj);
 833
 834         intel_gt_chipset_flush(&i915->gt);
 835
 836         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
 837         if (IS_ERR(vma)) {
 838                 err = PTR_ERR(vma);
 839                 goto err;
 840         }
 841
 842         err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_GLOBAL);
 843         if (err)
 844                 goto err;
 845
 846         /* Force the wait wait now to avoid including it in the benchmark */
 847         err = i915_vma_sync(vma);
 848         if (err)
 849                 goto err_pin;
 850
 851         return vma;
 852
 853 err_pin:
 854         i915_vma_unpin(vma);
 855 err:
 856         i915_gem_object_put(obj);
 857         return ERR_PTR(err);
 858 }
 859
 860 static struct i915_request *
 861 empty_request(struct intel_engine_cs *engine,
 862               struct i915_vma *batch)
 863 {
 864         struct i915_request *request;
 865         int err;
 866
 867         request = i915_request_create(engine->kernel_context);
 868         if (IS_ERR(request))
 869                 return request;
 870
 871         err = engine->emit_bb_start(request,
 872                                     batch->node.start,
 873                                     batch->node.size,
 874                                     I915_DISPATCH_SECURE);
 875         if (err)
 876                 goto out_request;
 877
 878         i915_request_get(request);
 879 out_request:
 880         i915_request_add(request);
 881         return err ? ERR_PTR(err) : request;
 882 }
 883
 884 static int live_empty_request(void *arg)
 885 {
 886         struct drm_i915_private *i915 = arg;
 887         struct intel_engine_cs *engine;
 888         struct igt_live_test t;
 889         struct i915_vma *batch;
 890         int err = 0;
 891
 892         /*
 893          * Submit various sized batches of empty requests, to each engine
 894          * (individually), and wait for the batch to complete. We can check
 895          * the overhead of submitting requests to the hardware.
 896          */
 897
 898         batch = empty_batch(i915);
 899         if (IS_ERR(batch))
 900                 return PTR_ERR(batch);
 901
 902         for_each_uabi_engine(engine, i915) {
 903                 IGT_TIMEOUT(end_time);
 904                 struct i915_request *request;
 905                 unsigned long n, prime;
 906                 ktime_t times[2] = {};
 907
 908                 err = igt_live_test_begin(&t, i915, __func__, engine->name);
 909                 if (err)
 910                         goto out_batch;
 911
 912                 intel_engine_pm_get(engine);
 913
 914                 /* Warmup / preload */
 915                 request = empty_request(engine, batch);
 916                 if (IS_ERR(request)) {
 917                         err = PTR_ERR(request);
 918                         intel_engine_pm_put(engine);
 919                         goto out_batch;
 920                 }
 921                 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
 922
 923                 for_each_prime_number_from(prime, 1, 8192) {
 924                         times[1] = ktime_get_raw();
 925
 926                         for (n = 0; n < prime; n++) {
 927                                 i915_request_put(request);
 928                                 request = empty_request(engine, batch);
 929                                 if (IS_ERR(request)) {
 930                                         err = PTR_ERR(request);
 931                                         intel_engine_pm_put(engine);
 932                                         goto out_batch;
 933                                 }
 934                         }
 935                         i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
 936
 937                         times[1] = ktime_sub(ktime_get_raw(), times[1]);
 938                         if (prime == 1)
 939                                 times[0] = times[1];
 940
 941                         if (__igt_timeout(end_time, NULL))
 942                                 break;
 943                 }
 944                 i915_request_put(request);
 945                 intel_engine_pm_put(engine);
 946
 947                 err = igt_live_test_end(&t);
 948                 if (err)
 949                         goto out_batch;
 950
 951                 pr_info("Batch latencies on %s: 1 = %lluns, %lu = %lluns\n",
 952                         engine->name,
 953                         ktime_to_ns(times[0]),
 954                         prime, div64_u64(ktime_to_ns(times[1]), prime));
 955         }
 956
 957 out_batch:
 958         i915_vma_unpin(batch);
 959         i915_vma_put(batch);
 960         return err;
 961 }
 962
 963 static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
 964 {
 965         struct drm_i915_gem_object *obj;
 966         const int gen = INTEL_GEN(i915);
 967         struct i915_vma *vma;
 968         u32 *cmd;
 969         int err;
 970
 971         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 972         if (IS_ERR(obj))
 973                 return ERR_CAST(obj);
 974
 975         vma = i915_vma_instance(obj, i915->gt.vm, NULL);
 976         if (IS_ERR(vma)) {
 977                 err = PTR_ERR(vma);
 978                 goto err;
 979         }
 980
 981         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 982         if (err)
 983                 goto err;
 984
 985         cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 986         if (IS_ERR(cmd)) {
 987                 err = PTR_ERR(cmd);
 988                 goto err;
 989         }
 990
 991         if (gen >= 8) {
 992                 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
 993                 *cmd++ = lower_32_bits(vma->node.start);
 994                 *cmd++ = upper_32_bits(vma->node.start);
 995         } else if (gen >= 6) {
 996                 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8;
 997                 *cmd++ = lower_32_bits(vma->node.start);
 998         } else {
 999                 *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1000                 *cmd++ = lower_32_bits(vma->node.start);
1001         }
1002         *cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
1003
1004         __i915_gem_object_flush_map(obj, 0, 64);
1005         i915_gem_object_unpin_map(obj);
1006
1007         intel_gt_chipset_flush(&i915->gt);
1008
1009         return vma;
1010
1011 err:
1012         i915_gem_object_put(obj);
1013         return ERR_PTR(err);
1014 }
1015
1016 static int recursive_batch_resolve(struct i915_vma *batch)
1017 {
1018         u32 *cmd;
1019
1020         cmd = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1021         if (IS_ERR(cmd))
1022                 return PTR_ERR(cmd);
1023
1024         *cmd = MI_BATCH_BUFFER_END;
1025
1026         __i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd));
1027         i915_gem_object_unpin_map(batch->obj);
1028
1029         intel_gt_chipset_flush(batch->vm->gt);
1030
1031         return 0;
1032 }
1033
1034 static int live_all_engines(void *arg)
1035 {
1036         struct drm_i915_private *i915 = arg;
1037         const unsigned int nengines = num_uabi_engines(i915);
1038         struct intel_engine_cs *engine;
1039         struct i915_request **request;
1040         struct igt_live_test t;
1041         struct i915_vma *batch;
1042         unsigned int idx;
1043         int err;
1044
1045         /*
1046          * Check we can submit requests to all engines simultaneously. We
1047          * send a recursive batch to each engine - checking that we don't
1048          * block doing so, and that they don't complete too soon.
1049          */
1050
1051         request = kcalloc(nengines, sizeof(*request), GFP_KERNEL);
1052         if (!request)
1053                 return -ENOMEM;
1054
1055         err = igt_live_test_begin(&t, i915, __func__, "");
1056         if (err)
1057                 goto out_free;
1058
1059         batch = recursive_batch(i915);
1060         if (IS_ERR(batch)) {
1061                 err = PTR_ERR(batch);
1062                 pr_err("%s: Unable to create batch, err=%d\n", __func__, err);
1063                 goto out_free;
1064         }
1065
1066         i915_vma_lock(batch);
1067
1068         idx = 0;
1069         for_each_uabi_engine(engine, i915) {
1070                 request[idx] = intel_engine_create_kernel_request(engine);
1071                 if (IS_ERR(request[idx])) {
1072                         err = PTR_ERR(request[idx]);
1073                         pr_err("%s: Request allocation failed with err=%d\n",
1074                                __func__, err);
1075                         goto out_request;
1076                 }
1077
1078                 err = i915_request_await_object(request[idx], batch->obj, 0);
1079                 if (err == 0)
1080                         err = i915_vma_move_to_active(batch, request[idx], 0);
1081                 GEM_BUG_ON(err);
1082
1083                 err = engine->emit_bb_start(request[idx],
1084                                             batch->node.start,
1085                                             batch->node.size,
1086                                             0);
1087                 GEM_BUG_ON(err);
1088                 request[idx]->batch = batch;
1089
1090                 i915_request_get(request[idx]);
1091                 i915_request_add(request[idx]);
1092                 idx++;
1093         }
1094
1095         i915_vma_unlock(batch);
1096
1097         idx = 0;
1098         for_each_uabi_engine(engine, i915) {
1099                 if (i915_request_completed(request[idx])) {
1100                         pr_err("%s(%s): request completed too early!\n",
1101                                __func__, engine->name);
1102                         err = -EINVAL;
1103                         goto out_request;
1104                 }
1105                 idx++;
1106         }
1107
1108         err = recursive_batch_resolve(batch);
1109         if (err) {
1110                 pr_err("%s: failed to resolve batch, err=%d\n", __func__, err);
1111                 goto out_request;
1112         }
1113
1114         idx = 0;
1115         for_each_uabi_engine(engine, i915) {
1116                 long timeout;
1117
1118                 timeout = i915_request_wait(request[idx], 0,
1119                                             MAX_SCHEDULE_TIMEOUT);
1120                 if (timeout < 0) {
1121                         err = timeout;
1122                         pr_err("%s: error waiting for request on %s, err=%d\n",
1123                                __func__, engine->name, err);
1124                         goto out_request;
1125                 }
1126
1127                 GEM_BUG_ON(!i915_request_completed(request[idx]));
1128                 i915_request_put(request[idx]);
1129                 request[idx] = NULL;
1130                 idx++;
1131         }
1132
1133         err = igt_live_test_end(&t);
1134
1135 out_request:
1136         idx = 0;
1137         for_each_uabi_engine(engine, i915) {
1138                 if (request[idx])
1139                         i915_request_put(request[idx]);
1140                 idx++;
1141         }
1142         i915_vma_unpin(batch);
1143         i915_vma_put(batch);
1144 out_free:
1145         kfree(request);
1146         return err;
1147 }
1148
1149 static int live_sequential_engines(void *arg)
1150 {
1151         struct drm_i915_private *i915 = arg;
1152         const unsigned int nengines = num_uabi_engines(i915);
1153         struct i915_request **request;
1154         struct i915_request *prev = NULL;
1155         struct intel_engine_cs *engine;
1156         struct igt_live_test t;
1157         unsigned int idx;
1158         int err;
1159
1160         /*
1161          * Check we can submit requests to all engines sequentially, such
1162          * that each successive request waits for the earlier ones. This
1163          * tests that we don't execute requests out of order, even though
1164          * they are running on independent engines.
1165          */
1166
1167         request = kcalloc(nengines, sizeof(*request), GFP_KERNEL);
1168         if (!request)
1169                 return -ENOMEM;
1170
1171         err = igt_live_test_begin(&t, i915, __func__, "");
1172         if (err)
1173                 goto out_free;
1174
1175         idx = 0;
1176         for_each_uabi_engine(engine, i915) {
1177                 struct i915_vma *batch;
1178
1179                 batch = recursive_batch(i915);
1180                 if (IS_ERR(batch)) {
1181                         err = PTR_ERR(batch);
1182                         pr_err("%s: Unable to create batch for %s, err=%d\n",
1183                                __func__, engine->name, err);
1184                         goto out_free;
1185                 }
1186
1187                 i915_vma_lock(batch);
1188                 request[idx] = intel_engine_create_kernel_request(engine);
1189                 if (IS_ERR(request[idx])) {
1190                         err = PTR_ERR(request[idx]);
1191                         pr_err("%s: Request allocation failed for %s with err=%d\n",
1192                                __func__, engine->name, err);
1193                         goto out_unlock;
1194                 }
1195
1196                 if (prev) {
1197                         err = i915_request_await_dma_fence(request[idx],
1198                                                            &prev->fence);
1199                         if (err) {
1200                                 i915_request_add(request[idx]);
1201                                 pr_err("%s: Request await failed for %s with err=%d\n",
1202                                        __func__, engine->name, err);
1203                                 goto out_unlock;
1204                         }
1205                 }
1206
1207                 err = i915_request_await_object(request[idx],
1208                                                 batch->obj, false);
1209                 if (err == 0)
1210                         err = i915_vma_move_to_active(batch, request[idx], 0);
1211                 GEM_BUG_ON(err);
1212
1213                 err = engine->emit_bb_start(request[idx],
1214                                             batch->node.start,
1215                                             batch->node.size,
1216                                             0);
1217                 GEM_BUG_ON(err);
1218                 request[idx]->batch = batch;
1219
1220                 i915_request_get(request[idx]);
1221                 i915_request_add(request[idx]);
1222
1223                 prev = request[idx];
1224                 idx++;
1225
1226 out_unlock:
1227                 i915_vma_unlock(batch);
1228                 if (err)
1229                         goto out_request;
1230         }
1231
1232         idx = 0;
1233         for_each_uabi_engine(engine, i915) {
1234                 long timeout;
1235
1236                 if (i915_request_completed(request[idx])) {
1237                         pr_err("%s(%s): request completed too early!\n",
1238                                __func__, engine->name);
1239                         err = -EINVAL;
1240                         goto out_request;
1241                 }
1242
1243                 err = recursive_batch_resolve(request[idx]->batch);
1244                 if (err) {
1245                         pr_err("%s: failed to resolve batch, err=%d\n",
1246                                __func__, err);
1247                         goto out_request;
1248                 }
1249
1250                 timeout = i915_request_wait(request[idx], 0,
1251                                             MAX_SCHEDULE_TIMEOUT);
1252                 if (timeout < 0) {
1253                         err = timeout;
1254                         pr_err("%s: error waiting for request on %s, err=%d\n",
1255                                __func__, engine->name, err);
1256                         goto out_request;
1257                 }
1258
1259                 GEM_BUG_ON(!i915_request_completed(request[idx]));
1260                 idx++;
1261         }
1262
1263         err = igt_live_test_end(&t);
1264
1265 out_request:
1266         idx = 0;
1267         for_each_uabi_engine(engine, i915) {
1268                 u32 *cmd;
1269
1270                 if (!request[idx])
1271                         break;
1272
1273                 cmd = i915_gem_object_pin_map_unlocked(request[idx]->batch->obj,
1274                                                        I915_MAP_WC);
1275                 if (!IS_ERR(cmd)) {
1276                         *cmd = MI_BATCH_BUFFER_END;
1277
1278                         __i915_gem_object_flush_map(request[idx]->batch->obj,
1279                                                     0, sizeof(*cmd));
1280                         i915_gem_object_unpin_map(request[idx]->batch->obj);
1281
1282                         intel_gt_chipset_flush(engine->gt);
1283                 }
1284
1285                 i915_vma_put(request[idx]->batch);
1286                 i915_request_put(request[idx]);
1287                 idx++;
1288         }
1289 out_free:
1290         kfree(request);
1291         return err;
1292 }
1293
1294 static int __live_parallel_engine1(void *arg)
1295 {
1296         struct intel_engine_cs *engine = arg;
1297         IGT_TIMEOUT(end_time);
1298         unsigned long count;
1299         int err = 0;
1300
1301         count = 0;
1302         intel_engine_pm_get(engine);
1303         do {
1304                 struct i915_request *rq;
1305
1306                 rq = i915_request_create(engine->kernel_context);
1307                 if (IS_ERR(rq)) {
1308                         err = PTR_ERR(rq);
1309                         break;
1310                 }
1311
1312                 i915_request_get(rq);
1313                 i915_request_add(rq);
1314
1315                 err = 0;
1316                 if (i915_request_wait(rq, 0, HZ / 5) < 0)
1317                         err = -ETIME;
1318                 i915_request_put(rq);
1319                 if (err)
1320                         break;
1321
1322                 count++;
1323         } while (!__igt_timeout(end_time, NULL));
1324         intel_engine_pm_put(engine);
1325
1326         pr_info("%s: %lu request + sync\n", engine->name, count);
1327         return err;
1328 }
1329
1330 static int __live_parallel_engineN(void *arg)
1331 {
1332         struct intel_engine_cs *engine = arg;
1333         IGT_TIMEOUT(end_time);
1334         unsigned long count;
1335         int err = 0;
1336
1337         count = 0;
1338         intel_engine_pm_get(engine);
1339         do {
1340                 struct i915_request *rq;
1341
1342                 rq = i915_request_create(engine->kernel_context);
1343                 if (IS_ERR(rq)) {
1344                         err = PTR_ERR(rq);
1345                         break;
1346                 }
1347
1348                 i915_request_add(rq);
1349                 count++;
1350         } while (!__igt_timeout(end_time, NULL));
1351         intel_engine_pm_put(engine);
1352
1353         pr_info("%s: %lu requests\n", engine->name, count);
1354         return err;
1355 }
1356
1357 static bool wake_all(struct drm_i915_private *i915)
1358 {
1359         if (atomic_dec_and_test(&i915->selftest.counter)) {
1360                 wake_up_var(&i915->selftest.counter);
1361                 return true;
1362         }
1363
1364         return false;
1365 }
1366
1367 static int wait_for_all(struct drm_i915_private *i915)
1368 {
1369         if (wake_all(i915))
1370                 return 0;
1371
1372         if (wait_var_event_timeout(&i915->selftest.counter,
1373                                    !atomic_read(&i915->selftest.counter),
1374                                    i915_selftest.timeout_jiffies))
1375                 return 0;
1376
1377         return -ETIME;
1378 }
1379
1380 static int __live_parallel_spin(void *arg)
1381 {
1382         struct intel_engine_cs *engine = arg;
1383         struct igt_spinner spin;
1384         struct i915_request *rq;
1385         int err = 0;
1386
1387         /*
1388          * Create a spinner running for eternity on each engine. If a second
1389          * spinner is incorrectly placed on the same engine, it will not be
1390          * able to start in time.
1391          */
1392
1393         if (igt_spinner_init(&spin, engine->gt)) {
1394                 wake_all(engine->i915);
1395                 return -ENOMEM;
1396         }
1397
1398         intel_engine_pm_get(engine);
1399         rq = igt_spinner_create_request(&spin,
1400                                         engine->kernel_context,
1401                                         MI_NOOP); /* no preemption */
1402         intel_engine_pm_put(engine);
1403         if (IS_ERR(rq)) {
1404                 err = PTR_ERR(rq);
1405                 if (err == -ENODEV)
1406                         err = 0;
1407                 wake_all(engine->i915);
1408                 goto out_spin;
1409         }
1410
1411         i915_request_get(rq);
1412         i915_request_add(rq);
1413         if (igt_wait_for_spinner(&spin, rq)) {
1414                 /* Occupy this engine for the whole test */
1415                 err = wait_for_all(engine->i915);
1416         } else {
1417                 pr_err("Failed to start spinner on %s\n", engine->name);
1418                 err = -EINVAL;
1419         }
1420         igt_spinner_end(&spin);
1421
1422         if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0)
1423                 err = -EIO;
1424         i915_request_put(rq);
1425
1426 out_spin:
1427         igt_spinner_fini(&spin);
1428         return err;
1429 }
1430
1431 static int live_parallel_engines(void *arg)
1432 {
1433         struct drm_i915_private *i915 = arg;
1434         static int (* const func[])(void *arg) = {
1435                 __live_parallel_engine1,
1436                 __live_parallel_engineN,
1437                 __live_parallel_spin,
1438                 NULL,
1439         };
1440         const unsigned int nengines = num_uabi_engines(i915);
1441         struct intel_engine_cs *engine;
1442         int (* const *fn)(void *arg);
1443         struct task_struct **tsk;
1444         int err = 0;
1445
1446         /*
1447          * Check we can submit requests to all engines concurrently. This
1448          * tests that we load up the system maximally.
1449          */
1450
1451         tsk = kcalloc(nengines, sizeof(*tsk), GFP_KERNEL);
1452         if (!tsk)
1453                 return -ENOMEM;
1454
1455         for (fn = func; !err && *fn; fn++) {
1456                 char name[KSYM_NAME_LEN];
1457                 struct igt_live_test t;
1458                 unsigned int idx;
1459
1460                 snprintf(name, sizeof(name), "%ps", *fn);
1461                 err = igt_live_test_begin(&t, i915, __func__, name);
1462                 if (err)
1463                         break;
1464
1465                 atomic_set(&i915->selftest.counter, nengines);
1466
1467                 idx = 0;
1468                 for_each_uabi_engine(engine, i915) {
1469                         tsk[idx] = kthread_run(*fn, engine,
1470                                                "igt/parallel:%s",
1471                                                engine->name);
1472                         if (IS_ERR(tsk[idx])) {
1473                                 err = PTR_ERR(tsk[idx]);
1474                                 break;
1475                         }
1476                         get_task_struct(tsk[idx++]);
1477                 }
1478
1479                 yield(); /* start all threads before we kthread_stop() */
1480
1481                 idx = 0;
1482                 for_each_uabi_engine(engine, i915) {
1483                         int status;
1484
1485                         if (IS_ERR(tsk[idx]))
1486                                 break;
1487
1488                         status = kthread_stop(tsk[idx]);
1489                         if (status && !err)
1490                                 err = status;
1491
1492                         put_task_struct(tsk[idx++]);
1493                 }
1494
1495                 if (igt_live_test_end(&t))
1496                         err = -EIO;
1497         }
1498
1499         kfree(tsk);
1500         return err;
1501 }
1502
1503 static int
1504 max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
1505 {
1506         struct i915_request *rq;
1507         int ret;
1508
1509         /*
1510          * Before execlists, all contexts share the same ringbuffer. With
1511          * execlists, each context/engine has a separate ringbuffer and
1512          * for the purposes of this test, inexhaustible.
1513          *
1514          * For the global ringbuffer though, we have to be very careful
1515          * that we do not wrap while preventing the execution of requests
1516          * with a unsignaled fence.
1517          */
1518         if (HAS_EXECLISTS(ctx->i915))
1519                 return INT_MAX;
1520
1521         rq = igt_request_alloc(ctx, engine);
1522         if (IS_ERR(rq)) {
1523                 ret = PTR_ERR(rq);
1524         } else {
1525                 int sz;
1526
1527                 ret = rq->ring->size - rq->reserved_space;
1528                 i915_request_add(rq);
1529
1530                 sz = rq->ring->emit - rq->head;
1531                 if (sz < 0)
1532                         sz += rq->ring->size;
1533                 ret /= sz;
1534                 ret /= 2; /* leave half spare, in case of emergency! */
1535         }
1536
1537         return ret;
1538 }
1539
1540 static int live_breadcrumbs_smoketest(void *arg)
1541 {
1542         struct drm_i915_private *i915 = arg;
1543         const unsigned int nengines = num_uabi_engines(i915);
1544         const unsigned int ncpus = num_online_cpus();
1545         unsigned long num_waits, num_fences;
1546         struct intel_engine_cs *engine;
1547         struct task_struct **threads;
1548         struct igt_live_test live;
1549         intel_wakeref_t wakeref;
1550         struct smoketest *smoke;
1551         unsigned int n, idx;
1552         struct file *file;
1553         int ret = 0;
1554
1555         /*
1556          * Smoketest our breadcrumb/signal handling for requests across multiple
1557          * threads. A very simple test to only catch the most egregious of bugs.
1558          * See __igt_breadcrumbs_smoketest();
1559          *
1560          * On real hardware this time.
1561          */
1562
1563         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1564
1565         file = mock_file(i915);
1566         if (IS_ERR(file)) {
1567                 ret = PTR_ERR(file);
1568                 goto out_rpm;
1569         }
1570
1571         smoke = kcalloc(nengines, sizeof(*smoke), GFP_KERNEL);
1572         if (!smoke) {
1573                 ret = -ENOMEM;
1574                 goto out_file;
1575         }
1576
1577         threads = kcalloc(ncpus * nengines, sizeof(*threads), GFP_KERNEL);
1578         if (!threads) {
1579                 ret = -ENOMEM;
1580                 goto out_smoke;
1581         }
1582
1583         smoke[0].request_alloc = __live_request_alloc;
1584         smoke[0].ncontexts = 64;
1585         smoke[0].contexts = kcalloc(smoke[0].ncontexts,
1586                                     sizeof(*smoke[0].contexts),
1587                                     GFP_KERNEL);
1588         if (!smoke[0].contexts) {
1589                 ret = -ENOMEM;
1590                 goto out_threads;
1591         }
1592
1593         for (n = 0; n < smoke[0].ncontexts; n++) {
1594                 smoke[0].contexts[n] = live_context(i915, file);
1595                 if (IS_ERR(smoke[0].contexts[n])) {
1596                         ret = PTR_ERR(smoke[0].contexts[n]);
1597                         goto out_contexts;
1598                 }
1599         }
1600
1601         ret = igt_live_test_begin(&live, i915, __func__, "");
1602         if (ret)
1603                 goto out_contexts;
1604
1605         idx = 0;
1606         for_each_uabi_engine(engine, i915) {
1607                 smoke[idx] = smoke[0];
1608                 smoke[idx].engine = engine;
1609                 smoke[idx].max_batch =
1610                         max_batches(smoke[0].contexts[0], engine);
1611                 if (smoke[idx].max_batch < 0) {
1612                         ret = smoke[idx].max_batch;
1613                         goto out_flush;
1614                 }
1615                 /* One ring interleaved between requests from all cpus */
1616                 smoke[idx].max_batch /= num_online_cpus() + 1;
1617                 pr_debug("Limiting batches to %d requests on %s\n",
1618                          smoke[idx].max_batch, engine->name);
1619
1620                 for (n = 0; n < ncpus; n++) {
1621                         struct task_struct *tsk;
1622
1623                         tsk = kthread_run(__igt_breadcrumbs_smoketest,
1624                                           &smoke[idx], "igt/%d.%d", idx, n);
1625                         if (IS_ERR(tsk)) {
1626                                 ret = PTR_ERR(tsk);
1627                                 goto out_flush;
1628                         }
1629
1630                         get_task_struct(tsk);
1631                         threads[idx * ncpus + n] = tsk;
1632                 }
1633
1634                 idx++;
1635         }
1636
1637         yield(); /* start all threads before we begin */
1638         msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
1639
1640 out_flush:
1641         idx = 0;
1642         num_waits = 0;
1643         num_fences = 0;
1644         for_each_uabi_engine(engine, i915) {
1645                 for (n = 0; n < ncpus; n++) {
1646                         struct task_struct *tsk = threads[idx * ncpus + n];
1647                         int err;
1648
1649                         if (!tsk)
1650                                 continue;
1651
1652                         err = kthread_stop(tsk);
1653                         if (err < 0 && !ret)
1654                                 ret = err;
1655
1656                         put_task_struct(tsk);
1657                 }
1658
1659                 num_waits += atomic_long_read(&smoke[idx].num_waits);
1660                 num_fences += atomic_long_read(&smoke[idx].num_fences);
1661                 idx++;
1662         }
1663         pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n",
1664                 num_waits, num_fences, idx, ncpus);
1665
1666         ret = igt_live_test_end(&live) ?: ret;
1667 out_contexts:
1668         kfree(smoke[0].contexts);
1669 out_threads:
1670         kfree(threads);
1671 out_smoke:
1672         kfree(smoke);
1673 out_file:
1674         fput(file);
1675 out_rpm:
1676         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1677
1678         return ret;
1679 }
1680
1681 int i915_request_live_selftests(struct drm_i915_private *i915)
1682 {
1683         static const struct i915_subtest tests[] = {
1684                 SUBTEST(live_nop_request),
1685                 SUBTEST(live_all_engines),
1686                 SUBTEST(live_sequential_engines),
1687                 SUBTEST(live_parallel_engines),
1688                 SUBTEST(live_empty_request),
1689                 SUBTEST(live_cancel_request),
1690                 SUBTEST(live_breadcrumbs_smoketest),
1691         };
1692
1693         if (intel_gt_is_wedged(&i915->gt))
1694                 return 0;
1695
1696         return i915_subtests(tests, i915);
1697 }
1698
1699 static int switch_to_kernel_sync(struct intel_context *ce, int err)
1700 {
1701         struct i915_request *rq;
1702         struct dma_fence *fence;
1703
1704         rq = intel_engine_create_kernel_request(ce->engine);
1705         if (IS_ERR(rq))
1706                 return PTR_ERR(rq);
1707
1708         fence = i915_active_fence_get(&ce->timeline->last_request);
1709         if (fence) {
1710                 i915_request_await_dma_fence(rq, fence);
1711                 dma_fence_put(fence);
1712         }
1713
1714         rq = i915_request_get(rq);
1715         i915_request_add(rq);
1716         if (i915_request_wait(rq, 0, HZ / 2) < 0 && !err)
1717                 err = -ETIME;
1718         i915_request_put(rq);
1719
1720         while (!err && !intel_engine_is_idle(ce->engine))
1721                 intel_engine_flush_submission(ce->engine);
1722
1723         return err;
1724 }
1725
1726 struct perf_stats {
1727         struct intel_engine_cs *engine;
1728         unsigned long count;
1729         ktime_t time;
1730         ktime_t busy;
1731         u64 runtime;
1732 };
1733
1734 struct perf_series {
1735         struct drm_i915_private *i915;
1736         unsigned int nengines;
1737         struct intel_context *ce[];
1738 };
1739
1740 static int cmp_u32(const void *A, const void *B)
1741 {
1742         const u32 *a = A, *b = B;
1743
1744         return *a - *b;
1745 }
1746
1747 static u32 trifilter(u32 *a)
1748 {
1749         u64 sum;
1750
1751 #define TF_COUNT 5
1752         sort(a, TF_COUNT, sizeof(*a), cmp_u32, NULL);
1753
1754         sum = mul_u32_u32(a[2], 2);
1755         sum += a[1];
1756         sum += a[3];
1757
1758         GEM_BUG_ON(sum > U32_MAX);
1759         return sum;
1760 #define TF_BIAS 2
1761 }
1762
1763 static u64 cycles_to_ns(struct intel_engine_cs *engine, u32 cycles)
1764 {
1765         u64 ns = intel_gt_clock_interval_to_ns(engine->gt, cycles);
1766
1767         return DIV_ROUND_CLOSEST(ns, 1 << TF_BIAS);
1768 }
1769
1770 static u32 *emit_timestamp_store(u32 *cs, struct intel_context *ce, u32 offset)
1771 {
1772         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1773         *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP((ce->engine->mmio_base)));
1774         *cs++ = offset;
1775         *cs++ = 0;
1776
1777         return cs;
1778 }
1779
1780 static u32 *emit_store_dw(u32 *cs, u32 offset, u32 value)
1781 {
1782         *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1783         *cs++ = offset;
1784         *cs++ = 0;
1785         *cs++ = value;
1786
1787         return cs;
1788 }
1789
1790 static u32 *emit_semaphore_poll(u32 *cs, u32 mode, u32 value, u32 offset)
1791 {
1792         *cs++ = MI_SEMAPHORE_WAIT |
1793                 MI_SEMAPHORE_GLOBAL_GTT |
1794                 MI_SEMAPHORE_POLL |
1795                 mode;
1796         *cs++ = value;
1797         *cs++ = offset;
1798         *cs++ = 0;
1799
1800         return cs;
1801 }
1802
1803 static u32 *emit_semaphore_poll_until(u32 *cs, u32 offset, u32 value)
1804 {
1805         return emit_semaphore_poll(cs, MI_SEMAPHORE_SAD_EQ_SDD, value, offset);
1806 }
1807
1808 static void semaphore_set(u32 *sema, u32 value)
1809 {
1810         WRITE_ONCE(*sema, value);
1811         wmb(); /* flush the update to the cache, and beyond */
1812 }
1813
1814 static u32 *hwsp_scratch(const struct intel_context *ce)
1815 {
1816         return memset32(ce->engine->status_page.addr + 1000, 0, 21);
1817 }
1818
1819 static u32 hwsp_offset(const struct intel_context *ce, u32 *dw)
1820 {
1821         return (i915_ggtt_offset(ce->engine->status_page.vma) +
1822                 offset_in_page(dw));
1823 }
1824
1825 static int measure_semaphore_response(struct intel_context *ce)
1826 {
1827         u32 *sema = hwsp_scratch(ce);
1828         const u32 offset = hwsp_offset(ce, sema);
1829         u32 elapsed[TF_COUNT], cycles;
1830         struct i915_request *rq;
1831         u32 *cs;
1832         int err;
1833         int i;
1834
1835         /*
1836          * Measure how many cycles it takes for the HW to detect the change
1837          * in a semaphore value.
1838          *
1839          *    A: read CS_TIMESTAMP from CPU
1840          *    poke semaphore
1841          *    B: read CS_TIMESTAMP on GPU
1842          *
1843          * Semaphore latency: B - A
1844          */
1845
1846         semaphore_set(sema, -1);
1847
1848         rq = i915_request_create(ce);
1849         if (IS_ERR(rq))
1850                 return PTR_ERR(rq);
1851
1852         cs = intel_ring_begin(rq, 4 + 12 * ARRAY_SIZE(elapsed));
1853         if (IS_ERR(cs)) {
1854                 i915_request_add(rq);
1855                 err = PTR_ERR(cs);
1856                 goto err;
1857         }
1858
1859         cs = emit_store_dw(cs, offset, 0);
1860         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
1861                 cs = emit_semaphore_poll_until(cs, offset, i);
1862                 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
1863                 cs = emit_store_dw(cs, offset, 0);
1864         }
1865
1866         intel_ring_advance(rq, cs);
1867         i915_request_add(rq);
1868
1869         if (wait_for(READ_ONCE(*sema) == 0, 50)) {
1870                 err = -EIO;
1871                 goto err;
1872         }
1873
1874         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
1875                 preempt_disable();
1876                 cycles = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
1877                 semaphore_set(sema, i);
1878                 preempt_enable();
1879
1880                 if (wait_for(READ_ONCE(*sema) == 0, 50)) {
1881                         err = -EIO;
1882                         goto err;
1883                 }
1884
1885                 elapsed[i - 1] = sema[i] - cycles;
1886         }
1887
1888         cycles = trifilter(elapsed);
1889         pr_info("%s: semaphore response %d cycles, %lluns\n",
1890                 ce->engine->name, cycles >> TF_BIAS,
1891                 cycles_to_ns(ce->engine, cycles));
1892
1893         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
1894
1895 err:
1896         intel_gt_set_wedged(ce->engine->gt);
1897         return err;
1898 }
1899
1900 static int measure_idle_dispatch(struct intel_context *ce)
1901 {
1902         u32 *sema = hwsp_scratch(ce);
1903         const u32 offset = hwsp_offset(ce, sema);
1904         u32 elapsed[TF_COUNT], cycles;
1905         u32 *cs;
1906         int err;
1907         int i;
1908
1909         /*
1910          * Measure how long it takes for us to submit a request while the
1911          * engine is idle, but is resting in our context.
1912          *
1913          *    A: read CS_TIMESTAMP from CPU
1914          *    submit request
1915          *    B: read CS_TIMESTAMP on GPU
1916          *
1917          * Submission latency: B - A
1918          */
1919
1920         for (i = 0; i < ARRAY_SIZE(elapsed); i++) {
1921                 struct i915_request *rq;
1922
1923                 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
1924                 if (err)
1925                         return err;
1926
1927                 rq = i915_request_create(ce);
1928                 if (IS_ERR(rq)) {
1929                         err = PTR_ERR(rq);
1930                         goto err;
1931                 }
1932
1933                 cs = intel_ring_begin(rq, 4);
1934                 if (IS_ERR(cs)) {
1935                         i915_request_add(rq);
1936                         err = PTR_ERR(cs);
1937                         goto err;
1938                 }
1939
1940                 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
1941
1942                 intel_ring_advance(rq, cs);
1943
1944                 preempt_disable();
1945                 local_bh_disable();
1946                 elapsed[i] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
1947                 i915_request_add(rq);
1948                 local_bh_enable();
1949                 preempt_enable();
1950         }
1951
1952         err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
1953         if (err)
1954                 goto err;
1955
1956         for (i = 0; i < ARRAY_SIZE(elapsed); i++)
1957                 elapsed[i] = sema[i] - elapsed[i];
1958
1959         cycles = trifilter(elapsed);
1960         pr_info("%s: idle dispatch latency %d cycles, %lluns\n",
1961                 ce->engine->name, cycles >> TF_BIAS,
1962                 cycles_to_ns(ce->engine, cycles));
1963
1964         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
1965
1966 err:
1967         intel_gt_set_wedged(ce->engine->gt);
1968         return err;
1969 }
1970
1971 static int measure_busy_dispatch(struct intel_context *ce)
1972 {
1973         u32 *sema = hwsp_scratch(ce);
1974         const u32 offset = hwsp_offset(ce, sema);
1975         u32 elapsed[TF_COUNT + 1], cycles;
1976         u32 *cs;
1977         int err;
1978         int i;
1979
1980         /*
1981          * Measure how long it takes for us to submit a request while the
1982          * engine is busy, polling on a semaphore in our context. With
1983          * direct submission, this will include the cost of a lite restore.
1984          *
1985          *    A: read CS_TIMESTAMP from CPU
1986          *    submit request
1987          *    B: read CS_TIMESTAMP on GPU
1988          *
1989          * Submission latency: B - A
1990          */
1991
1992         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
1993                 struct i915_request *rq;
1994
1995                 rq = i915_request_create(ce);
1996                 if (IS_ERR(rq)) {
1997                         err = PTR_ERR(rq);
1998                         goto err;
1999                 }
2000
2001                 cs = intel_ring_begin(rq, 12);
2002                 if (IS_ERR(cs)) {
2003                         i915_request_add(rq);
2004                         err = PTR_ERR(cs);
2005                         goto err;
2006                 }
2007
2008                 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1);
2009                 cs = emit_semaphore_poll_until(cs, offset, i);
2010                 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2011
2012                 intel_ring_advance(rq, cs);
2013
2014                 if (i > 1 && wait_for(READ_ONCE(sema[i - 1]), 500)) {
2015                         err = -EIO;
2016                         goto err;
2017                 }
2018
2019                 preempt_disable();
2020                 local_bh_disable();
2021                 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2022                 i915_request_add(rq);
2023                 local_bh_enable();
2024                 semaphore_set(sema, i - 1);
2025                 preempt_enable();
2026         }
2027
2028         wait_for(READ_ONCE(sema[i - 1]), 500);
2029         semaphore_set(sema, i - 1);
2030
2031         for (i = 1; i <= TF_COUNT; i++) {
2032                 GEM_BUG_ON(sema[i] == -1);
2033                 elapsed[i - 1] = sema[i] - elapsed[i];
2034         }
2035
2036         cycles = trifilter(elapsed);
2037         pr_info("%s: busy dispatch latency %d cycles, %lluns\n",
2038                 ce->engine->name, cycles >> TF_BIAS,
2039                 cycles_to_ns(ce->engine, cycles));
2040
2041         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2042
2043 err:
2044         intel_gt_set_wedged(ce->engine->gt);
2045         return err;
2046 }
2047
2048 static int plug(struct intel_engine_cs *engine, u32 *sema, u32 mode, int value)
2049 {
2050         const u32 offset =
2051                 i915_ggtt_offset(engine->status_page.vma) +
2052                 offset_in_page(sema);
2053         struct i915_request *rq;
2054         u32 *cs;
2055
2056         rq = i915_request_create(engine->kernel_context);
2057         if (IS_ERR(rq))
2058                 return PTR_ERR(rq);
2059
2060         cs = intel_ring_begin(rq, 4);
2061         if (IS_ERR(cs)) {
2062                 i915_request_add(rq);
2063                 return PTR_ERR(cs);
2064         }
2065
2066         cs = emit_semaphore_poll(cs, mode, value, offset);
2067
2068         intel_ring_advance(rq, cs);
2069         i915_request_add(rq);
2070
2071         return 0;
2072 }
2073
2074 static int measure_inter_request(struct intel_context *ce)
2075 {
2076         u32 *sema = hwsp_scratch(ce);
2077         const u32 offset = hwsp_offset(ce, sema);
2078         u32 elapsed[TF_COUNT + 1], cycles;
2079         struct i915_sw_fence *submit;
2080         int i, err;
2081
2082         /*
2083          * Measure how long it takes to advance from one request into the
2084          * next. Between each request we flush the GPU caches to memory,
2085          * update the breadcrumbs, and then invalidate those caches.
2086          * We queue up all the requests to be submitted in one batch so
2087          * it should be one set of contiguous measurements.
2088          *
2089          *    A: read CS_TIMESTAMP on GPU
2090          *    advance request
2091          *    B: read CS_TIMESTAMP on GPU
2092          *
2093          * Request latency: B - A
2094          */
2095
2096         err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0);
2097         if (err)
2098                 return err;
2099
2100         submit = heap_fence_create(GFP_KERNEL);
2101         if (!submit) {
2102                 semaphore_set(sema, 1);
2103                 return -ENOMEM;
2104         }
2105
2106         intel_engine_flush_submission(ce->engine);
2107         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2108                 struct i915_request *rq;
2109                 u32 *cs;
2110
2111                 rq = i915_request_create(ce);
2112                 if (IS_ERR(rq)) {
2113                         err = PTR_ERR(rq);
2114                         goto err_submit;
2115                 }
2116
2117                 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
2118                                                        submit,
2119                                                        GFP_KERNEL);
2120                 if (err < 0) {
2121                         i915_request_add(rq);
2122                         goto err_submit;
2123                 }
2124
2125                 cs = intel_ring_begin(rq, 4);
2126                 if (IS_ERR(cs)) {
2127                         i915_request_add(rq);
2128                         err = PTR_ERR(cs);
2129                         goto err_submit;
2130                 }
2131
2132                 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2133
2134                 intel_ring_advance(rq, cs);
2135                 i915_request_add(rq);
2136         }
2137         i915_sw_fence_commit(submit);
2138         intel_engine_flush_submission(ce->engine);
2139         heap_fence_put(submit);
2140
2141         semaphore_set(sema, 1);
2142         err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2143         if (err)
2144                 goto err;
2145
2146         for (i = 1; i <= TF_COUNT; i++)
2147                 elapsed[i - 1] = sema[i + 1] - sema[i];
2148
2149         cycles = trifilter(elapsed);
2150         pr_info("%s: inter-request latency %d cycles, %lluns\n",
2151                 ce->engine->name, cycles >> TF_BIAS,
2152                 cycles_to_ns(ce->engine, cycles));
2153
2154         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2155
2156 err_submit:
2157         i915_sw_fence_commit(submit);
2158         heap_fence_put(submit);
2159         semaphore_set(sema, 1);
2160 err:
2161         intel_gt_set_wedged(ce->engine->gt);
2162         return err;
2163 }
2164
2165 static int measure_context_switch(struct intel_context *ce)
2166 {
2167         u32 *sema = hwsp_scratch(ce);
2168         const u32 offset = hwsp_offset(ce, sema);
2169         struct i915_request *fence = NULL;
2170         u32 elapsed[TF_COUNT + 1], cycles;
2171         int i, j, err;
2172         u32 *cs;
2173
2174         /*
2175          * Measure how long it takes to advance from one request in one
2176          * context to a request in another context. This allows us to
2177          * measure how long the context save/restore take, along with all
2178          * the inter-context setup we require.
2179          *
2180          *    A: read CS_TIMESTAMP on GPU
2181          *    switch context
2182          *    B: read CS_TIMESTAMP on GPU
2183          *
2184          * Context switch latency: B - A
2185          */
2186
2187         err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0);
2188         if (err)
2189                 return err;
2190
2191         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2192                 struct intel_context *arr[] = {
2193                         ce, ce->engine->kernel_context
2194                 };
2195                 u32 addr = offset + ARRAY_SIZE(arr) * i * sizeof(u32);
2196
2197                 for (j = 0; j < ARRAY_SIZE(arr); j++) {
2198                         struct i915_request *rq;
2199
2200                         rq = i915_request_create(arr[j]);
2201                         if (IS_ERR(rq)) {
2202                                 err = PTR_ERR(rq);
2203                                 goto err_fence;
2204                         }
2205
2206                         if (fence) {
2207                                 err = i915_request_await_dma_fence(rq,
2208                                                                    &fence->fence);
2209                                 if (err) {
2210                                         i915_request_add(rq);
2211                                         goto err_fence;
2212                                 }
2213                         }
2214
2215                         cs = intel_ring_begin(rq, 4);
2216                         if (IS_ERR(cs)) {
2217                                 i915_request_add(rq);
2218                                 err = PTR_ERR(cs);
2219                                 goto err_fence;
2220                         }
2221
2222                         cs = emit_timestamp_store(cs, ce, addr);
2223                         addr += sizeof(u32);
2224
2225                         intel_ring_advance(rq, cs);
2226
2227                         i915_request_put(fence);
2228                         fence = i915_request_get(rq);
2229
2230                         i915_request_add(rq);
2231                 }
2232         }
2233         i915_request_put(fence);
2234         intel_engine_flush_submission(ce->engine);
2235
2236         semaphore_set(sema, 1);
2237         err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2238         if (err)
2239                 goto err;
2240
2241         for (i = 1; i <= TF_COUNT; i++)
2242                 elapsed[i - 1] = sema[2 * i + 2] - sema[2 * i + 1];
2243
2244         cycles = trifilter(elapsed);
2245         pr_info("%s: context switch latency %d cycles, %lluns\n",
2246                 ce->engine->name, cycles >> TF_BIAS,
2247                 cycles_to_ns(ce->engine, cycles));
2248
2249         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2250
2251 err_fence:
2252         i915_request_put(fence);
2253         semaphore_set(sema, 1);
2254 err:
2255         intel_gt_set_wedged(ce->engine->gt);
2256         return err;
2257 }
2258
2259 static int measure_preemption(struct intel_context *ce)
2260 {
2261         u32 *sema = hwsp_scratch(ce);
2262         const u32 offset = hwsp_offset(ce, sema);
2263         u32 elapsed[TF_COUNT], cycles;
2264         u32 *cs;
2265         int err;
2266         int i;
2267
2268         /*
2269          * We measure two latencies while triggering preemption. The first
2270          * latency is how long it takes for us to submit a preempting request.
2271          * The second latency is how it takes for us to return from the
2272          * preemption back to the original context.
2273          *
2274          *    A: read CS_TIMESTAMP from CPU
2275          *    submit preemption
2276          *    B: read CS_TIMESTAMP on GPU (in preempting context)
2277          *    context switch
2278          *    C: read CS_TIMESTAMP on GPU (in original context)
2279          *
2280          * Preemption dispatch latency: B - A
2281          * Preemption switch latency: C - B
2282          */
2283
2284         if (!intel_engine_has_preemption(ce->engine))
2285                 return 0;
2286
2287         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2288                 u32 addr = offset + 2 * i * sizeof(u32);
2289                 struct i915_request *rq;
2290
2291                 rq = i915_request_create(ce);
2292                 if (IS_ERR(rq)) {
2293                         err = PTR_ERR(rq);
2294                         goto err;
2295                 }
2296
2297                 cs = intel_ring_begin(rq, 12);
2298                 if (IS_ERR(cs)) {
2299                         i915_request_add(rq);
2300                         err = PTR_ERR(cs);
2301                         goto err;
2302                 }
2303
2304                 cs = emit_store_dw(cs, addr, -1);
2305                 cs = emit_semaphore_poll_until(cs, offset, i);
2306                 cs = emit_timestamp_store(cs, ce, addr + sizeof(u32));
2307
2308                 intel_ring_advance(rq, cs);
2309                 i915_request_add(rq);
2310
2311                 if (wait_for(READ_ONCE(sema[2 * i]) == -1, 500)) {
2312                         err = -EIO;
2313                         goto err;
2314                 }
2315
2316                 rq = i915_request_create(ce->engine->kernel_context);
2317                 if (IS_ERR(rq)) {
2318                         err = PTR_ERR(rq);
2319                         goto err;
2320                 }
2321
2322                 cs = intel_ring_begin(rq, 8);
2323                 if (IS_ERR(cs)) {
2324                         i915_request_add(rq);
2325                         err = PTR_ERR(cs);
2326                         goto err;
2327                 }
2328
2329                 cs = emit_timestamp_store(cs, ce, addr);
2330                 cs = emit_store_dw(cs, offset, i);
2331
2332                 intel_ring_advance(rq, cs);
2333                 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2334
2335                 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2336                 i915_request_add(rq);
2337         }
2338
2339         if (wait_for(READ_ONCE(sema[2 * i - 2]) != -1, 500)) {
2340                 err = -EIO;
2341                 goto err;
2342         }
2343
2344         for (i = 1; i <= TF_COUNT; i++)
2345                 elapsed[i - 1] = sema[2 * i + 0] - elapsed[i - 1];
2346
2347         cycles = trifilter(elapsed);
2348         pr_info("%s: preemption dispatch latency %d cycles, %lluns\n",
2349                 ce->engine->name, cycles >> TF_BIAS,
2350                 cycles_to_ns(ce->engine, cycles));
2351
2352         for (i = 1; i <= TF_COUNT; i++)
2353                 elapsed[i - 1] = sema[2 * i + 1] - sema[2 * i + 0];
2354
2355         cycles = trifilter(elapsed);
2356         pr_info("%s: preemption switch latency %d cycles, %lluns\n",
2357                 ce->engine->name, cycles >> TF_BIAS,
2358                 cycles_to_ns(ce->engine, cycles));
2359
2360         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2361
2362 err:
2363         intel_gt_set_wedged(ce->engine->gt);
2364         return err;
2365 }
2366
2367 struct signal_cb {
2368         struct dma_fence_cb base;
2369         bool seen;
2370 };
2371
2372 static void signal_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
2373 {
2374         struct signal_cb *s = container_of(cb, typeof(*s), base);
2375
2376         smp_store_mb(s->seen, true); /* be safe, be strong */
2377 }
2378
2379 static int measure_completion(struct intel_context *ce)
2380 {
2381         u32 *sema = hwsp_scratch(ce);
2382         const u32 offset = hwsp_offset(ce, sema);
2383         u32 elapsed[TF_COUNT], cycles;
2384         u32 *cs;
2385         int err;
2386         int i;
2387
2388         /*
2389          * Measure how long it takes for the signal (interrupt) to be
2390          * sent from the GPU to be processed by the CPU.
2391          *
2392          *    A: read CS_TIMESTAMP on GPU
2393          *    signal
2394          *    B: read CS_TIMESTAMP from CPU
2395          *
2396          * Completion latency: B - A
2397          */
2398
2399         for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2400                 struct signal_cb cb = { .seen = false };
2401                 struct i915_request *rq;
2402
2403                 rq = i915_request_create(ce);
2404                 if (IS_ERR(rq)) {
2405                         err = PTR_ERR(rq);
2406                         goto err;
2407                 }
2408
2409                 cs = intel_ring_begin(rq, 12);
2410                 if (IS_ERR(cs)) {
2411                         i915_request_add(rq);
2412                         err = PTR_ERR(cs);
2413                         goto err;
2414                 }
2415
2416                 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1);
2417                 cs = emit_semaphore_poll_until(cs, offset, i);
2418                 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2419
2420                 intel_ring_advance(rq, cs);
2421
2422                 dma_fence_add_callback(&rq->fence, &cb.base, signal_cb);
2423                 i915_request_add(rq);
2424
2425                 intel_engine_flush_submission(ce->engine);
2426                 if (wait_for(READ_ONCE(sema[i]) == -1, 50)) {
2427                         err = -EIO;
2428                         goto err;
2429                 }
2430
2431                 preempt_disable();
2432                 semaphore_set(sema, i);
2433                 while (!READ_ONCE(cb.seen))
2434                         cpu_relax();
2435
2436                 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2437                 preempt_enable();
2438         }
2439
2440         err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2441         if (err)
2442                 goto err;
2443
2444         for (i = 0; i < ARRAY_SIZE(elapsed); i++) {
2445                 GEM_BUG_ON(sema[i + 1] == -1);
2446                 elapsed[i] = elapsed[i] - sema[i + 1];
2447         }
2448
2449         cycles = trifilter(elapsed);
2450         pr_info("%s: completion latency %d cycles, %lluns\n",
2451                 ce->engine->name, cycles >> TF_BIAS,
2452                 cycles_to_ns(ce->engine, cycles));
2453
2454         return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2455
2456 err:
2457         intel_gt_set_wedged(ce->engine->gt);
2458         return err;
2459 }
2460
2461 static void rps_pin(struct intel_gt *gt)
2462 {
2463         /* Pin the frequency to max */
2464         atomic_inc(&gt->rps.num_waiters);
2465         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
2466
2467         mutex_lock(&gt->rps.lock);
2468         intel_rps_set(&gt->rps, gt->rps.max_freq);
2469         mutex_unlock(&gt->rps.lock);
2470 }
2471
2472 static void rps_unpin(struct intel_gt *gt)
2473 {
2474         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
2475         atomic_dec(&gt->rps.num_waiters);
2476 }
2477
2478 static int perf_request_latency(void *arg)
2479 {
2480         struct drm_i915_private *i915 = arg;
2481         struct intel_engine_cs *engine;
2482         struct pm_qos_request qos;
2483         int err = 0;
2484
2485         if (INTEL_GEN(i915) < 8) /* per-engine CS timestamp, semaphores */
2486                 return 0;
2487
2488         cpu_latency_qos_add_request(&qos, 0); /* disable cstates */
2489
2490         for_each_uabi_engine(engine, i915) {
2491                 struct intel_context *ce;
2492
2493                 ce = intel_context_create(engine);
2494                 if (IS_ERR(ce)) {
2495                         err = PTR_ERR(ce);
2496                         goto out;
2497                 }
2498
2499                 err = intel_context_pin(ce);
2500                 if (err) {
2501                         intel_context_put(ce);
2502                         goto out;
2503                 }
2504
2505                 st_engine_heartbeat_disable(engine);
2506                 rps_pin(engine->gt);
2507
2508                 if (err == 0)
2509                         err = measure_semaphore_response(ce);
2510                 if (err == 0)
2511                         err = measure_idle_dispatch(ce);
2512                 if (err == 0)
2513                         err = measure_busy_dispatch(ce);
2514                 if (err == 0)
2515                         err = measure_inter_request(ce);
2516                 if (err == 0)
2517                         err = measure_context_switch(ce);
2518                 if (err == 0)
2519                         err = measure_preemption(ce);
2520                 if (err == 0)
2521                         err = measure_completion(ce);
2522
2523                 rps_unpin(engine->gt);
2524                 st_engine_heartbeat_enable(engine);
2525
2526                 intel_context_unpin(ce);
2527                 intel_context_put(ce);
2528                 if (err)
2529                         goto out;
2530         }
2531
2532 out:
2533         if (igt_flush_test(i915))
2534                 err = -EIO;
2535
2536         cpu_latency_qos_remove_request(&qos);
2537         return err;
2538 }
2539
2540 static int s_sync0(void *arg)
2541 {
2542         struct perf_series *ps = arg;
2543         IGT_TIMEOUT(end_time);
2544         unsigned int idx = 0;
2545         int err = 0;
2546
2547         GEM_BUG_ON(!ps->nengines);
2548         do {
2549                 struct i915_request *rq;
2550
2551                 rq = i915_request_create(ps->ce[idx]);
2552                 if (IS_ERR(rq)) {
2553                         err = PTR_ERR(rq);
2554                         break;
2555                 }
2556
2557                 i915_request_get(rq);
2558                 i915_request_add(rq);
2559
2560                 if (i915_request_wait(rq, 0, HZ / 5) < 0)
2561                         err = -ETIME;
2562                 i915_request_put(rq);
2563                 if (err)
2564                         break;
2565
2566                 if (++idx == ps->nengines)
2567                         idx = 0;
2568         } while (!__igt_timeout(end_time, NULL));
2569
2570         return err;
2571 }
2572
2573 static int s_sync1(void *arg)
2574 {
2575         struct perf_series *ps = arg;
2576         struct i915_request *prev = NULL;
2577         IGT_TIMEOUT(end_time);
2578         unsigned int idx = 0;
2579         int err = 0;
2580
2581         GEM_BUG_ON(!ps->nengines);
2582         do {
2583                 struct i915_request *rq;
2584
2585                 rq = i915_request_create(ps->ce[idx]);
2586                 if (IS_ERR(rq)) {
2587                         err = PTR_ERR(rq);
2588                         break;
2589                 }
2590
2591                 i915_request_get(rq);
2592                 i915_request_add(rq);
2593
2594                 if (prev && i915_request_wait(prev, 0, HZ / 5) < 0)
2595                         err = -ETIME;
2596                 i915_request_put(prev);
2597                 prev = rq;
2598                 if (err)
2599                         break;
2600
2601                 if (++idx == ps->nengines)
2602                         idx = 0;
2603         } while (!__igt_timeout(end_time, NULL));
2604         i915_request_put(prev);
2605
2606         return err;
2607 }
2608
2609 static int s_many(void *arg)
2610 {
2611         struct perf_series *ps = arg;
2612         IGT_TIMEOUT(end_time);
2613         unsigned int idx = 0;
2614
2615         GEM_BUG_ON(!ps->nengines);
2616         do {
2617                 struct i915_request *rq;
2618
2619                 rq = i915_request_create(ps->ce[idx]);
2620                 if (IS_ERR(rq))
2621                         return PTR_ERR(rq);
2622
2623                 i915_request_add(rq);
2624
2625                 if (++idx == ps->nengines)
2626                         idx = 0;
2627         } while (!__igt_timeout(end_time, NULL));
2628
2629         return 0;
2630 }
2631
2632 static int perf_series_engines(void *arg)
2633 {
2634         struct drm_i915_private *i915 = arg;
2635         static int (* const func[])(void *arg) = {
2636                 s_sync0,
2637                 s_sync1,
2638                 s_many,
2639                 NULL,
2640         };
2641         const unsigned int nengines = num_uabi_engines(i915);
2642         struct intel_engine_cs *engine;
2643         int (* const *fn)(void *arg);
2644         struct pm_qos_request qos;
2645         struct perf_stats *stats;
2646         struct perf_series *ps;
2647         unsigned int idx;
2648         int err = 0;
2649
2650         stats = kcalloc(nengines, sizeof(*stats), GFP_KERNEL);
2651         if (!stats)
2652                 return -ENOMEM;
2653
2654         ps = kzalloc(struct_size(ps, ce, nengines), GFP_KERNEL);
2655         if (!ps) {
2656                 kfree(stats);
2657                 return -ENOMEM;
2658         }
2659
2660         cpu_latency_qos_add_request(&qos, 0); /* disable cstates */
2661
2662         ps->i915 = i915;
2663         ps->nengines = nengines;
2664
2665         idx = 0;
2666         for_each_uabi_engine(engine, i915) {
2667                 struct intel_context *ce;
2668
2669                 ce = intel_context_create(engine);
2670                 if (IS_ERR(ce)) {
2671                         err = PTR_ERR(ce);
2672                         goto out;
2673                 }
2674
2675                 err = intel_context_pin(ce);
2676                 if (err) {
2677                         intel_context_put(ce);
2678                         goto out;
2679                 }
2680
2681                 ps->ce[idx++] = ce;
2682         }
2683         GEM_BUG_ON(idx != ps->nengines);
2684
2685         for (fn = func; *fn && !err; fn++) {
2686                 char name[KSYM_NAME_LEN];
2687                 struct igt_live_test t;
2688
2689                 snprintf(name, sizeof(name), "%ps", *fn);
2690                 err = igt_live_test_begin(&t, i915, __func__, name);
2691                 if (err)
2692                         break;
2693
2694                 for (idx = 0; idx < nengines; idx++) {
2695                         struct perf_stats *p =
2696                                 memset(&stats[idx], 0, sizeof(stats[idx]));
2697                         struct intel_context *ce = ps->ce[idx];
2698
2699                         p->engine = ps->ce[idx]->engine;
2700                         intel_engine_pm_get(p->engine);
2701
2702                         if (intel_engine_supports_stats(p->engine))
2703                                 p->busy = intel_engine_get_busy_time(p->engine,
2704                                                                      &p->time) + 1;
2705                         else
2706                                 p->time = ktime_get();
2707                         p->runtime = -intel_context_get_total_runtime_ns(ce);
2708                 }
2709
2710                 err = (*fn)(ps);
2711                 if (igt_live_test_end(&t))
2712                         err = -EIO;
2713
2714                 for (idx = 0; idx < nengines; idx++) {
2715                         struct perf_stats *p = &stats[idx];
2716                         struct intel_context *ce = ps->ce[idx];
2717                         int integer, decimal;
2718                         u64 busy, dt, now;
2719
2720                         if (p->busy)
2721                                 p->busy = ktime_sub(intel_engine_get_busy_time(p->engine,
2722                                                                                &now),
2723                                                     p->busy - 1);
2724                         else
2725                                 now = ktime_get();
2726                         p->time = ktime_sub(now, p->time);
2727
2728                         err = switch_to_kernel_sync(ce, err);
2729                         p->runtime += intel_context_get_total_runtime_ns(ce);
2730                         intel_engine_pm_put(p->engine);
2731
2732                         busy = 100 * ktime_to_ns(p->busy);
2733                         dt = ktime_to_ns(p->time);
2734                         if (dt) {
2735                                 integer = div64_u64(busy, dt);
2736                                 busy -= integer * dt;
2737                                 decimal = div64_u64(100 * busy, dt);
2738                         } else {
2739                                 integer = 0;
2740                                 decimal = 0;
2741                         }
2742
2743                         pr_info("%s %5s: { seqno:%d, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n",
2744                                 name, p->engine->name, ce->timeline->seqno,
2745                                 integer, decimal,
2746                                 div_u64(p->runtime, 1000 * 1000),
2747                                 div_u64(ktime_to_ns(p->time), 1000 * 1000));
2748                 }
2749         }
2750
2751 out:
2752         for (idx = 0; idx < nengines; idx++) {
2753                 if (IS_ERR_OR_NULL(ps->ce[idx]))
2754                         break;
2755
2756                 intel_context_unpin(ps->ce[idx]);
2757                 intel_context_put(ps->ce[idx]);
2758         }
2759         kfree(ps);
2760
2761         cpu_latency_qos_remove_request(&qos);
2762         kfree(stats);
2763         return err;
2764 }
2765
2766 static int p_sync0(void *arg)
2767 {
2768         struct perf_stats *p = arg;
2769         struct intel_engine_cs *engine = p->engine;
2770         struct intel_context *ce;
2771         IGT_TIMEOUT(end_time);
2772         unsigned long count;
2773         bool busy;
2774         int err = 0;
2775
2776         ce = intel_context_create(engine);
2777         if (IS_ERR(ce))
2778                 return PTR_ERR(ce);
2779
2780         err = intel_context_pin(ce);
2781         if (err) {
2782                 intel_context_put(ce);
2783                 return err;
2784         }
2785
2786         if (intel_engine_supports_stats(engine)) {
2787                 p->busy = intel_engine_get_busy_time(engine, &p->time);
2788                 busy = true;
2789         } else {
2790                 p->time = ktime_get();
2791                 busy = false;
2792         }
2793
2794         count = 0;
2795         do {
2796                 struct i915_request *rq;
2797
2798                 rq = i915_request_create(ce);
2799                 if (IS_ERR(rq)) {
2800                         err = PTR_ERR(rq);
2801                         break;
2802                 }
2803
2804                 i915_request_get(rq);
2805                 i915_request_add(rq);
2806
2807                 err = 0;
2808                 if (i915_request_wait(rq, 0, HZ / 5) < 0)
2809                         err = -ETIME;
2810                 i915_request_put(rq);
2811                 if (err)
2812                         break;
2813
2814                 count++;
2815         } while (!__igt_timeout(end_time, NULL));
2816
2817         if (busy) {
2818                 ktime_t now;
2819
2820                 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
2821                                     p->busy);
2822                 p->time = ktime_sub(now, p->time);
2823         } else {
2824                 p->time = ktime_sub(ktime_get(), p->time);
2825         }
2826
2827         err = switch_to_kernel_sync(ce, err);
2828         p->runtime = intel_context_get_total_runtime_ns(ce);
2829         p->count = count;
2830
2831         intel_context_unpin(ce);
2832         intel_context_put(ce);
2833         return err;
2834 }
2835
2836 static int p_sync1(void *arg)
2837 {
2838         struct perf_stats *p = arg;
2839         struct intel_engine_cs *engine = p->engine;
2840         struct i915_request *prev = NULL;
2841         struct intel_context *ce;
2842         IGT_TIMEOUT(end_time);
2843         unsigned long count;
2844         bool busy;
2845         int err = 0;
2846
2847         ce = intel_context_create(engine);
2848         if (IS_ERR(ce))
2849                 return PTR_ERR(ce);
2850
2851         err = intel_context_pin(ce);
2852         if (err) {
2853                 intel_context_put(ce);
2854                 return err;
2855         }
2856
2857         if (intel_engine_supports_stats(engine)) {
2858                 p->busy = intel_engine_get_busy_time(engine, &p->time);
2859                 busy = true;
2860         } else {
2861                 p->time = ktime_get();
2862                 busy = false;
2863         }
2864
2865         count = 0;
2866         do {
2867                 struct i915_request *rq;
2868
2869                 rq = i915_request_create(ce);
2870                 if (IS_ERR(rq)) {
2871                         err = PTR_ERR(rq);
2872                         break;
2873                 }
2874
2875                 i915_request_get(rq);
2876                 i915_request_add(rq);
2877
2878                 err = 0;
2879                 if (prev && i915_request_wait(prev, 0, HZ / 5) < 0)
2880                         err = -ETIME;
2881                 i915_request_put(prev);
2882                 prev = rq;
2883                 if (err)
2884                         break;
2885
2886                 count++;
2887         } while (!__igt_timeout(end_time, NULL));
2888         i915_request_put(prev);
2889
2890         if (busy) {
2891                 ktime_t now;
2892
2893                 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
2894                                     p->busy);
2895                 p->time = ktime_sub(now, p->time);
2896         } else {
2897                 p->time = ktime_sub(ktime_get(), p->time);
2898         }
2899
2900         err = switch_to_kernel_sync(ce, err);
2901         p->runtime = intel_context_get_total_runtime_ns(ce);
2902         p->count = count;
2903
2904         intel_context_unpin(ce);
2905         intel_context_put(ce);
2906         return err;
2907 }
2908
2909 static int p_many(void *arg)
2910 {
2911         struct perf_stats *p = arg;
2912         struct intel_engine_cs *engine = p->engine;
2913         struct intel_context *ce;
2914         IGT_TIMEOUT(end_time);
2915         unsigned long count;
2916         int err = 0;
2917         bool busy;
2918
2919         ce = intel_context_create(engine);
2920         if (IS_ERR(ce))
2921                 return PTR_ERR(ce);
2922
2923         err = intel_context_pin(ce);
2924         if (err) {
2925                 intel_context_put(ce);
2926                 return err;
2927         }
2928
2929         if (intel_engine_supports_stats(engine)) {
2930                 p->busy = intel_engine_get_busy_time(engine, &p->time);
2931                 busy = true;
2932         } else {
2933                 p->time = ktime_get();
2934                 busy = false;
2935         }
2936
2937         count = 0;
2938         do {
2939                 struct i915_request *rq;
2940
2941                 rq = i915_request_create(ce);
2942                 if (IS_ERR(rq)) {
2943                         err = PTR_ERR(rq);
2944                         break;
2945                 }
2946
2947                 i915_request_add(rq);
2948                 count++;
2949         } while (!__igt_timeout(end_time, NULL));
2950
2951         if (busy) {
2952                 ktime_t now;
2953
2954                 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
2955                                     p->busy);
2956                 p->time = ktime_sub(now, p->time);
2957         } else {
2958                 p->time = ktime_sub(ktime_get(), p->time);
2959         }
2960
2961         err = switch_to_kernel_sync(ce, err);
2962         p->runtime = intel_context_get_total_runtime_ns(ce);
2963         p->count = count;
2964
2965         intel_context_unpin(ce);
2966         intel_context_put(ce);
2967         return err;
2968 }
2969
2970 static int perf_parallel_engines(void *arg)
2971 {
2972         struct drm_i915_private *i915 = arg;
2973         static int (* const func[])(void *arg) = {
2974                 p_sync0,
2975                 p_sync1,
2976                 p_many,
2977                 NULL,
2978         };
2979         const unsigned int nengines = num_uabi_engines(i915);
2980         struct intel_engine_cs *engine;
2981         int (* const *fn)(void *arg);
2982         struct pm_qos_request qos;
2983         struct {
2984                 struct perf_stats p;
2985                 struct task_struct *tsk;
2986         } *engines;
2987         int err = 0;
2988
2989         engines = kcalloc(nengines, sizeof(*engines), GFP_KERNEL);
2990         if (!engines)
2991                 return -ENOMEM;
2992
2993         cpu_latency_qos_add_request(&qos, 0);
2994
2995         for (fn = func; *fn; fn++) {
2996                 char name[KSYM_NAME_LEN];
2997                 struct igt_live_test t;
2998                 unsigned int idx;
2999
3000                 snprintf(name, sizeof(name), "%ps", *fn);
3001                 err = igt_live_test_begin(&t, i915, __func__, name);
3002                 if (err)
3003                         break;
3004
3005                 atomic_set(&i915->selftest.counter, nengines);
3006
3007                 idx = 0;
3008                 for_each_uabi_engine(engine, i915) {
3009                         intel_engine_pm_get(engine);
3010
3011                         memset(&engines[idx].p, 0, sizeof(engines[idx].p));
3012                         engines[idx].p.engine = engine;
3013
3014                         engines[idx].tsk = kthread_run(*fn, &engines[idx].p,
3015                                                        "igt:%s", engine->name);
3016                         if (IS_ERR(engines[idx].tsk)) {
3017                                 err = PTR_ERR(engines[idx].tsk);
3018                                 intel_engine_pm_put(engine);
3019                                 break;
3020                         }
3021                         get_task_struct(engines[idx++].tsk);
3022                 }
3023
3024                 yield(); /* start all threads before we kthread_stop() */
3025
3026                 idx = 0;
3027                 for_each_uabi_engine(engine, i915) {
3028                         int status;
3029
3030                         if (IS_ERR(engines[idx].tsk))
3031                                 break;
3032
3033                         status = kthread_stop(engines[idx].tsk);
3034                         if (status && !err)
3035                                 err = status;
3036
3037                         intel_engine_pm_put(engine);
3038                         put_task_struct(engines[idx++].tsk);
3039                 }
3040
3041                 if (igt_live_test_end(&t))
3042                         err = -EIO;
3043                 if (err)
3044                         break;
3045
3046                 idx = 0;
3047                 for_each_uabi_engine(engine, i915) {
3048                         struct perf_stats *p = &engines[idx].p;
3049                         u64 busy = 100 * ktime_to_ns(p->busy);
3050                         u64 dt = ktime_to_ns(p->time);
3051                         int integer, decimal;
3052
3053                         if (dt) {
3054                                 integer = div64_u64(busy, dt);
3055                                 busy -= integer * dt;
3056                                 decimal = div64_u64(100 * busy, dt);
3057                         } else {
3058                                 integer = 0;
3059                                 decimal = 0;
3060                         }
3061
3062                         GEM_BUG_ON(engine != p->engine);
3063                         pr_info("%s %5s: { count:%lu, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n",
3064                                 name, engine->name, p->count, integer, decimal,
3065                                 div_u64(p->runtime, 1000 * 1000),
3066                                 div_u64(ktime_to_ns(p->time), 1000 * 1000));
3067                         idx++;
3068                 }
3069         }
3070
3071         cpu_latency_qos_remove_request(&qos);
3072         kfree(engines);
3073         return err;
3074 }
3075
3076 int i915_request_perf_selftests(struct drm_i915_private *i915)
3077 {
3078         static const struct i915_subtest tests[] = {
3079                 SUBTEST(perf_request_latency),
3080                 SUBTEST(perf_series_engines),
3081                 SUBTEST(perf_parallel_engines),
3082         };
3083
3084         if (intel_gt_is_wedged(&i915->gt))
3085                 return 0;
3086
3087         return i915_subtests(tests, i915);
3088 }