drivers/gpu/drm/i915/intel_engine_cs.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drm_print.h>
  26
  27 #include "i915_drv.h"
  28 #include "i915_reset.h"
  29 #include "intel_ringbuffer.h"
  30 #include "intel_lrc.h"
  31
  32 /* Haswell does have the CXT_SIZE register however it does not appear to be
  33  * valid. Now, docs explain in dwords what is in the context object. The full
  34  * size is 70720 bytes, however, the power context and execlist context will
  35  * never be saved (power context is stored elsewhere, and execlists don't work
  36  * on HSW) - so the final size, including the extra state required for the
  37  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
  38  */
  39 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
  40
  41 #define DEFAULT_LR_CONTEXT_RENDER_SIZE  (22 * PAGE_SIZE)
  42 #define GEN8_LR_CONTEXT_RENDER_SIZE     (20 * PAGE_SIZE)
  43 #define GEN9_LR_CONTEXT_RENDER_SIZE     (22 * PAGE_SIZE)
  44 #define GEN10_LR_CONTEXT_RENDER_SIZE    (18 * PAGE_SIZE)
  45 #define GEN11_LR_CONTEXT_RENDER_SIZE    (14 * PAGE_SIZE)
  46
  47 #define GEN8_LR_CONTEXT_OTHER_SIZE      ( 2 * PAGE_SIZE)
  48
  49 struct engine_class_info {
  50         const char *name;
  51         int (*init_legacy)(struct intel_engine_cs *engine);
  52         int (*init_execlists)(struct intel_engine_cs *engine);
  53
  54         u8 uabi_class;
  55 };
  56
  57 static const struct engine_class_info intel_engine_classes[] = {
  58         [RENDER_CLASS] = {
  59                 .name = "rcs",
  60                 .init_execlists = logical_render_ring_init,
  61                 .init_legacy = intel_init_render_ring_buffer,
  62                 .uabi_class = I915_ENGINE_CLASS_RENDER,
  63         },
  64         [COPY_ENGINE_CLASS] = {
  65                 .name = "bcs",
  66                 .init_execlists = logical_xcs_ring_init,
  67                 .init_legacy = intel_init_blt_ring_buffer,
  68                 .uabi_class = I915_ENGINE_CLASS_COPY,
  69         },
  70         [VIDEO_DECODE_CLASS] = {
  71                 .name = "vcs",
  72                 .init_execlists = logical_xcs_ring_init,
  73                 .init_legacy = intel_init_bsd_ring_buffer,
  74                 .uabi_class = I915_ENGINE_CLASS_VIDEO,
  75         },
  76         [VIDEO_ENHANCEMENT_CLASS] = {
  77                 .name = "vecs",
  78                 .init_execlists = logical_xcs_ring_init,
  79                 .init_legacy = intel_init_vebox_ring_buffer,
  80                 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
  81         },
  82 };
  83
  84 #define MAX_MMIO_BASES 3
  85 struct engine_info {
  86         unsigned int hw_id;
  87         u8 class;
  88         u8 instance;
  89         /* mmio bases table *must* be sorted in reverse gen order */
  90         struct engine_mmio_base {
  91                 u32 gen : 8;
  92                 u32 base : 24;
  93         } mmio_bases[MAX_MMIO_BASES];
  94 };
  95
  96 static const struct engine_info intel_engines[] = {
  97         [RCS0] = {
  98                 .hw_id = RCS0_HW,
  99                 .class = RENDER_CLASS,
 100                 .instance = 0,
 101                 .mmio_bases = {
 102                         { .gen = 1, .base = RENDER_RING_BASE }
 103                 },
 104         },
 105         [BCS0] = {
 106                 .hw_id = BCS0_HW,
 107                 .class = COPY_ENGINE_CLASS,
 108                 .instance = 0,
 109                 .mmio_bases = {
 110                         { .gen = 6, .base = BLT_RING_BASE }
 111                 },
 112         },
 113         [VCS0] = {
 114                 .hw_id = VCS0_HW,
 115                 .class = VIDEO_DECODE_CLASS,
 116                 .instance = 0,
 117                 .mmio_bases = {
 118                         { .gen = 11, .base = GEN11_BSD_RING_BASE },
 119                         { .gen = 6, .base = GEN6_BSD_RING_BASE },
 120                         { .gen = 4, .base = BSD_RING_BASE }
 121                 },
 122         },
 123         [VCS1] = {
 124                 .hw_id = VCS1_HW,
 125                 .class = VIDEO_DECODE_CLASS,
 126                 .instance = 1,
 127                 .mmio_bases = {
 128                         { .gen = 11, .base = GEN11_BSD2_RING_BASE },
 129                         { .gen = 8, .base = GEN8_BSD2_RING_BASE }
 130                 },
 131         },
 132         [VCS2] = {
 133                 .hw_id = VCS2_HW,
 134                 .class = VIDEO_DECODE_CLASS,
 135                 .instance = 2,
 136                 .mmio_bases = {
 137                         { .gen = 11, .base = GEN11_BSD3_RING_BASE }
 138                 },
 139         },
 140         [VCS3] = {
 141                 .hw_id = VCS3_HW,
 142                 .class = VIDEO_DECODE_CLASS,
 143                 .instance = 3,
 144                 .mmio_bases = {
 145                         { .gen = 11, .base = GEN11_BSD4_RING_BASE }
 146                 },
 147         },
 148         [VECS0] = {
 149                 .hw_id = VECS0_HW,
 150                 .class = VIDEO_ENHANCEMENT_CLASS,
 151                 .instance = 0,
 152                 .mmio_bases = {
 153                         { .gen = 11, .base = GEN11_VEBOX_RING_BASE },
 154                         { .gen = 7, .base = VEBOX_RING_BASE }
 155                 },
 156         },
 157         [VECS1] = {
 158                 .hw_id = VECS1_HW,
 159                 .class = VIDEO_ENHANCEMENT_CLASS,
 160                 .instance = 1,
 161                 .mmio_bases = {
 162                         { .gen = 11, .base = GEN11_VEBOX2_RING_BASE }
 163                 },
 164         },
 165 };
 166
 167 /**
 168  * ___intel_engine_context_size() - return the size of the context for an engine
 169  * @dev_priv: i915 device private
 170  * @class: engine class
 171  *
 172  * Each engine class may require a different amount of space for a context
 173  * image.
 174  *
 175  * Return: size (in bytes) of an engine class specific context image
 176  *
 177  * Note: this size includes the HWSP, which is part of the context image
 178  * in LRC mode, but does not include the "shared data page" used with
 179  * GuC submission. The caller should account for this if using the GuC.
 180  */
 181 static u32
 182 __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 183 {
 184         u32 cxt_size;
 185
 186         BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
 187
 188         switch (class) {
 189         case RENDER_CLASS:
 190                 switch (INTEL_GEN(dev_priv)) {
 191                 default:
 192                         MISSING_CASE(INTEL_GEN(dev_priv));
 193                         return DEFAULT_LR_CONTEXT_RENDER_SIZE;
 194                 case 11:
 195                         return GEN11_LR_CONTEXT_RENDER_SIZE;
 196                 case 10:
 197                         return GEN10_LR_CONTEXT_RENDER_SIZE;
 198                 case 9:
 199                         return GEN9_LR_CONTEXT_RENDER_SIZE;
 200                 case 8:
 201                         return GEN8_LR_CONTEXT_RENDER_SIZE;
 202                 case 7:
 203                         if (IS_HASWELL(dev_priv))
 204                                 return HSW_CXT_TOTAL_SIZE;
 205
 206                         cxt_size = I915_READ(GEN7_CXT_SIZE);
 207                         return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
 208                                         PAGE_SIZE);
 209                 case 6:
 210                         cxt_size = I915_READ(CXT_SIZE);
 211                         return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
 212                                         PAGE_SIZE);
 213                 case 5:
 214                 case 4:
 215                 case 3:
 216                 case 2:
 217                 /* For the special day when i810 gets merged. */
 218                 case 1:
 219                         return 0;
 220                 }
 221                 break;
 222         default:
 223                 MISSING_CASE(class);
 224                 /* fall through */
 225         case VIDEO_DECODE_CLASS:
 226         case VIDEO_ENHANCEMENT_CLASS:
 227         case COPY_ENGINE_CLASS:
 228                 if (INTEL_GEN(dev_priv) < 8)
 229                         return 0;
 230                 return GEN8_LR_CONTEXT_OTHER_SIZE;
 231         }
 232 }
 233
 234 static u32 __engine_mmio_base(struct drm_i915_private *i915,
 235                               const struct engine_mmio_base *bases)
 236 {
 237         int i;
 238
 239         for (i = 0; i < MAX_MMIO_BASES; i++)
 240                 if (INTEL_GEN(i915) >= bases[i].gen)
 241                         break;
 242
 243         GEM_BUG_ON(i == MAX_MMIO_BASES);
 244         GEM_BUG_ON(!bases[i].base);
 245
 246         return bases[i].base;
 247 }
 248
 249 static void __sprint_engine_name(char *name, const struct engine_info *info)
 250 {
 251         WARN_ON(snprintf(name, INTEL_ENGINE_CS_MAX_NAME, "%s%u",
 252                          intel_engine_classes[info->class].name,
 253                          info->instance) >= INTEL_ENGINE_CS_MAX_NAME);
 254 }
 255
 256 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
 257 {
 258         struct drm_i915_private *dev_priv = engine->i915;
 259         i915_reg_t hwstam;
 260
 261         /*
 262          * Though they added more rings on g4x/ilk, they did not add
 263          * per-engine HWSTAM until gen6.
 264          */
 265         if (INTEL_GEN(dev_priv) < 6 && engine->class != RENDER_CLASS)
 266                 return;
 267
 268         hwstam = RING_HWSTAM(engine->mmio_base);
 269         if (INTEL_GEN(dev_priv) >= 3)
 270                 I915_WRITE(hwstam, mask);
 271         else
 272                 I915_WRITE16(hwstam, mask);
 273 }
 274
 275 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
 276 {
 277         /* Mask off all writes into the unknown HWSP */
 278         intel_engine_set_hwsp_writemask(engine, ~0u);
 279 }
 280
 281 static int
 282 intel_engine_setup(struct drm_i915_private *dev_priv,
 283                    enum intel_engine_id id)
 284 {
 285         const struct engine_info *info = &intel_engines[id];
 286         struct intel_engine_cs *engine;
 287
 288         GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
 289
 290         BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
 291         BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
 292
 293         if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
 294                 return -EINVAL;
 295
 296         if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
 297                 return -EINVAL;
 298
 299         if (GEM_DEBUG_WARN_ON(dev_priv->engine_class[info->class][info->instance]))
 300                 return -EINVAL;
 301
 302         GEM_BUG_ON(dev_priv->engine[id]);
 303         engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 304         if (!engine)
 305                 return -ENOMEM;
 306
 307         BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
 308
 309         engine->id = id;
 310         engine->mask = BIT(id);
 311         engine->i915 = dev_priv;
 312         __sprint_engine_name(engine->name, info);
 313         engine->hw_id = engine->guc_id = info->hw_id;
 314         engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
 315         engine->class = info->class;
 316         engine->instance = info->instance;
 317
 318         engine->uabi_class = intel_engine_classes[info->class].uabi_class;
 319
 320         engine->context_size = __intel_engine_context_size(dev_priv,
 321                                                            engine->class);
 322         if (WARN_ON(engine->context_size > BIT(20)))
 323                 engine->context_size = 0;
 324         if (engine->context_size)
 325                 DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
 326
 327         /* Nothing to do here, execute in order of dependencies */
 328         engine->schedule = NULL;
 329
 330         seqlock_init(&engine->stats.lock);
 331
 332         ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 333
 334         /* Scrub mmio state on takeover */
 335         intel_engine_sanitize_mmio(engine);
 336
 337         dev_priv->engine_class[info->class][info->instance] = engine;
 338         dev_priv->engine[id] = engine;
 339         return 0;
 340 }
 341
 342 /**
 343  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
 344  * @dev_priv: i915 device private
 345  *
 346  * Return: non-zero if the initialization failed.
 347  */
 348 int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 349 {
 350         struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
 351         const unsigned int engine_mask = INTEL_INFO(dev_priv)->engine_mask;
 352         struct intel_engine_cs *engine;
 353         enum intel_engine_id id;
 354         unsigned int mask = 0;
 355         unsigned int i;
 356         int err;
 357
 358         WARN_ON(engine_mask == 0);
 359         WARN_ON(engine_mask &
 360                 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
 361
 362         if (i915_inject_load_failure())
 363                 return -ENODEV;
 364
 365         for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
 366                 if (!HAS_ENGINE(dev_priv, i))
 367                         continue;
 368
 369                 err = intel_engine_setup(dev_priv, i);
 370                 if (err)
 371                         goto cleanup;
 372
 373                 mask |= BIT(i);
 374         }
 375
 376         /*
 377          * Catch failures to update intel_engines table when the new engines
 378          * are added to the driver by a warning and disabling the forgotten
 379          * engines.
 380          */
 381         if (WARN_ON(mask != engine_mask))
 382                 device_info->engine_mask = mask;
 383
 384         /* We always presume we have at least RCS available for later probing */
 385         if (WARN_ON(!HAS_ENGINE(dev_priv, RCS0))) {
 386                 err = -ENODEV;
 387                 goto cleanup;
 388         }
 389
 390         RUNTIME_INFO(dev_priv)->num_engines = hweight32(mask);
 391
 392         i915_check_and_clear_faults(dev_priv);
 393
 394         return 0;
 395
 396 cleanup:
 397         for_each_engine(engine, dev_priv, id)
 398                 kfree(engine);
 399         return err;
 400 }
 401
 402 /**
 403  * intel_engines_init() - init the Engine Command Streamers
 404  * @dev_priv: i915 device private
 405  *
 406  * Return: non-zero if the initialization failed.
 407  */
 408 int intel_engines_init(struct drm_i915_private *dev_priv)
 409 {
 410         struct intel_engine_cs *engine;
 411         enum intel_engine_id id, err_id;
 412         int err;
 413
 414         for_each_engine(engine, dev_priv, id) {
 415                 const struct engine_class_info *class_info =
 416                         &intel_engine_classes[engine->class];
 417                 int (*init)(struct intel_engine_cs *engine);
 418
 419                 if (HAS_EXECLISTS(dev_priv))
 420                         init = class_info->init_execlists;
 421                 else
 422                         init = class_info->init_legacy;
 423
 424                 err = -EINVAL;
 425                 err_id = id;
 426
 427                 if (GEM_DEBUG_WARN_ON(!init))
 428                         goto cleanup;
 429
 430                 err = init(engine);
 431                 if (err)
 432                         goto cleanup;
 433
 434                 GEM_BUG_ON(!engine->submit_request);
 435         }
 436
 437         return 0;
 438
 439 cleanup:
 440         for_each_engine(engine, dev_priv, id) {
 441                 if (id >= err_id) {
 442                         kfree(engine);
 443                         dev_priv->engine[id] = NULL;
 444                 } else {
 445                         dev_priv->gt.cleanup_engine(engine);
 446                 }
 447         }
 448         return err;
 449 }
 450
 451 static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 452 {
 453         i915_gem_batch_pool_init(&engine->batch_pool, engine);
 454 }
 455
 456 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 457 {
 458         struct intel_engine_execlists * const execlists = &engine->execlists;
 459
 460         execlists->port_mask = 1;
 461         GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
 462         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
 463
 464         execlists->queue_priority_hint = INT_MIN;
 465         execlists->queue = RB_ROOT_CACHED;
 466 }
 467
 468 static void cleanup_status_page(struct intel_engine_cs *engine)
 469 {
 470         struct i915_vma *vma;
 471
 472         /* Prevent writes into HWSP after returning the page to the system */
 473         intel_engine_set_hwsp_writemask(engine, ~0u);
 474
 475         vma = fetch_and_zero(&engine->status_page.vma);
 476         if (!vma)
 477                 return;
 478
 479         if (!HWS_NEEDS_PHYSICAL(engine->i915))
 480                 i915_vma_unpin(vma);
 481
 482         i915_gem_object_unpin_map(vma->obj);
 483         __i915_gem_object_release_unless_active(vma->obj);
 484 }
 485
 486 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
 487                                 struct i915_vma *vma)
 488 {
 489         unsigned int flags;
 490
 491         flags = PIN_GLOBAL;
 492         if (!HAS_LLC(engine->i915))
 493                 /*
 494                  * On g33, we cannot place HWS above 256MiB, so
 495                  * restrict its pinning to the low mappable arena.
 496                  * Though this restriction is not documented for
 497                  * gen4, gen5, or byt, they also behave similarly
 498                  * and hang if the HWS is placed at the top of the
 499                  * GTT. To generalise, it appears that all !llc
 500                  * platforms have issues with us placing the HWS
 501                  * above the mappable region (even though we never
 502                  * actually map it).
 503                  */
 504                 flags |= PIN_MAPPABLE;
 505         else
 506                 flags |= PIN_HIGH;
 507
 508         return i915_vma_pin(vma, 0, 0, flags);
 509 }
 510
 511 static int init_status_page(struct intel_engine_cs *engine)
 512 {
 513         struct drm_i915_gem_object *obj;
 514         struct i915_vma *vma;
 515         void *vaddr;
 516         int ret;
 517
 518         /*
 519          * Though the HWS register does support 36bit addresses, historically
 520          * we have had hangs and corruption reported due to wild writes if
 521          * the HWS is placed above 4G. We only allow objects to be allocated
 522          * in GFP_DMA32 for i965, and no earlier physical address users had
 523          * access to more than 4G.
 524          */
 525         obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 526         if (IS_ERR(obj)) {
 527                 DRM_ERROR("Failed to allocate status page\n");
 528                 return PTR_ERR(obj);
 529         }
 530
 531         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 532
 533         vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
 534         if (IS_ERR(vma)) {
 535                 ret = PTR_ERR(vma);
 536                 goto err;
 537         }
 538
 539         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 540         if (IS_ERR(vaddr)) {
 541                 ret = PTR_ERR(vaddr);
 542                 goto err;
 543         }
 544
 545         engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
 546         engine->status_page.vma = vma;
 547
 548         if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
 549                 ret = pin_ggtt_status_page(engine, vma);
 550                 if (ret)
 551                         goto err_unpin;
 552         }
 553
 554         return 0;
 555
 556 err_unpin:
 557         i915_gem_object_unpin_map(obj);
 558 err:
 559         i915_gem_object_put(obj);
 560         return ret;
 561 }
 562
 563 /**
 564  * intel_engines_setup_common - setup engine state not requiring hw access
 565  * @engine: Engine to setup.
 566  *
 567  * Initializes @engine@ structure members shared between legacy and execlists
 568  * submission modes which do not require hardware access.
 569  *
 570  * Typically done early in the submission mode specific engine setup stage.
 571  */
 572 int intel_engine_setup_common(struct intel_engine_cs *engine)
 573 {
 574         int err;
 575
 576         err = init_status_page(engine);
 577         if (err)
 578                 return err;
 579
 580         err = i915_timeline_init(engine->i915,
 581                                  &engine->timeline,
 582                                  engine->status_page.vma);
 583         if (err)
 584                 goto err_hwsp;
 585
 586         i915_timeline_set_subclass(&engine->timeline, TIMELINE_ENGINE);
 587
 588         intel_engine_init_breadcrumbs(engine);
 589         intel_engine_init_execlist(engine);
 590         intel_engine_init_hangcheck(engine);
 591         intel_engine_init_batch_pool(engine);
 592         intel_engine_init_cmd_parser(engine);
 593
 594         return 0;
 595
 596 err_hwsp:
 597         cleanup_status_page(engine);
 598         return err;
 599 }
 600
 601 void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
 602 {
 603         static const struct {
 604                 u8 engine;
 605                 u8 sched;
 606         } map[] = {
 607 #define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) }
 608                 MAP(PREEMPTION, PREEMPTION),
 609                 MAP(SEMAPHORES, SEMAPHORES),
 610 #undef MAP
 611         };
 612         struct intel_engine_cs *engine;
 613         enum intel_engine_id id;
 614         u32 enabled, disabled;
 615
 616         enabled = 0;
 617         disabled = 0;
 618         for_each_engine(engine, i915, id) { /* all engines must agree! */
 619                 int i;
 620
 621                 if (engine->schedule)
 622                         enabled |= (I915_SCHEDULER_CAP_ENABLED |
 623                                     I915_SCHEDULER_CAP_PRIORITY);
 624                 else
 625                         disabled |= (I915_SCHEDULER_CAP_ENABLED |
 626                                      I915_SCHEDULER_CAP_PRIORITY);
 627
 628                 for (i = 0; i < ARRAY_SIZE(map); i++) {
 629                         if (engine->flags & BIT(map[i].engine))
 630                                 enabled |= BIT(map[i].sched);
 631                         else
 632                                 disabled |= BIT(map[i].sched);
 633                 }
 634         }
 635
 636         i915->caps.scheduler = enabled & ~disabled;
 637         if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
 638                 i915->caps.scheduler = 0;
 639 }
 640
 641 struct measure_breadcrumb {
 642         struct i915_request rq;
 643         struct i915_timeline timeline;
 644         struct intel_ring ring;
 645         u32 cs[1024];
 646 };
 647
 648 static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
 649 {
 650         struct measure_breadcrumb *frame;
 651         int dw = -ENOMEM;
 652
 653         GEM_BUG_ON(!engine->i915->gt.scratch);
 654
 655         frame = kzalloc(sizeof(*frame), GFP_KERNEL);
 656         if (!frame)
 657                 return -ENOMEM;
 658
 659         if (i915_timeline_init(engine->i915,
 660                                &frame->timeline,
 661                                engine->status_page.vma))
 662                 goto out_frame;
 663
 664         INIT_LIST_HEAD(&frame->ring.request_list);
 665         frame->ring.timeline = &frame->timeline;
 666         frame->ring.vaddr = frame->cs;
 667         frame->ring.size = sizeof(frame->cs);
 668         frame->ring.effective_size = frame->ring.size;
 669         intel_ring_update_space(&frame->ring);
 670
 671         frame->rq.i915 = engine->i915;
 672         frame->rq.engine = engine;
 673         frame->rq.ring = &frame->ring;
 674         frame->rq.timeline = &frame->timeline;
 675
 676         dw = i915_timeline_pin(&frame->timeline);
 677         if (dw < 0)
 678                 goto out_timeline;
 679
 680         dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
 681
 682         i915_timeline_unpin(&frame->timeline);
 683
 684 out_timeline:
 685         i915_timeline_fini(&frame->timeline);
 686 out_frame:
 687         kfree(frame);
 688         return dw;
 689 }
 690
 691 static int pin_context(struct i915_gem_context *ctx,
 692                        struct intel_engine_cs *engine,
 693                        struct intel_context **out)
 694 {
 695         struct intel_context *ce;
 696
 697         ce = intel_context_pin(ctx, engine);
 698         if (IS_ERR(ce))
 699                 return PTR_ERR(ce);
 700
 701         *out = ce;
 702         return 0;
 703 }
 704
 705 /**
 706  * intel_engines_init_common - initialize cengine state which might require hw access
 707  * @engine: Engine to initialize.
 708  *
 709  * Initializes @engine@ structure members shared between legacy and execlists
 710  * submission modes which do require hardware access.
 711  *
 712  * Typcally done at later stages of submission mode specific engine setup.
 713  *
 714  * Returns zero on success or an error code on failure.
 715  */
 716 int intel_engine_init_common(struct intel_engine_cs *engine)
 717 {
 718         struct drm_i915_private *i915 = engine->i915;
 719         int ret;
 720
 721         /* We may need to do things with the shrinker which
 722          * require us to immediately switch back to the default
 723          * context. This can cause a problem as pinning the
 724          * default context also requires GTT space which may not
 725          * be available. To avoid this we always pin the default
 726          * context.
 727          */
 728         ret = pin_context(i915->kernel_context, engine,
 729                           &engine->kernel_context);
 730         if (ret)
 731                 return ret;
 732
 733         /*
 734          * Similarly the preempt context must always be available so that
 735          * we can interrupt the engine at any time. However, as preemption
 736          * is optional, we allow it to fail.
 737          */
 738         if (i915->preempt_context)
 739                 pin_context(i915->preempt_context, engine,
 740                             &engine->preempt_context);
 741
 742         ret = measure_breadcrumb_dw(engine);
 743         if (ret < 0)
 744                 goto err_unpin;
 745
 746         engine->emit_fini_breadcrumb_dw = ret;
 747
 748         engine->set_default_submission(engine);
 749
 750         return 0;
 751
 752 err_unpin:
 753         if (engine->preempt_context)
 754                 intel_context_unpin(engine->preempt_context);
 755         intel_context_unpin(engine->kernel_context);
 756         return ret;
 757 }
 758
 759 /**
 760  * intel_engines_cleanup_common - cleans up the engine state created by
 761  *                                the common initiailizers.
 762  * @engine: Engine to cleanup.
 763  *
 764  * This cleans up everything created by the common helpers.
 765  */
 766 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 767 {
 768         cleanup_status_page(engine);
 769
 770         intel_engine_fini_breadcrumbs(engine);
 771         intel_engine_cleanup_cmd_parser(engine);
 772         i915_gem_batch_pool_fini(&engine->batch_pool);
 773
 774         if (engine->default_state)
 775                 i915_gem_object_put(engine->default_state);
 776
 777         if (engine->preempt_context)
 778                 intel_context_unpin(engine->preempt_context);
 779         intel_context_unpin(engine->kernel_context);
 780
 781         i915_timeline_fini(&engine->timeline);
 782
 783         intel_wa_list_free(&engine->ctx_wa_list);
 784         intel_wa_list_free(&engine->wa_list);
 785         intel_wa_list_free(&engine->whitelist);
 786 }
 787
 788 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
 789 {
 790         struct drm_i915_private *dev_priv = engine->i915;
 791         u64 acthd;
 792
 793         if (INTEL_GEN(dev_priv) >= 8)
 794                 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
 795                                          RING_ACTHD_UDW(engine->mmio_base));
 796         else if (INTEL_GEN(dev_priv) >= 4)
 797                 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
 798         else
 799                 acthd = I915_READ(ACTHD);
 800
 801         return acthd;
 802 }
 803
 804 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
 805 {
 806         struct drm_i915_private *dev_priv = engine->i915;
 807         u64 bbaddr;
 808
 809         if (INTEL_GEN(dev_priv) >= 8)
 810                 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
 811                                           RING_BBADDR_UDW(engine->mmio_base));
 812         else
 813                 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
 814
 815         return bbaddr;
 816 }
 817
 818 int intel_engine_stop_cs(struct intel_engine_cs *engine)
 819 {
 820         struct drm_i915_private *dev_priv = engine->i915;
 821         const u32 base = engine->mmio_base;
 822         const i915_reg_t mode = RING_MI_MODE(base);
 823         int err;
 824
 825         if (INTEL_GEN(dev_priv) < 3)
 826                 return -ENODEV;
 827
 828         GEM_TRACE("%s\n", engine->name);
 829
 830         I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
 831
 832         err = 0;
 833         if (__intel_wait_for_register_fw(dev_priv,
 834                                          mode, MODE_IDLE, MODE_IDLE,
 835                                          1000, 0,
 836                                          NULL)) {
 837                 GEM_TRACE("%s: timed out on STOP_RING -> IDLE\n", engine->name);
 838                 err = -ETIMEDOUT;
 839         }
 840
 841         /* A final mmio read to let GPU writes be hopefully flushed to memory */
 842         POSTING_READ_FW(mode);
 843
 844         return err;
 845 }
 846
 847 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
 848 {
 849         struct drm_i915_private *dev_priv = engine->i915;
 850
 851         GEM_TRACE("%s\n", engine->name);
 852
 853         I915_WRITE_FW(RING_MI_MODE(engine->mmio_base),
 854                       _MASKED_BIT_DISABLE(STOP_RING));
 855 }
 856
 857 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 858 {
 859         switch (type) {
 860         case I915_CACHE_NONE: return " uncached";
 861         case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
 862         case I915_CACHE_L3_LLC: return " L3+LLC";
 863         case I915_CACHE_WT: return " WT";
 864         default: return "";
 865         }
 866 }
 867
 868 u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv)
 869 {
 870         const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
 871         u32 mcr_s_ss_select;
 872         u32 slice = fls(sseu->slice_mask);
 873         u32 subslice = fls(sseu->subslice_mask[slice]);
 874
 875         if (IS_GEN(dev_priv, 10))
 876                 mcr_s_ss_select = GEN8_MCR_SLICE(slice) |
 877                                   GEN8_MCR_SUBSLICE(subslice);
 878         else if (INTEL_GEN(dev_priv) >= 11)
 879                 mcr_s_ss_select = GEN11_MCR_SLICE(slice) |
 880                                   GEN11_MCR_SUBSLICE(subslice);
 881         else
 882                 mcr_s_ss_select = 0;
 883
 884         return mcr_s_ss_select;
 885 }
 886
 887 static inline u32
 888 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 889                   int subslice, i915_reg_t reg)
 890 {
 891         u32 mcr_slice_subslice_mask;
 892         u32 mcr_slice_subslice_select;
 893         u32 default_mcr_s_ss_select;
 894         u32 mcr;
 895         u32 ret;
 896         enum forcewake_domains fw_domains;
 897
 898         if (INTEL_GEN(dev_priv) >= 11) {
 899                 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
 900                                           GEN11_MCR_SUBSLICE_MASK;
 901                 mcr_slice_subslice_select = GEN11_MCR_SLICE(slice) |
 902                                             GEN11_MCR_SUBSLICE(subslice);
 903         } else {
 904                 mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
 905                                           GEN8_MCR_SUBSLICE_MASK;
 906                 mcr_slice_subslice_select = GEN8_MCR_SLICE(slice) |
 907                                             GEN8_MCR_SUBSLICE(subslice);
 908         }
 909
 910         default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(dev_priv);
 911
 912         fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
 913                                                     FW_REG_READ);
 914         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
 915                                                      GEN8_MCR_SELECTOR,
 916                                                      FW_REG_READ | FW_REG_WRITE);
 917
 918         spin_lock_irq(&dev_priv->uncore.lock);
 919         intel_uncore_forcewake_get__locked(&dev_priv->uncore, fw_domains);
 920
 921         mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
 922
 923         WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
 924                      default_mcr_s_ss_select);
 925
 926         mcr &= ~mcr_slice_subslice_mask;
 927         mcr |= mcr_slice_subslice_select;
 928         I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
 929
 930         ret = I915_READ_FW(reg);
 931
 932         mcr &= ~mcr_slice_subslice_mask;
 933         mcr |= default_mcr_s_ss_select;
 934
 935         I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
 936
 937         intel_uncore_forcewake_put__locked(&dev_priv->uncore, fw_domains);
 938         spin_unlock_irq(&dev_priv->uncore.lock);
 939
 940         return ret;
 941 }
 942
 943 /* NB: please notice the memset */
 944 void intel_engine_get_instdone(struct intel_engine_cs *engine,
 945                                struct intel_instdone *instdone)
 946 {
 947         struct drm_i915_private *dev_priv = engine->i915;
 948         u32 mmio_base = engine->mmio_base;
 949         int slice;
 950         int subslice;
 951
 952         memset(instdone, 0, sizeof(*instdone));
 953
 954         switch (INTEL_GEN(dev_priv)) {
 955         default:
 956                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 957
 958                 if (engine->id != RCS0)
 959                         break;
 960
 961                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
 962                 for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
 963                         instdone->sampler[slice][subslice] =
 964                                 read_subslice_reg(dev_priv, slice, subslice,
 965                                                   GEN7_SAMPLER_INSTDONE);
 966                         instdone->row[slice][subslice] =
 967                                 read_subslice_reg(dev_priv, slice, subslice,
 968                                                   GEN7_ROW_INSTDONE);
 969                 }
 970                 break;
 971         case 7:
 972                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 973
 974                 if (engine->id != RCS0)
 975                         break;
 976
 977                 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
 978                 instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
 979                 instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
 980
 981                 break;
 982         case 6:
 983         case 5:
 984         case 4:
 985                 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
 986
 987                 if (engine->id == RCS0)
 988                         /* HACK: Using the wrong struct member */
 989                         instdone->slice_common = I915_READ(GEN4_INSTDONE1);
 990                 break;
 991         case 3:
 992         case 2:
 993                 instdone->instdone = I915_READ(GEN2_INSTDONE);
 994                 break;
 995         }
 996 }
 997
 998 static bool ring_is_idle(struct intel_engine_cs *engine)
 999 {
1000         struct drm_i915_private *dev_priv = engine->i915;
1001         intel_wakeref_t wakeref;
1002         bool idle = true;
1003
1004         if (I915_SELFTEST_ONLY(!engine->mmio_base))
1005                 return true;
1006
1007         /* If the whole device is asleep, the engine must be idle */
1008         wakeref = intel_runtime_pm_get_if_in_use(dev_priv);
1009         if (!wakeref)
1010                 return true;
1011
1012         /* First check that no commands are left in the ring */
1013         if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
1014             (I915_READ_TAIL(engine) & TAIL_ADDR))
1015                 idle = false;
1016
1017         /* No bit for gen2, so assume the CS parser is idle */
1018         if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
1019                 idle = false;
1020
1021         intel_runtime_pm_put(dev_priv, wakeref);
1022
1023         return idle;
1024 }
1025
1026 /**
1027  * intel_engine_is_idle() - Report if the engine has finished process all work
1028  * @engine: the intel_engine_cs
1029  *
1030  * Return true if there are no requests pending, nothing left to be submitted
1031  * to hardware, and that the engine is idle.
1032  */
1033 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1034 {
1035         /* More white lies, if wedged, hw state is inconsistent */
1036         if (i915_reset_failed(engine->i915))
1037                 return true;
1038
1039         /* Waiting to drain ELSP? */
1040         if (READ_ONCE(engine->execlists.active)) {
1041                 struct tasklet_struct *t = &engine->execlists.tasklet;
1042
1043                 local_bh_disable();
1044                 if (tasklet_trylock(t)) {
1045                         /* Must wait for any GPU reset in progress. */
1046                         if (__tasklet_is_enabled(t))
1047                                 t->func(t->data);
1048                         tasklet_unlock(t);
1049                 }
1050                 local_bh_enable();
1051
1052                 /* Otherwise flush the tasklet if it was on another cpu */
1053                 tasklet_unlock_wait(t);
1054
1055                 if (READ_ONCE(engine->execlists.active))
1056                         return false;
1057         }
1058
1059         /* ELSP is empty, but there are ready requests? E.g. after reset */
1060         if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
1061                 return false;
1062
1063         /* Ring stopped? */
1064         return ring_is_idle(engine);
1065 }
1066
1067 bool intel_engines_are_idle(struct drm_i915_private *i915)
1068 {
1069         struct intel_engine_cs *engine;
1070         enum intel_engine_id id;
1071
1072         /*
1073          * If the driver is wedged, HW state may be very inconsistent and
1074          * report that it is still busy, even though we have stopped using it.
1075          */
1076         if (i915_reset_failed(i915))
1077                 return true;
1078
1079         /* Already parked (and passed an idleness test); must still be idle */
1080         if (!READ_ONCE(i915->gt.awake))
1081                 return true;
1082
1083         for_each_engine(engine, i915, id) {
1084                 if (!intel_engine_is_idle(engine))
1085                         return false;
1086         }
1087
1088         return true;
1089 }
1090
1091 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1092 {
1093         struct intel_engine_cs *engine;
1094         enum intel_engine_id id;
1095
1096         for_each_engine(engine, i915, id)
1097                 engine->set_default_submission(engine);
1098 }
1099
1100 static bool reset_engines(struct drm_i915_private *i915)
1101 {
1102         if (INTEL_INFO(i915)->gpu_reset_clobbers_display)
1103                 return false;
1104
1105         return intel_gpu_reset(i915, ALL_ENGINES) == 0;
1106 }
1107
1108 /**
1109  * intel_engines_sanitize: called after the GPU has lost power
1110  * @i915: the i915 device
1111  * @force: ignore a failed reset and sanitize engine state anyway
1112  *
1113  * Anytime we reset the GPU, either with an explicit GPU reset or through a
1114  * PCI power cycle, the GPU loses state and we must reset our state tracking
1115  * to match. Note that calling intel_engines_sanitize() if the GPU has not
1116  * been reset results in much confusion!
1117  */
1118 void intel_engines_sanitize(struct drm_i915_private *i915, bool force)
1119 {
1120         struct intel_engine_cs *engine;
1121         enum intel_engine_id id;
1122
1123         GEM_TRACE("\n");
1124
1125         if (!reset_engines(i915) && !force)
1126                 return;
1127
1128         for_each_engine(engine, i915, id)
1129                 intel_engine_reset(engine, false);
1130 }
1131
1132 /**
1133  * intel_engines_park: called when the GT is transitioning from busy->idle
1134  * @i915: the i915 device
1135  *
1136  * The GT is now idle and about to go to sleep (maybe never to wake again?).
1137  * Time for us to tidy and put away our toys (release resources back to the
1138  * system).
1139  */
1140 void intel_engines_park(struct drm_i915_private *i915)
1141 {
1142         struct intel_engine_cs *engine;
1143         enum intel_engine_id id;
1144
1145         for_each_engine(engine, i915, id) {
1146                 /* Flush the residual irq tasklets first. */
1147                 intel_engine_disarm_breadcrumbs(engine);
1148                 tasklet_kill(&engine->execlists.tasklet);
1149
1150                 /*
1151                  * We are committed now to parking the engines, make sure there
1152                  * will be no more interrupts arriving later and the engines
1153                  * are truly idle.
1154                  */
1155                 if (wait_for(intel_engine_is_idle(engine), 10)) {
1156                         struct drm_printer p = drm_debug_printer(__func__);
1157
1158                         dev_err(i915->drm.dev,
1159                                 "%s is not idle before parking\n",
1160                                 engine->name);
1161                         intel_engine_dump(engine, &p, NULL);
1162                 }
1163
1164                 /* Must be reset upon idling, or we may miss the busy wakeup. */
1165                 GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN);
1166
1167                 if (engine->park)
1168                         engine->park(engine);
1169
1170                 if (engine->pinned_default_state) {
1171                         i915_gem_object_unpin_map(engine->default_state);
1172                         engine->pinned_default_state = NULL;
1173                 }
1174
1175                 i915_gem_batch_pool_fini(&engine->batch_pool);
1176                 engine->execlists.no_priolist = false;
1177         }
1178
1179         i915->gt.active_engines = 0;
1180 }
1181
1182 /**
1183  * intel_engines_unpark: called when the GT is transitioning from idle->busy
1184  * @i915: the i915 device
1185  *
1186  * The GT was idle and now about to fire up with some new user requests.
1187  */
1188 void intel_engines_unpark(struct drm_i915_private *i915)
1189 {
1190         struct intel_engine_cs *engine;
1191         enum intel_engine_id id;
1192
1193         for_each_engine(engine, i915, id) {
1194                 void *map;
1195
1196                 /* Pin the default state for fast resets from atomic context. */
1197                 map = NULL;
1198                 if (engine->default_state)
1199                         map = i915_gem_object_pin_map(engine->default_state,
1200                                                       I915_MAP_WB);
1201                 if (!IS_ERR_OR_NULL(map))
1202                         engine->pinned_default_state = map;
1203
1204                 if (engine->unpark)
1205                         engine->unpark(engine);
1206
1207                 intel_engine_init_hangcheck(engine);
1208         }
1209 }
1210
1211 /**
1212  * intel_engine_lost_context: called when the GPU is reset into unknown state
1213  * @engine: the engine
1214  *
1215  * We have either reset the GPU or otherwise about to lose state tracking of
1216  * the current GPU logical state (e.g. suspend). On next use, it is therefore
1217  * imperative that we make no presumptions about the current state and load
1218  * from scratch.
1219  */
1220 void intel_engine_lost_context(struct intel_engine_cs *engine)
1221 {
1222         struct intel_context *ce;
1223
1224         lockdep_assert_held(&engine->i915->drm.struct_mutex);
1225
1226         ce = fetch_and_zero(&engine->last_retired_context);
1227         if (ce)
1228                 intel_context_unpin(ce);
1229 }
1230
1231 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1232 {
1233         switch (INTEL_GEN(engine->i915)) {
1234         case 2:
1235                 return false; /* uses physical not virtual addresses */
1236         case 3:
1237                 /* maybe only uses physical not virtual addresses */
1238                 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1239         case 6:
1240                 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1241         default:
1242                 return true;
1243         }
1244 }
1245
1246 unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1247 {
1248         struct intel_engine_cs *engine;
1249         enum intel_engine_id id;
1250         unsigned int which;
1251
1252         which = 0;
1253         for_each_engine(engine, i915, id)
1254                 if (engine->default_state)
1255                         which |= BIT(engine->uabi_class);
1256
1257         return which;
1258 }
1259
1260 static int print_sched_attr(struct drm_i915_private *i915,
1261                             const struct i915_sched_attr *attr,
1262                             char *buf, int x, int len)
1263 {
1264         if (attr->priority == I915_PRIORITY_INVALID)
1265                 return x;
1266
1267         x += snprintf(buf + x, len - x,
1268                       " prio=%d", attr->priority);
1269
1270         return x;
1271 }
1272
1273 static void print_request(struct drm_printer *m,
1274                           struct i915_request *rq,
1275                           const char *prefix)
1276 {
1277         const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
1278         char buf[80] = "";
1279         int x = 0;
1280
1281         x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
1282
1283         drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
1284                    prefix,
1285                    rq->fence.context, rq->fence.seqno,
1286                    i915_request_completed(rq) ? "!" :
1287                    i915_request_started(rq) ? "*" :
1288                    "",
1289                    test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1290                             &rq->fence.flags) ?  "+" : "",
1291                    buf,
1292                    jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1293                    name);
1294 }
1295
1296 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1297 {
1298         const size_t rowsize = 8 * sizeof(u32);
1299         const void *prev = NULL;
1300         bool skip = false;
1301         size_t pos;
1302
1303         for (pos = 0; pos < len; pos += rowsize) {
1304                 char line[128];
1305
1306                 if (prev && !memcmp(prev, buf + pos, rowsize)) {
1307                         if (!skip) {
1308                                 drm_printf(m, "*\n");
1309                                 skip = true;
1310                         }
1311                         continue;
1312                 }
1313
1314                 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1315                                                 rowsize, sizeof(u32),
1316                                                 line, sizeof(line),
1317                                                 false) >= sizeof(line));
1318                 drm_printf(m, "[%04zx] %s\n", pos, line);
1319
1320                 prev = buf + pos;
1321                 skip = false;
1322         }
1323 }
1324
1325 static void intel_engine_print_registers(const struct intel_engine_cs *engine,
1326                                          struct drm_printer *m)
1327 {
1328         struct drm_i915_private *dev_priv = engine->i915;
1329         const struct intel_engine_execlists * const execlists =
1330                 &engine->execlists;
1331         u64 addr;
1332
1333         if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
1334                 drm_printf(m, "\tCCID: 0x%08x\n", I915_READ(CCID));
1335         drm_printf(m, "\tRING_START: 0x%08x\n",
1336                    I915_READ(RING_START(engine->mmio_base)));
1337         drm_printf(m, "\tRING_HEAD:  0x%08x\n",
1338                    I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR);
1339         drm_printf(m, "\tRING_TAIL:  0x%08x\n",
1340                    I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR);
1341         drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
1342                    I915_READ(RING_CTL(engine->mmio_base)),
1343                    I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1344         if (INTEL_GEN(engine->i915) > 2) {
1345                 drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
1346                            I915_READ(RING_MI_MODE(engine->mmio_base)),
1347                            I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
1348         }
1349
1350         if (INTEL_GEN(dev_priv) >= 6) {
1351                 drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine));
1352         }
1353
1354         addr = intel_engine_get_active_head(engine);
1355         drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1356                    upper_32_bits(addr), lower_32_bits(addr));
1357         addr = intel_engine_get_last_batch_head(engine);
1358         drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1359                    upper_32_bits(addr), lower_32_bits(addr));
1360         if (INTEL_GEN(dev_priv) >= 8)
1361                 addr = I915_READ64_2x32(RING_DMA_FADD(engine->mmio_base),
1362                                         RING_DMA_FADD_UDW(engine->mmio_base));
1363         else if (INTEL_GEN(dev_priv) >= 4)
1364                 addr = I915_READ(RING_DMA_FADD(engine->mmio_base));
1365         else
1366                 addr = I915_READ(DMA_FADD_I8XX);
1367         drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1368                    upper_32_bits(addr), lower_32_bits(addr));
1369         if (INTEL_GEN(dev_priv) >= 4) {
1370                 drm_printf(m, "\tIPEIR: 0x%08x\n",
1371                            I915_READ(RING_IPEIR(engine->mmio_base)));
1372                 drm_printf(m, "\tIPEHR: 0x%08x\n",
1373                            I915_READ(RING_IPEHR(engine->mmio_base)));
1374         } else {
1375                 drm_printf(m, "\tIPEIR: 0x%08x\n", I915_READ(IPEIR));
1376                 drm_printf(m, "\tIPEHR: 0x%08x\n", I915_READ(IPEHR));
1377         }
1378
1379         if (HAS_EXECLISTS(dev_priv)) {
1380                 const u32 *hws =
1381                         &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1382                 unsigned int idx;
1383                 u8 read, write;
1384
1385                 drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
1386                            I915_READ(RING_EXECLIST_STATUS_LO(engine)),
1387                            I915_READ(RING_EXECLIST_STATUS_HI(engine)));
1388
1389                 read = execlists->csb_head;
1390                 write = READ_ONCE(*execlists->csb_write);
1391
1392                 drm_printf(m, "\tExeclist CSB read %d, write %d [mmio:%d], tasklet queued? %s (%s)\n",
1393                            read, write,
1394                            GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(engine))),
1395                            yesno(test_bit(TASKLET_STATE_SCHED,
1396                                           &engine->execlists.tasklet.state)),
1397                            enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
1398                 if (read >= GEN8_CSB_ENTRIES)
1399                         read = 0;
1400                 if (write >= GEN8_CSB_ENTRIES)
1401                         write = 0;
1402                 if (read > write)
1403                         write += GEN8_CSB_ENTRIES;
1404                 while (read < write) {
1405                         idx = ++read % GEN8_CSB_ENTRIES;
1406                         drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [mmio:0x%08x], context: %d [mmio:%d]\n",
1407                                    idx,
1408                                    hws[idx * 2],
1409                                    I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
1410                                    hws[idx * 2 + 1],
1411                                    I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
1412                 }
1413
1414                 rcu_read_lock();
1415                 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1416                         struct i915_request *rq;
1417                         unsigned int count;
1418
1419                         rq = port_unpack(&execlists->port[idx], &count);
1420                         if (rq) {
1421                                 char hdr[80];
1422
1423                                 snprintf(hdr, sizeof(hdr),
1424                                          "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
1425                                          idx, count,
1426                                          i915_ggtt_offset(rq->ring->vma),
1427                                          rq->timeline->hwsp_offset,
1428                                          hwsp_seqno(rq));
1429                                 print_request(m, rq, hdr);
1430                         } else {
1431                                 drm_printf(m, "\t\tELSP[%d] idle\n", idx);
1432                         }
1433                 }
1434                 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
1435                 rcu_read_unlock();
1436         } else if (INTEL_GEN(dev_priv) > 6) {
1437                 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1438                            I915_READ(RING_PP_DIR_BASE(engine)));
1439                 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1440                            I915_READ(RING_PP_DIR_BASE_READ(engine)));
1441                 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1442                            I915_READ(RING_PP_DIR_DCLV(engine)));
1443         }
1444 }
1445
1446 static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1447 {
1448         void *ring;
1449         int size;
1450
1451         drm_printf(m,
1452                    "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1453                    rq->head, rq->postfix, rq->tail,
1454                    rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1455                    rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1456
1457         size = rq->tail - rq->head;
1458         if (rq->tail < rq->head)
1459                 size += rq->ring->size;
1460
1461         ring = kmalloc(size, GFP_ATOMIC);
1462         if (ring) {
1463                 const void *vaddr = rq->ring->vaddr;
1464                 unsigned int head = rq->head;
1465                 unsigned int len = 0;
1466
1467                 if (rq->tail < head) {
1468                         len = rq->ring->size - head;
1469                         memcpy(ring, vaddr + head, len);
1470                         head = 0;
1471                 }
1472                 memcpy(ring + len, vaddr + head, size - len);
1473
1474                 hexdump(m, ring, size);
1475                 kfree(ring);
1476         }
1477 }
1478
1479 void intel_engine_dump(struct intel_engine_cs *engine,
1480                        struct drm_printer *m,
1481                        const char *header, ...)
1482 {
1483         struct i915_gpu_error * const error = &engine->i915->gpu_error;
1484         struct i915_request *rq;
1485         intel_wakeref_t wakeref;
1486
1487         if (header) {
1488                 va_list ap;
1489
1490                 va_start(ap, header);
1491                 drm_vprintf(m, header, &ap);
1492                 va_end(ap);
1493         }
1494
1495         if (i915_reset_failed(engine->i915))
1496                 drm_printf(m, "*** WEDGED ***\n");
1497
1498         drm_printf(m, "\tHangcheck %x:%x [%d ms]\n",
1499                    engine->hangcheck.last_seqno,
1500                    engine->hangcheck.next_seqno,
1501                    jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
1502         drm_printf(m, "\tReset count: %d (global %d)\n",
1503                    i915_reset_engine_count(error, engine),
1504                    i915_reset_count(error));
1505
1506         rcu_read_lock();
1507
1508         drm_printf(m, "\tRequests:\n");
1509
1510         rq = list_first_entry(&engine->timeline.requests,
1511                               struct i915_request, link);
1512         if (&rq->link != &engine->timeline.requests)
1513                 print_request(m, rq, "\t\tfirst  ");
1514
1515         rq = list_last_entry(&engine->timeline.requests,
1516                              struct i915_request, link);
1517         if (&rq->link != &engine->timeline.requests)
1518                 print_request(m, rq, "\t\tlast   ");
1519
1520         rq = intel_engine_find_active_request(engine);
1521         if (rq) {
1522                 print_request(m, rq, "\t\tactive ");
1523
1524                 drm_printf(m, "\t\tring->start:  0x%08x\n",
1525                            i915_ggtt_offset(rq->ring->vma));
1526                 drm_printf(m, "\t\tring->head:   0x%08x\n",
1527                            rq->ring->head);
1528                 drm_printf(m, "\t\tring->tail:   0x%08x\n",
1529                            rq->ring->tail);
1530                 drm_printf(m, "\t\tring->emit:   0x%08x\n",
1531                            rq->ring->emit);
1532                 drm_printf(m, "\t\tring->space:  0x%08x\n",
1533                            rq->ring->space);
1534                 drm_printf(m, "\t\tring->hwsp:   0x%08x\n",
1535                            rq->timeline->hwsp_offset);
1536
1537                 print_request_ring(m, rq);
1538         }
1539
1540         rcu_read_unlock();
1541
1542         wakeref = intel_runtime_pm_get_if_in_use(engine->i915);
1543         if (wakeref) {
1544                 intel_engine_print_registers(engine, m);
1545                 intel_runtime_pm_put(engine->i915, wakeref);
1546         } else {
1547                 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
1548         }
1549
1550         intel_execlists_show_requests(engine, m, print_request, 8);
1551
1552         drm_printf(m, "HWSP:\n");
1553         hexdump(m, engine->status_page.addr, PAGE_SIZE);
1554
1555         drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
1556
1557         intel_engine_print_breadcrumbs(engine, m);
1558 }
1559
1560 static u8 user_class_map[] = {
1561         [I915_ENGINE_CLASS_RENDER] = RENDER_CLASS,
1562         [I915_ENGINE_CLASS_COPY] = COPY_ENGINE_CLASS,
1563         [I915_ENGINE_CLASS_VIDEO] = VIDEO_DECODE_CLASS,
1564         [I915_ENGINE_CLASS_VIDEO_ENHANCE] = VIDEO_ENHANCEMENT_CLASS,
1565 };
1566
1567 struct intel_engine_cs *
1568 intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance)
1569 {
1570         if (class >= ARRAY_SIZE(user_class_map))
1571                 return NULL;
1572
1573         class = user_class_map[class];
1574
1575         GEM_BUG_ON(class > MAX_ENGINE_CLASS);
1576
1577         if (instance > MAX_ENGINE_INSTANCE)
1578                 return NULL;
1579
1580         return i915->engine_class[class][instance];
1581 }
1582
1583 /**
1584  * intel_enable_engine_stats() - Enable engine busy tracking on engine
1585  * @engine: engine to enable stats collection
1586  *
1587  * Start collecting the engine busyness data for @engine.
1588  *
1589  * Returns 0 on success or a negative error code.
1590  */
1591 int intel_enable_engine_stats(struct intel_engine_cs *engine)
1592 {
1593         struct intel_engine_execlists *execlists = &engine->execlists;
1594         unsigned long flags;
1595         int err = 0;
1596
1597         if (!intel_engine_supports_stats(engine))
1598                 return -ENODEV;
1599
1600         spin_lock_irqsave(&engine->timeline.lock, flags);
1601         write_seqlock(&engine->stats.lock);
1602
1603         if (unlikely(engine->stats.enabled == ~0)) {
1604                 err = -EBUSY;
1605                 goto unlock;
1606         }
1607
1608         if (engine->stats.enabled++ == 0) {
1609                 const struct execlist_port *port = execlists->port;
1610                 unsigned int num_ports = execlists_num_ports(execlists);
1611
1612                 engine->stats.enabled_at = ktime_get();
1613
1614                 /* XXX submission method oblivious? */
1615                 while (num_ports-- && port_isset(port)) {
1616                         engine->stats.active++;
1617                         port++;
1618                 }
1619
1620                 if (engine->stats.active)
1621                         engine->stats.start = engine->stats.enabled_at;
1622         }
1623
1624 unlock:
1625         write_sequnlock(&engine->stats.lock);
1626         spin_unlock_irqrestore(&engine->timeline.lock, flags);
1627
1628         return err;
1629 }
1630
1631 static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
1632 {
1633         ktime_t total = engine->stats.total;
1634
1635         /*
1636          * If the engine is executing something at the moment
1637          * add it to the total.
1638          */
1639         if (engine->stats.active)
1640                 total = ktime_add(total,
1641                                   ktime_sub(ktime_get(), engine->stats.start));
1642
1643         return total;
1644 }
1645
1646 /**
1647  * intel_engine_get_busy_time() - Return current accumulated engine busyness
1648  * @engine: engine to report on
1649  *
1650  * Returns accumulated time @engine was busy since engine stats were enabled.
1651  */
1652 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
1653 {
1654         unsigned int seq;
1655         ktime_t total;
1656
1657         do {
1658                 seq = read_seqbegin(&engine->stats.lock);
1659                 total = __intel_engine_get_busy_time(engine);
1660         } while (read_seqretry(&engine->stats.lock, seq));
1661
1662         return total;
1663 }
1664
1665 /**
1666  * intel_disable_engine_stats() - Disable engine busy tracking on engine
1667  * @engine: engine to disable stats collection
1668  *
1669  * Stops collecting the engine busyness data for @engine.
1670  */
1671 void intel_disable_engine_stats(struct intel_engine_cs *engine)
1672 {
1673         unsigned long flags;
1674
1675         if (!intel_engine_supports_stats(engine))
1676                 return;
1677
1678         write_seqlock_irqsave(&engine->stats.lock, flags);
1679         WARN_ON_ONCE(engine->stats.enabled == 0);
1680         if (--engine->stats.enabled == 0) {
1681                 engine->stats.total = __intel_engine_get_busy_time(engine);
1682                 engine->stats.active = 0;
1683         }
1684         write_sequnlock_irqrestore(&engine->stats.lock, flags);
1685 }
1686
1687 static bool match_ring(struct i915_request *rq)
1688 {
1689         struct drm_i915_private *dev_priv = rq->i915;
1690         u32 ring = I915_READ(RING_START(rq->engine->mmio_base));
1691
1692         return ring == i915_ggtt_offset(rq->ring->vma);
1693 }
1694
1695 struct i915_request *
1696 intel_engine_find_active_request(struct intel_engine_cs *engine)
1697 {
1698         struct i915_request *request, *active = NULL;
1699         unsigned long flags;
1700
1701         /*
1702          * We are called by the error capture, reset and to dump engine
1703          * state at random points in time. In particular, note that neither is
1704          * crucially ordered with an interrupt. After a hang, the GPU is dead
1705          * and we assume that no more writes can happen (we waited long enough
1706          * for all writes that were in transaction to be flushed) - adding an
1707          * extra delay for a recent interrupt is pointless. Hence, we do
1708          * not need an engine->irq_seqno_barrier() before the seqno reads.
1709          * At all other times, we must assume the GPU is still running, but
1710          * we only care about the snapshot of this moment.
1711          */
1712         spin_lock_irqsave(&engine->timeline.lock, flags);
1713         list_for_each_entry(request, &engine->timeline.requests, link) {
1714                 if (i915_request_completed(request))
1715                         continue;
1716
1717                 if (!i915_request_started(request))
1718                         break;
1719
1720                 /* More than one preemptible request may match! */
1721                 if (!match_ring(request))
1722                         break;
1723
1724                 active = request;
1725                 break;
1726         }
1727         spin_unlock_irqrestore(&engine->timeline.lock, flags);
1728
1729         return active;
1730 }
1731
1732 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1733 #include "selftests/mock_engine.c"
1734 #include "selftests/intel_engine_cs.c"
1735 #endif