1 /* SPDX-License-Identifier: MIT */
3 * Copyright © 2019 Intel Corporation
6 #ifndef __INTEL_ENGINE_TYPES__
7 #define __INTEL_ENGINE_TYPES__
9 #include <linux/average.h>
10 #include <linux/hashtable.h>
11 #include <linux/irq_work.h>
12 #include <linux/kref.h>
13 #include <linux/list.h>
14 #include <linux/llist.h>
15 #include <linux/rbtree.h>
16 #include <linux/timer.h>
17 #include <linux/types.h>
18 #include <linux/workqueue.h>
22 #include "i915_priolist_types.h"
23 #include "i915_selftest.h"
24 #include "intel_sseu.h"
25 #include "intel_timeline_types.h"
26 #include "intel_uncore.h"
27 #include "intel_wakeref.h"
28 #include "intel_workarounds_types.h"
30 /* HW Engine class + instance */
31 #define RENDER_CLASS 0
32 #define VIDEO_DECODE_CLASS 1
33 #define VIDEO_ENHANCEMENT_CLASS 2
34 #define COPY_ENGINE_CLASS 3
36 #define MAX_ENGINE_CLASS 4
37 #define MAX_ENGINE_INSTANCE 7
39 #define I915_MAX_SLICES 3
40 #define I915_MAX_SUBSLICES 8
42 #define I915_CMD_HASH_ORDER 9
45 struct drm_i915_gem_object;
46 struct drm_i915_reg_table;
47 struct i915_gem_context;
49 struct i915_sched_attr;
50 struct i915_sched_engine;
54 struct intel_breadcrumbs;
56 typedef u32 intel_engine_mask_t;
57 #define ALL_ENGINES ((intel_engine_mask_t)~0ul)
59 struct intel_hw_status_page {
60 struct list_head timelines;
65 struct intel_instdone {
67 /* The following exist only in the RCS engine */
69 u32 slice_common_extra[2];
70 u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
71 u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
75 * we use a single page to load ctx workarounds so all of these
76 * values are referred in terms of dwords
78 * struct i915_wa_ctx_bb:
79 * offset: specifies batch starting position, also helpful in case
80 * if we want to have multiple batches at different offsets based on
81 * some criteria. It is not a requirement at the moment but provides
82 * an option for future use.
83 * size: size of the batch in DWORDS
85 struct i915_ctx_workarounds {
86 struct i915_wa_ctx_bb {
89 } indirect_ctx, per_ctx;
93 #define I915_MAX_VCS 8
94 #define I915_MAX_VECS 4
97 * Engine IDs definitions.
98 * Keep instances of the same type engine together.
100 enum intel_engine_id {
111 #define _VCS(n) (VCS0 + (n))
116 #define _VECS(n) (VECS0 + (n))
118 #define INVALID_ENGINE ((enum intel_engine_id)-1)
121 /* A simple estimator for the round-trip latency of an engine */
122 DECLARE_EWMA(_engine_latency, 6, 4)
124 struct st_preempt_hang {
125 struct completion completion;
130 * struct intel_engine_execlists - execlist submission queue and port state
132 * The struct intel_engine_execlists represents the combined logical state of
133 * driver and the hardware state for execlist mode of submission.
135 struct intel_engine_execlists {
137 * @timer: kick the current context if its timeslice expires
139 struct timer_list timer;
142 * @preempt: reset the current context if it fails to give way
144 struct timer_list preempt;
147 * @ccid: identifier for contexts submitted to this engine
152 * @yield: CCID at the time of the last semaphore-wait interrupt.
154 * Instead of leaving a semaphore busy-spinning on an engine, we would
155 * like to switch to another ready context, i.e. yielding the semaphore
161 * @error_interrupt: CS Master EIR
163 * The CS generates an interrupt when it detects an error. We capture
164 * the first error interrupt, record the EIR and schedule the tasklet.
165 * In the tasklet, we process the pending CS events to ensure we have
166 * the guilty request, and then reset the engine.
168 * Low 16b are used by HW, with the upper 16b used as the enabling mask.
169 * Reserve the upper 16b for tracking internal errors.
172 #define ERROR_CSB BIT(31)
173 #define ERROR_PREEMPT BIT(30)
176 * @reset_ccid: Active CCID [EXECLISTS_STATUS_HI] at the time of reset
181 * @submit_reg: gen-specific execlist submission register
182 * set to the ExecList Submission Port (elsp) register pre-Gen11 and to
183 * the ExecList Submission Queue Contents register array for Gen11+
185 u32 __iomem *submit_reg;
188 * @ctrl_reg: the enhanced execlists control register, used to load the
189 * submit queue on the HW and to request preemptions to idle
191 u32 __iomem *ctrl_reg;
193 #define EXECLIST_MAX_PORTS 2
195 * @active: the currently known context executing on HW
197 struct i915_request * const *active;
199 * @inflight: the set of contexts submitted and acknowleged by HW
201 * The set of inflight contexts is managed by reading CS events
202 * from the HW. On a context-switch event (not preemption), we
203 * know the HW has transitioned from port0 to port1, and we
204 * advance our inflight/active tracking accordingly.
206 struct i915_request *inflight[EXECLIST_MAX_PORTS + 1 /* sentinel */];
208 * @pending: the next set of contexts submitted to ELSP
210 * We store the array of contexts that we submit to HW (via ELSP) and
211 * promote them to the inflight array once HW has signaled the
212 * preemption or idle-to-active event.
214 struct i915_request *pending[EXECLIST_MAX_PORTS + 1];
217 * @port_mask: number of execlist ports - 1
219 unsigned int port_mask;
222 * @virtual: Queue of requets on a virtual engine, sorted by priority.
223 * Each RB entry is a struct i915_priolist containing a list of requests
224 * of the same priority.
226 struct rb_root_cached virtual;
229 * @csb_write: control register for Context Switch buffer
231 * Note this register may be either mmio or HWSP shadow.
236 * @csb_status: status array for Context Switch buffer
238 * Note these register may be either mmio or HWSP shadow.
243 * @csb_size: context status buffer FIFO size
248 * @csb_head: context status buffer head
252 I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;)
255 #define INTEL_ENGINE_CS_MAX_NAME 8
257 struct intel_engine_cs {
258 struct drm_i915_private *i915;
260 struct intel_uncore *uncore;
261 char name[INTEL_ENGINE_CS_MAX_NAME];
263 enum intel_engine_id id;
264 enum intel_engine_id legacy_idx;
268 intel_engine_mask_t mask;
276 u32 uabi_capabilities;
281 * Some w/a require forcewake to be held (which prevents RC6) while
282 * a particular engine is active. If so, we set fw_domain to which
283 * domains need to be held for the duration of request activity,
284 * and 0 if none. We try to limit the duration of the hold as much
287 enum forcewake_domains fw_domain;
288 unsigned int fw_active;
290 unsigned long context_tag;
292 struct rb_node uabi_node;
294 struct intel_sseu sseu;
296 struct i915_sched_engine *sched_engine;
298 /* keep a request in reserve for a [pm] barrier under oom */
299 struct i915_request *request_pool;
301 struct intel_context *hung_ce;
303 struct llist_head barrier_tasks;
305 struct intel_context *kernel_context; /* pinned */
307 intel_engine_mask_t saturated; /* submitting semaphores too late? */
310 struct delayed_work work;
311 struct i915_request *systole;
312 unsigned long blocked;
315 unsigned long serial;
317 unsigned long wakeref_serial;
318 struct intel_wakeref wakeref;
319 struct file *default_state;
322 struct intel_ring *ring;
323 struct intel_timeline *timeline;
327 * We track the average duration of the idle pulse on parking the
328 * engine to keep an estimate of the how the fast the engine is
329 * under ideal conditions.
331 struct ewma__engine_latency latency;
333 /* Keep track of all the seqno used, a trail of breadcrumbs */
334 struct intel_breadcrumbs *breadcrumbs;
336 struct intel_engine_pmu {
338 * @enable: Bitmask of enable sample events on this engine.
340 * Bits correspond to sample event types, for instance
341 * I915_SAMPLE_QUEUED is bit 0 etc.
345 * @enable_count: Reference count for the enabled samplers.
347 * Index number corresponds to @enum drm_i915_pmu_engine_sample.
349 unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT];
351 * @sample: Counter values for sampling events.
353 * Our internal timer stores the current counters in this field.
355 * Index number corresponds to @enum drm_i915_pmu_engine_sample.
357 struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT];
360 struct intel_hw_status_page status_page;
361 struct i915_ctx_workarounds wa_ctx;
362 struct i915_wa_list ctx_wa_list;
363 struct i915_wa_list wa_list;
364 struct i915_wa_list whitelist;
366 u32 irq_keep_mask; /* always keep these interrupts */
367 u32 irq_enable_mask; /* bitmask to enable ring interrupt */
368 void (*irq_enable)(struct intel_engine_cs *engine);
369 void (*irq_disable)(struct intel_engine_cs *engine);
370 void (*irq_handler)(struct intel_engine_cs *engine, u16 iir);
372 void (*sanitize)(struct intel_engine_cs *engine);
373 int (*resume)(struct intel_engine_cs *engine);
376 void (*prepare)(struct intel_engine_cs *engine);
378 void (*rewind)(struct intel_engine_cs *engine, bool stalled);
379 void (*cancel)(struct intel_engine_cs *engine);
381 void (*finish)(struct intel_engine_cs *engine);
384 void (*park)(struct intel_engine_cs *engine);
385 void (*unpark)(struct intel_engine_cs *engine);
387 void (*bump_serial)(struct intel_engine_cs *engine);
389 void (*set_default_submission)(struct intel_engine_cs *engine);
391 const struct intel_context_ops *cops;
393 int (*request_alloc)(struct i915_request *rq);
395 int (*emit_flush)(struct i915_request *request, u32 mode);
396 #define EMIT_INVALIDATE BIT(0)
397 #define EMIT_FLUSH BIT(1)
398 #define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH)
399 int (*emit_bb_start)(struct i915_request *rq,
400 u64 offset, u32 length,
401 unsigned int dispatch_flags);
402 #define I915_DISPATCH_SECURE BIT(0)
403 #define I915_DISPATCH_PINNED BIT(1)
404 int (*emit_init_breadcrumb)(struct i915_request *rq);
405 u32 *(*emit_fini_breadcrumb)(struct i915_request *rq,
407 unsigned int emit_fini_breadcrumb_dw;
409 /* Pass the request to the hardware queue (e.g. directly into
410 * the legacy ringbuffer or to the end of an execlist).
412 * This is called from an atomic context with irqs disabled; must
415 void (*submit_request)(struct i915_request *rq);
417 void (*release)(struct intel_engine_cs *engine);
420 * Add / remove request from engine active tracking
422 void (*add_active_request)(struct i915_request *rq);
423 void (*remove_active_request)(struct i915_request *rq);
425 struct intel_engine_execlists execlists;
428 * Keep track of completed timelines on this engine for early
429 * retirement with the goal of quickly enabling powersaving as
430 * soon as the engine is idle.
432 struct intel_timeline *retire;
433 struct work_struct retire_work;
435 /* status_notifier: list of callbacks for context-switch changes */
436 struct atomic_notifier_head context_status_notifier;
438 #define I915_ENGINE_USING_CMD_PARSER BIT(0)
439 #define I915_ENGINE_SUPPORTS_STATS BIT(1)
440 #define I915_ENGINE_HAS_PREEMPTION BIT(2)
441 #define I915_ENGINE_HAS_SEMAPHORES BIT(3)
442 #define I915_ENGINE_HAS_TIMESLICES BIT(4)
443 #define I915_ENGINE_IS_VIRTUAL BIT(5)
444 #define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6)
445 #define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7)
446 #define I915_ENGINE_WANT_FORCED_PREEMPTION BIT(8)
450 * Table of commands the command parser needs to know about
453 DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
456 * Table of registers allowed in commands that read/write registers.
458 const struct drm_i915_reg_table *reg_tables;
462 * Returns the bitmask for the length field of the specified command.
463 * Return 0 for an unrecognized/invalid command.
465 * If the command parser finds an entry for a command in the engine's
466 * cmd_tables, it gets the command's length based on the table entry.
467 * If not, it calls this function to determine the per-engine length
468 * field encoding for the command (i.e. different opcode ranges use
469 * certain bits to encode the command length in the header).
471 u32 (*get_cmd_length_mask)(u32 cmd_header);
475 * @active: Number of contexts currently scheduled in.
480 * @lock: Lock protecting the below fields.
485 * @total: Total time this engine was busy.
487 * Accumulated time not counting the most recent block in cases
488 * where engine is currently busy (active > 0).
493 * @start: Timestamp of the last idle to active transition.
495 * Idle is defined as active == 0, active is active > 0.
500 * @rps: Utilisation at last RPS sampling.
506 unsigned long heartbeat_interval_ms;
507 unsigned long max_busywait_duration_ns;
508 unsigned long preempt_timeout_ms;
509 unsigned long stop_timeout_ms;
510 unsigned long timeslice_duration_ms;
513 I915_SELFTEST_DECLARE(struct fault_attr reset_timeout);
517 intel_engine_using_cmd_parser(const struct intel_engine_cs *engine)
519 return engine->flags & I915_ENGINE_USING_CMD_PARSER;
523 intel_engine_requires_cmd_parser(const struct intel_engine_cs *engine)
525 return engine->flags & I915_ENGINE_REQUIRES_CMD_PARSER;
529 intel_engine_supports_stats(const struct intel_engine_cs *engine)
531 return engine->flags & I915_ENGINE_SUPPORTS_STATS;
535 intel_engine_has_preemption(const struct intel_engine_cs *engine)
537 return engine->flags & I915_ENGINE_HAS_PREEMPTION;
541 intel_engine_has_semaphores(const struct intel_engine_cs *engine)
543 return engine->flags & I915_ENGINE_HAS_SEMAPHORES;
547 intel_engine_has_timeslices(const struct intel_engine_cs *engine)
549 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
552 return engine->flags & I915_ENGINE_HAS_TIMESLICES;
556 intel_engine_is_virtual(const struct intel_engine_cs *engine)
558 return engine->flags & I915_ENGINE_IS_VIRTUAL;
562 intel_engine_has_relative_mmio(const struct intel_engine_cs * const engine)
564 return engine->flags & I915_ENGINE_HAS_RELATIVE_MMIO;
567 #define instdone_has_slice(dev_priv___, sseu___, slice___) \
568 ((GRAPHICS_VER(dev_priv___) == 7 ? 1 : ((sseu___)->slice_mask)) & BIT(slice___))
570 #define instdone_has_subslice(dev_priv__, sseu__, slice__, subslice__) \
571 (GRAPHICS_VER(dev_priv__) == 7 ? (1 & BIT(subslice__)) : \
572 intel_sseu_has_subslice(sseu__, 0, subslice__))
574 #define for_each_instdone_slice_subslice(dev_priv_, sseu_, slice_, subslice_) \
575 for ((slice_) = 0, (subslice_) = 0; (slice_) < I915_MAX_SLICES; \
576 (subslice_) = ((subslice_) + 1) % I915_MAX_SUBSLICES, \
577 (slice_) += ((subslice_) == 0)) \
578 for_each_if((instdone_has_slice(dev_priv_, sseu_, slice_)) && \
579 (instdone_has_subslice(dev_priv_, sseu_, slice_, \
581 #endif /* __INTEL_ENGINE_TYPES_H__ */