drivers/gpu/drm/i915/gt/uc/intel_guc.h

   1 /* SPDX-License-Identifier: MIT */
   2 /*
   3  * Copyright © 2014-2019 Intel Corporation
   4  */
   5
   6 #ifndef _INTEL_GUC_H_
   7 #define _INTEL_GUC_H_
   8
   9 #include <linux/delay.h>
  10 #include <linux/iosys-map.h>
  11 #include <linux/xarray.h>
  12
  13 #include "intel_guc_ct.h"
  14 #include "intel_guc_fw.h"
  15 #include "intel_guc_fwif.h"
  16 #include "intel_guc_log.h"
  17 #include "intel_guc_reg.h"
  18 #include "intel_guc_slpc_types.h"
  19 #include "intel_uc_fw.h"
  20 #include "intel_uncore.h"
  21 #include "i915_utils.h"
  22 #include "i915_vma.h"
  23
  24 struct __guc_ads_blob;
  25 struct intel_guc_state_capture;
  26
  27 /**
  28  * struct intel_guc - Top level structure of GuC.
  29  *
  30  * It handles firmware loading and manages client pool. intel_guc owns an
  31  * i915_sched_engine for submission.
  32  */
  33 struct intel_guc {
  34         /** @fw: the GuC firmware */
  35         struct intel_uc_fw fw;
  36         /** @log: sub-structure containing GuC log related data and objects */
  37         struct intel_guc_log log;
  38         /** @ct: the command transport communication channel */
  39         struct intel_guc_ct ct;
  40         /** @slpc: sub-structure containing SLPC related data and objects */
  41         struct intel_guc_slpc slpc;
  42         /** @capture: the error-state-capture module's data and objects */
  43         struct intel_guc_state_capture *capture;
  44
  45         /** @dbgfs_node: debugfs node */
  46         struct dentry *dbgfs_node;
  47
  48         /** @sched_engine: Global engine used to submit requests to GuC */
  49         struct i915_sched_engine *sched_engine;
  50         /**
  51          * @stalled_request: if GuC can't process a request for any reason, we
  52          * save it until GuC restarts processing. No other request can be
  53          * submitted until the stalled request is processed.
  54          */
  55         struct i915_request *stalled_request;
  56         /**
  57          * @submission_stall_reason: reason why submission is stalled
  58          */
  59         enum {
  60                 STALL_NONE,
  61                 STALL_REGISTER_CONTEXT,
  62                 STALL_MOVE_LRC_TAIL,
  63                 STALL_ADD_REQUEST,
  64         } submission_stall_reason;
  65
  66         /* intel_guc_recv interrupt related state */
  67         /** @irq_lock: protects GuC irq state */
  68         spinlock_t irq_lock;
  69         /**
  70          * @msg_enabled_mask: mask of events that are processed when receiving
  71          * an INTEL_GUC_ACTION_DEFAULT G2H message.
  72          */
  73         unsigned int msg_enabled_mask;
  74
  75         /**
  76          * @outstanding_submission_g2h: number of outstanding GuC to Host
  77          * responses related to GuC submission, used to determine if the GT is
  78          * idle
  79          */
  80         atomic_t outstanding_submission_g2h;
  81
  82         /** @tlb_lookup: xarray to store all pending TLB invalidation requests */
  83         struct xarray tlb_lookup;
  84
  85         /**
  86          * @serial_slot: id to the initial waiter created in tlb_lookup,
  87          * which is used only when failed to allocate new waiter.
  88          */
  89         u32 serial_slot;
  90
  91         /** @next_seqno: the next id (sequence number) to allocate. */
  92         u32 next_seqno;
  93
  94         /** @interrupts: pointers to GuC interrupt-managing functions. */
  95         struct {
  96                 bool enabled;
  97                 void (*reset)(struct intel_guc *guc);
  98                 void (*enable)(struct intel_guc *guc);
  99                 void (*disable)(struct intel_guc *guc);
 100         } interrupts;
 101
 102         /**
 103          * @submission_state: sub-structure for submission state protected by
 104          * single lock
 105          */
 106         struct {
 107                 /**
 108                  * @submission_state.lock: protects everything in
 109                  * submission_state, ce->guc_id.id, and ce->guc_id.ref
 110                  * when transitioning in and out of zero
 111                  */
 112                 spinlock_t lock;
 113                 /**
 114                  * @submission_state.guc_ids: used to allocate new
 115                  * guc_ids, single-lrc
 116                  */
 117                 struct ida guc_ids;
 118                 /**
 119                  * @submission_state.num_guc_ids: Number of guc_ids, selftest
 120                  * feature to be able to reduce this number while testing.
 121                  */
 122                 int num_guc_ids;
 123                 /**
 124                  * @submission_state.guc_ids_bitmap: used to allocate
 125                  * new guc_ids, multi-lrc
 126                  */
 127                 unsigned long *guc_ids_bitmap;
 128                 /**
 129                  * @submission_state.guc_id_list: list of intel_context
 130                  * with valid guc_ids but no refs
 131                  */
 132                 struct list_head guc_id_list;
 133                 /**
 134                  * @submission_state.guc_ids_in_use: Number single-lrc
 135                  * guc_ids in use
 136                  */
 137                 unsigned int guc_ids_in_use;
 138                 /**
 139                  * @submission_state.destroyed_contexts: list of contexts
 140                  * waiting to be destroyed (deregistered with the GuC)
 141                  */
 142                 struct list_head destroyed_contexts;
 143                 /**
 144                  * @submission_state.destroyed_worker: worker to deregister
 145                  * contexts, need as we need to take a GT PM reference and
 146                  * can't from destroy function as it might be in an atomic
 147                  * context (no sleeping)
 148                  */
 149                 struct work_struct destroyed_worker;
 150                 /**
 151                  * @submission_state.reset_fail_worker: worker to trigger
 152                  * a GT reset after an engine reset fails
 153                  */
 154                 struct work_struct reset_fail_worker;
 155                 /**
 156                  * @submission_state.reset_fail_mask: mask of engines that
 157                  * failed to reset
 158                  */
 159                 intel_engine_mask_t reset_fail_mask;
 160                 /**
 161                  * @submission_state.sched_disable_delay_ms: schedule
 162                  * disable delay, in ms, for contexts
 163                  */
 164                 unsigned int sched_disable_delay_ms;
 165                 /**
 166                  * @submission_state.sched_disable_gucid_threshold:
 167                  * threshold of min remaining available guc_ids before
 168                  * we start bypassing the schedule disable delay
 169                  */
 170                 unsigned int sched_disable_gucid_threshold;
 171         } submission_state;
 172
 173         /**
 174          * @submission_supported: tracks whether we support GuC submission on
 175          * the current platform
 176          */
 177         bool submission_supported;
 178         /** @submission_selected: tracks whether the user enabled GuC submission */
 179         bool submission_selected;
 180         /** @submission_initialized: tracks whether GuC submission has been initialised */
 181         bool submission_initialized;
 182         /** @submission_version: Submission API version of the currently loaded firmware */
 183         struct intel_uc_fw_ver submission_version;
 184
 185         /**
 186          * @rc_supported: tracks whether we support GuC rc on the current platform
 187          */
 188         bool rc_supported;
 189         /** @rc_selected: tracks whether the user enabled GuC rc */
 190         bool rc_selected;
 191
 192         /** @ads_vma: object allocated to hold the GuC ADS */
 193         struct i915_vma *ads_vma;
 194         /** @ads_map: contents of the GuC ADS */
 195         struct iosys_map ads_map;
 196         /** @ads_regset_size: size of the save/restore regsets in the ADS */
 197         u32 ads_regset_size;
 198         /**
 199          * @ads_regset_count: number of save/restore registers in the ADS for
 200          * each engine
 201          */
 202         u32 ads_regset_count[I915_NUM_ENGINES];
 203         /** @ads_regset: save/restore regsets in the ADS */
 204         struct guc_mmio_reg *ads_regset;
 205         /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */
 206         u32 ads_golden_ctxt_size;
 207         /** @ads_waklv_size: size of workaround KLVs */
 208         u32 ads_waklv_size;
 209         /** @ads_capture_size: size of register lists in the ADS used for error capture */
 210         u32 ads_capture_size;
 211
 212         /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */
 213         struct i915_vma *lrc_desc_pool_v69;
 214         /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */
 215         void *lrc_desc_pool_vaddr_v69;
 216
 217         /**
 218          * @context_lookup: used to resolve intel_context from guc_id, if a
 219          * context is present in this structure it is registered with the GuC
 220          */
 221         struct xarray context_lookup;
 222
 223         /** @params: Control params for fw initialization */
 224         u32 params[GUC_CTL_MAX_DWORDS];
 225
 226         /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */
 227         struct {
 228                 u32 base;
 229                 unsigned int count;
 230                 enum forcewake_domains fw_domains;
 231         } send_regs;
 232
 233         /** @notify_reg: register used to send interrupts to the GuC FW */
 234         i915_reg_t notify_reg;
 235
 236         /**
 237          * @mmio_msg: notification bitmask that the GuC writes in one of its
 238          * registers when the CT channel is disabled, to be processed when the
 239          * channel is back up.
 240          */
 241         u32 mmio_msg;
 242
 243         /** @send_mutex: used to serialize the intel_guc_send actions */
 244         struct mutex send_mutex;
 245
 246         /**
 247          * @timestamp: GT timestamp object that stores a copy of the timestamp
 248          * and adjusts it for overflow using a worker.
 249          */
 250         struct {
 251                 /**
 252                  * @timestamp.lock: Lock protecting the below fields and
 253                  * the engine stats.
 254                  */
 255                 spinlock_t lock;
 256
 257                 /**
 258                  * @timestamp.gt_stamp: 64-bit extended value of the GT
 259                  * timestamp.
 260                  */
 261                 u64 gt_stamp;
 262
 263                 /**
 264                  * @timestamp.ping_delay: Period for polling the GT
 265                  * timestamp for overflow.
 266                  */
 267                 unsigned long ping_delay;
 268
 269                 /**
 270                  * @timestamp.work: Periodic work to adjust GT timestamp,
 271                  * engine and context usage for overflows.
 272                  */
 273                 struct delayed_work work;
 274
 275                 /**
 276                  * @timestamp.shift: Right shift value for the gpm timestamp
 277                  */
 278                 u32 shift;
 279
 280                 /**
 281                  * @timestamp.last_stat_jiffies: jiffies at last actual
 282                  * stats collection time. We use this timestamp to ensure
 283                  * we don't oversample the stats because runtime power
 284                  * management events can trigger stats collection at much
 285                  * higher rates than required.
 286                  */
 287                 unsigned long last_stat_jiffies;
 288         } timestamp;
 289
 290         /**
 291          * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset.
 292          * Specifically used when the G2H handler wants to issue a reset. Resets
 293          * require flushing the G2H queue. So, the G2H processing itself must not
 294          * trigger a reset directly. Instead, go via this worker.
 295          */
 296         struct work_struct dead_guc_worker;
 297         /**
 298          * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance
 299          * used to prevent a fundamentally broken system from continuously
 300          * reloading the GuC.
 301          */
 302         unsigned long last_dead_guc_jiffies;
 303
 304 #ifdef CONFIG_DRM_I915_SELFTEST
 305         /**
 306          * @number_guc_id_stolen: The number of guc_ids that have been stolen
 307          */
 308         int number_guc_id_stolen;
 309         /**
 310          * @fast_response_selftest: Backdoor to CT handler for fast response selftest
 311          */
 312         u32 fast_response_selftest;
 313 #endif
 314 };
 315
 316 struct intel_guc_tlb_wait {
 317         struct wait_queue_head wq;
 318         bool busy;
 319 };
 320
 321 /*
 322  * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8
 323  * integer works.
 324  */
 325 #define MAKE_GUC_VER(maj, min, pat)     (((maj) << 16) | ((min) << 8) | (pat))
 326 #define MAKE_GUC_VER_STRUCT(ver)        MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
 327 #define GUC_SUBMIT_VER(guc)             MAKE_GUC_VER_STRUCT((guc)->submission_version)
 328 #define GUC_FIRMWARE_VER(guc)           MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver)
 329
 330 static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
 331 {
 332         return container_of(log, struct intel_guc, log);
 333 }
 334
 335 static
 336 inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)
 337 {
 338         return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 0);
 339 }
 340
 341 static
 342 inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len,
 343                              u32 g2h_len_dw)
 344 {
 345         return intel_guc_ct_send(&guc->ct, action, len, NULL, 0,
 346                                  MAKE_SEND_FLAGS(g2h_len_dw));
 347 }
 348
 349 static inline int
 350 intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len,
 351                            u32 *response_buf, u32 response_buf_size)
 352 {
 353         return intel_guc_ct_send(&guc->ct, action, len,
 354                                  response_buf, response_buf_size, 0);
 355 }
 356
 357 static inline int intel_guc_send_busy_loop(struct intel_guc *guc,
 358                                            const u32 *action,
 359                                            u32 len,
 360                                            u32 g2h_len_dw,
 361                                            bool loop)
 362 {
 363         int err;
 364         unsigned int sleep_period_ms = 1;
 365         bool not_atomic = !in_atomic() && !irqs_disabled();
 366
 367         /*
 368          * FIXME: Have caller pass in if we are in an atomic context to avoid
 369          * using in_atomic(). It is likely safe here as we check for irqs
 370          * disabled which basically all the spin locks in the i915 do but
 371          * regardless this should be cleaned up.
 372          */
 373
 374         /* No sleeping with spin locks, just busy loop */
 375         might_sleep_if(loop && not_atomic);
 376
 377 retry:
 378         err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
 379         if (unlikely(err == -EBUSY && loop)) {
 380                 if (likely(not_atomic)) {
 381                         if (msleep_interruptible(sleep_period_ms))
 382                                 return -EINTR;
 383                         sleep_period_ms = sleep_period_ms << 1;
 384                 } else {
 385                         cpu_relax();
 386                 }
 387                 goto retry;
 388         }
 389
 390         return err;
 391 }
 392
 393 /* Only call this from the interrupt handler code */
 394 static inline void intel_guc_to_host_event_handler(struct intel_guc *guc)
 395 {
 396         if (guc->interrupts.enabled)
 397                 intel_guc_ct_event_handler(&guc->ct);
 398 }
 399
 400 /* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
 401 #define GUC_GGTT_TOP    0xFEE00000
 402
 403 /**
 404  * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma
 405  * @guc: intel_guc structure.
 406  * @vma: i915 graphics virtual memory area.
 407  *
 408  * GuC does not allow any gfx GGTT address that falls into range
 409  * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM.
 410  * Currently, in order to exclude [0, ggtt.pin_bias) address space from
 411  * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma()
 412  * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias.
 413  *
 414  * Return: GGTT offset of the @vma.
 415  */
 416 static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,
 417                                         struct i915_vma *vma)
 418 {
 419         u32 offset = i915_ggtt_offset(vma);
 420
 421         GEM_BUG_ON(offset < i915_ggtt_pin_bias(vma));
 422         GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP));
 423
 424         return offset;
 425 }
 426
 427 void intel_guc_init_early(struct intel_guc *guc);
 428 void intel_guc_init_late(struct intel_guc *guc);
 429 void intel_guc_init_send_regs(struct intel_guc *guc);
 430 void intel_guc_write_params(struct intel_guc *guc);
 431 int intel_guc_init(struct intel_guc *guc);
 432 void intel_guc_fini(struct intel_guc *guc);
 433 void intel_guc_notify(struct intel_guc *guc);
 434 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 435                         u32 *response_buf, u32 response_buf_size);
 436 int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,
 437                                        const u32 *payload, u32 len);
 438 int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
 439 int intel_guc_suspend(struct intel_guc *guc);
 440 int intel_guc_resume(struct intel_guc *guc);
 441 struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);
 442 int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,
 443                                    struct i915_vma **out_vma, void **out_vaddr);
 444 int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);
 445 int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);
 446
 447 static inline bool intel_guc_is_supported(struct intel_guc *guc)
 448 {
 449         return intel_uc_fw_is_supported(&guc->fw);
 450 }
 451
 452 static inline bool intel_guc_is_wanted(struct intel_guc *guc)
 453 {
 454         return intel_uc_fw_is_enabled(&guc->fw);
 455 }
 456
 457 static inline bool intel_guc_is_used(struct intel_guc *guc)
 458 {
 459         GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED);
 460         return intel_uc_fw_is_available(&guc->fw);
 461 }
 462
 463 static inline bool intel_guc_is_fw_running(struct intel_guc *guc)
 464 {
 465         return intel_uc_fw_is_running(&guc->fw);
 466 }
 467
 468 static inline bool intel_guc_is_ready(struct intel_guc *guc)
 469 {
 470         return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(&guc->ct);
 471 }
 472
 473 static inline void intel_guc_reset_interrupts(struct intel_guc *guc)
 474 {
 475         guc->interrupts.reset(guc);
 476 }
 477
 478 static inline void intel_guc_enable_interrupts(struct intel_guc *guc)
 479 {
 480         guc->interrupts.enable(guc);
 481 }
 482
 483 static inline void intel_guc_disable_interrupts(struct intel_guc *guc)
 484 {
 485         guc->interrupts.disable(guc);
 486 }
 487
 488 static inline int intel_guc_sanitize(struct intel_guc *guc)
 489 {
 490         intel_uc_fw_sanitize(&guc->fw);
 491         intel_guc_disable_interrupts(guc);
 492         intel_guc_ct_sanitize(&guc->ct);
 493         guc->mmio_msg = 0;
 494
 495         return 0;
 496 }
 497
 498 static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask)
 499 {
 500         spin_lock_irq(&guc->irq_lock);
 501         guc->msg_enabled_mask |= mask;
 502         spin_unlock_irq(&guc->irq_lock);
 503 }
 504
 505 static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask)
 506 {
 507         spin_lock_irq(&guc->irq_lock);
 508         guc->msg_enabled_mask &= ~mask;
 509         spin_unlock_irq(&guc->irq_lock);
 510 }
 511
 512 int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout);
 513
 514 int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
 515                                           const u32 *msg, u32 len);
 516 int intel_guc_sched_done_process_msg(struct intel_guc *guc,
 517                                      const u32 *msg, u32 len);
 518 int intel_guc_context_reset_process_msg(struct intel_guc *guc,
 519                                         const u32 *msg, u32 len);
 520 int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
 521                                          const u32 *msg, u32 len);
 522 int intel_guc_error_capture_process_msg(struct intel_guc *guc,
 523                                         const u32 *msg, u32 len);
 524 int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action);
 525
 526 struct intel_engine_cs *
 527 intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance);
 528
 529 void intel_guc_find_hung_context(struct intel_engine_cs *engine);
 530
 531 int intel_guc_global_policies_update(struct intel_guc *guc);
 532
 533 void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq);
 534
 535 void intel_guc_submission_reset_prepare(struct intel_guc *guc);
 536 void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled);
 537 void intel_guc_submission_reset_finish(struct intel_guc *guc);
 538 void intel_guc_submission_cancel_requests(struct intel_guc *guc);
 539
 540 void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p);
 541
 542 void intel_guc_write_barrier(struct intel_guc *guc);
 543
 544 void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);
 545
 546 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc);
 547
 548 bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc);
 549 int intel_guc_invalidate_tlb_engines(struct intel_guc *guc);
 550 int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
 551 int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
 552                                     const u32 *payload, u32 len);
 553 void wake_up_all_tlb_invalidate(struct intel_guc *guc);
 554 #endif