1 // SPDX-License-Identifier: MIT
3 * Copyright © 2016-2019 Intel Corporation
6 #include "gt/intel_gt.h"
7 #include "gt/intel_reset.h"
9 #include "intel_guc_ads.h"
10 #include "intel_guc_submission.h"
15 static const struct intel_uc_ops uc_ops_off;
16 static const struct intel_uc_ops uc_ops_on;
18 static void uc_expand_default_options(struct intel_uc *uc)
20 struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
22 if (i915->params.enable_guc != -1)
25 /* Don't enable GuC/HuC on pre-Gen12 */
26 if (GRAPHICS_VER(i915) < 12) {
27 i915->params.enable_guc = 0;
31 /* Don't enable GuC/HuC on older Gen12 platforms */
32 if (IS_TIGERLAKE(i915) || IS_ROCKETLAKE(i915)) {
33 i915->params.enable_guc = 0;
37 /* Default: enable HuC authentication only */
38 i915->params.enable_guc = ENABLE_GUC_LOAD_HUC;
41 /* Reset GuC providing us with fresh state for both GuC and HuC.
43 static int __intel_uc_reset_hw(struct intel_uc *uc)
45 struct intel_gt *gt = uc_to_gt(uc);
49 ret = i915_inject_probe_error(gt->i915, -ENXIO);
53 ret = intel_reset_guc(gt);
55 DRM_ERROR("Failed to reset GuC, ret = %d\n", ret);
59 guc_status = intel_uncore_read(gt->uncore, GUC_STATUS);
60 WARN(!(guc_status & GS_MIA_IN_RESET),
61 "GuC status: 0x%x, MIA core expected to be in reset\n",
67 static void __confirm_options(struct intel_uc *uc)
69 struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
72 "enable_guc=%d (guc:%s submission:%s huc:%s)\n",
73 i915->params.enable_guc,
74 yesno(intel_uc_wants_guc(uc)),
75 yesno(intel_uc_wants_guc_submission(uc)),
76 yesno(intel_uc_wants_huc(uc)));
78 if (i915->params.enable_guc == 0) {
79 GEM_BUG_ON(intel_uc_wants_guc(uc));
80 GEM_BUG_ON(intel_uc_wants_guc_submission(uc));
81 GEM_BUG_ON(intel_uc_wants_huc(uc));
85 if (!intel_uc_supports_guc(uc))
87 "Incompatible option enable_guc=%d - %s\n",
88 i915->params.enable_guc, "GuC is not supported!");
90 if (i915->params.enable_guc & ENABLE_GUC_LOAD_HUC &&
91 !intel_uc_supports_huc(uc))
93 "Incompatible option enable_guc=%d - %s\n",
94 i915->params.enable_guc, "HuC is not supported!");
96 if (i915->params.enable_guc & ENABLE_GUC_SUBMISSION &&
97 !intel_uc_supports_guc_submission(uc))
99 "Incompatible option enable_guc=%d - %s\n",
100 i915->params.enable_guc, "GuC submission is N/A");
102 if (i915->params.enable_guc & ~ENABLE_GUC_MASK)
104 "Incompatible option enable_guc=%d - %s\n",
105 i915->params.enable_guc, "undocumented flag");
108 void intel_uc_init_early(struct intel_uc *uc)
110 uc_expand_default_options(uc);
112 intel_guc_init_early(&uc->guc);
113 intel_huc_init_early(&uc->huc);
115 __confirm_options(uc);
117 if (intel_uc_wants_guc(uc))
118 uc->ops = &uc_ops_on;
120 uc->ops = &uc_ops_off;
123 void intel_uc_driver_late_release(struct intel_uc *uc)
128 * intel_uc_init_mmio - setup uC MMIO access
129 * @uc: the intel_uc structure
131 * Setup minimal state necessary for MMIO accesses later in the
132 * initialization sequence.
134 void intel_uc_init_mmio(struct intel_uc *uc)
136 intel_guc_init_send_regs(&uc->guc);
139 static void __uc_capture_load_err_log(struct intel_uc *uc)
141 struct intel_guc *guc = &uc->guc;
143 if (guc->log.vma && !uc->load_err_log)
144 uc->load_err_log = i915_gem_object_get(guc->log.vma->obj);
147 static void __uc_free_load_err_log(struct intel_uc *uc)
149 struct drm_i915_gem_object *log = fetch_and_zero(&uc->load_err_log);
152 i915_gem_object_put(log);
155 void intel_uc_driver_remove(struct intel_uc *uc)
157 intel_uc_fini_hw(uc);
159 __uc_free_load_err_log(uc);
162 static inline bool guc_communication_enabled(struct intel_guc *guc)
164 return intel_guc_ct_enabled(&guc->ct);
168 * Events triggered while CT buffers are disabled are logged in the SCRATCH_15
169 * register using the same bits used in the CT message payload. Since our
170 * communication channel with guc is turned off at this point, we can save the
171 * message and handle it after we turn it back on.
173 static void guc_clear_mmio_msg(struct intel_guc *guc)
175 intel_uncore_write(guc_to_gt(guc)->uncore, SOFT_SCRATCH(15), 0);
178 static void guc_get_mmio_msg(struct intel_guc *guc)
182 spin_lock_irq(&guc->irq_lock);
184 val = intel_uncore_read(guc_to_gt(guc)->uncore, SOFT_SCRATCH(15));
185 guc->mmio_msg |= val & guc->msg_enabled_mask;
188 * clear all events, including the ones we're not currently servicing,
189 * to make sure we don't try to process a stale message if we enable
190 * handling of more events later.
192 guc_clear_mmio_msg(guc);
194 spin_unlock_irq(&guc->irq_lock);
197 static void guc_handle_mmio_msg(struct intel_guc *guc)
199 /* we need communication to be enabled to reply to GuC */
200 GEM_BUG_ON(!guc_communication_enabled(guc));
202 spin_lock_irq(&guc->irq_lock);
204 intel_guc_to_host_process_recv_msg(guc, &guc->mmio_msg, 1);
207 spin_unlock_irq(&guc->irq_lock);
210 static void guc_reset_interrupts(struct intel_guc *guc)
212 guc->interrupts.reset(guc);
215 static void guc_enable_interrupts(struct intel_guc *guc)
217 guc->interrupts.enable(guc);
220 static void guc_disable_interrupts(struct intel_guc *guc)
222 guc->interrupts.disable(guc);
225 static int guc_enable_communication(struct intel_guc *guc)
227 struct intel_gt *gt = guc_to_gt(guc);
228 struct drm_i915_private *i915 = gt->i915;
231 GEM_BUG_ON(guc_communication_enabled(guc));
233 ret = i915_inject_probe_error(i915, -ENXIO);
237 ret = intel_guc_ct_enable(&guc->ct);
241 /* check for mmio messages received before/during the CT enable */
242 guc_get_mmio_msg(guc);
243 guc_handle_mmio_msg(guc);
245 guc_enable_interrupts(guc);
247 /* check for CT messages received before we enabled interrupts */
248 spin_lock_irq(>->irq_lock);
249 intel_guc_ct_event_handler(&guc->ct);
250 spin_unlock_irq(>->irq_lock);
252 drm_dbg(&i915->drm, "GuC communication enabled\n");
257 static void guc_disable_communication(struct intel_guc *guc)
259 struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
262 * Events generated during or after CT disable are logged by guc in
263 * via mmio. Make sure the register is clear before disabling CT since
264 * all events we cared about have already been processed via CT.
266 guc_clear_mmio_msg(guc);
268 guc_disable_interrupts(guc);
270 intel_guc_ct_disable(&guc->ct);
273 * Check for messages received during/after the CT disable. We do not
274 * expect any messages to have arrived via CT between the interrupt
275 * disable and the CT disable because GuC should've been idle until we
276 * triggered the CT disable protocol.
278 guc_get_mmio_msg(guc);
280 drm_dbg(&i915->drm, "GuC communication disabled\n");
283 static void __uc_fetch_firmwares(struct intel_uc *uc)
287 GEM_BUG_ON(!intel_uc_wants_guc(uc));
289 err = intel_uc_fw_fetch(&uc->guc.fw);
291 /* Make sure we transition out of transient "SELECTED" state */
292 if (intel_uc_wants_huc(uc)) {
293 drm_dbg(&uc_to_gt(uc)->i915->drm,
294 "Failed to fetch GuC: %d disabling HuC\n", err);
295 intel_uc_fw_change_status(&uc->huc.fw,
296 INTEL_UC_FIRMWARE_ERROR);
302 if (intel_uc_wants_huc(uc))
303 intel_uc_fw_fetch(&uc->huc.fw);
306 static void __uc_cleanup_firmwares(struct intel_uc *uc)
308 intel_uc_fw_cleanup_fetch(&uc->huc.fw);
309 intel_uc_fw_cleanup_fetch(&uc->guc.fw);
312 static int __uc_init(struct intel_uc *uc)
314 struct intel_guc *guc = &uc->guc;
315 struct intel_huc *huc = &uc->huc;
318 GEM_BUG_ON(!intel_uc_wants_guc(uc));
320 if (!intel_uc_uses_guc(uc))
323 if (i915_inject_probe_failure(uc_to_gt(uc)->i915))
326 /* XXX: GuC submission is unavailable for now */
327 GEM_BUG_ON(intel_uc_uses_guc_submission(uc));
329 ret = intel_guc_init(guc);
333 if (intel_uc_uses_huc(uc)) {
334 ret = intel_huc_init(huc);
346 static void __uc_fini(struct intel_uc *uc)
348 intel_huc_fini(&uc->huc);
349 intel_guc_fini(&uc->guc);
352 static int __uc_sanitize(struct intel_uc *uc)
354 struct intel_guc *guc = &uc->guc;
355 struct intel_huc *huc = &uc->huc;
357 GEM_BUG_ON(!intel_uc_supports_guc(uc));
359 intel_huc_sanitize(huc);
360 intel_guc_sanitize(guc);
362 return __intel_uc_reset_hw(uc);
365 /* Initialize and verify the uC regs related to uC positioning in WOPCM */
366 static int uc_init_wopcm(struct intel_uc *uc)
368 struct intel_gt *gt = uc_to_gt(uc);
369 struct intel_uncore *uncore = gt->uncore;
370 u32 base = intel_wopcm_guc_base(>->i915->wopcm);
371 u32 size = intel_wopcm_guc_size(>->i915->wopcm);
372 u32 huc_agent = intel_uc_uses_huc(uc) ? HUC_LOADING_AGENT_GUC : 0;
376 if (unlikely(!base || !size)) {
377 i915_probe_error(gt->i915, "Unsuccessful WOPCM partitioning\n");
381 GEM_BUG_ON(!intel_uc_supports_guc(uc));
382 GEM_BUG_ON(!(base & GUC_WOPCM_OFFSET_MASK));
383 GEM_BUG_ON(base & ~GUC_WOPCM_OFFSET_MASK);
384 GEM_BUG_ON(!(size & GUC_WOPCM_SIZE_MASK));
385 GEM_BUG_ON(size & ~GUC_WOPCM_SIZE_MASK);
387 err = i915_inject_probe_error(gt->i915, -ENXIO);
391 mask = GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED;
392 err = intel_uncore_write_and_verify(uncore, GUC_WOPCM_SIZE, size, mask,
393 size | GUC_WOPCM_SIZE_LOCKED);
397 mask = GUC_WOPCM_OFFSET_MASK | GUC_WOPCM_OFFSET_VALID | huc_agent;
398 err = intel_uncore_write_and_verify(uncore, DMA_GUC_WOPCM_OFFSET,
399 base | huc_agent, mask,
401 GUC_WOPCM_OFFSET_VALID);
408 i915_probe_error(gt->i915, "Failed to init uC WOPCM registers!\n");
409 i915_probe_error(gt->i915, "%s(%#x)=%#x\n", "DMA_GUC_WOPCM_OFFSET",
410 i915_mmio_reg_offset(DMA_GUC_WOPCM_OFFSET),
411 intel_uncore_read(uncore, DMA_GUC_WOPCM_OFFSET));
412 i915_probe_error(gt->i915, "%s(%#x)=%#x\n", "GUC_WOPCM_SIZE",
413 i915_mmio_reg_offset(GUC_WOPCM_SIZE),
414 intel_uncore_read(uncore, GUC_WOPCM_SIZE));
419 static bool uc_is_wopcm_locked(struct intel_uc *uc)
421 struct intel_gt *gt = uc_to_gt(uc);
422 struct intel_uncore *uncore = gt->uncore;
424 return (intel_uncore_read(uncore, GUC_WOPCM_SIZE) & GUC_WOPCM_SIZE_LOCKED) ||
425 (intel_uncore_read(uncore, DMA_GUC_WOPCM_OFFSET) & GUC_WOPCM_OFFSET_VALID);
428 static int __uc_check_hw(struct intel_uc *uc)
430 if (!intel_uc_supports_guc(uc))
434 * We can silently continue without GuC only if it was never enabled
435 * before on this system after reboot, otherwise we risk GPU hangs.
436 * To check if GuC was loaded before we look at WOPCM registers.
438 if (uc_is_wopcm_locked(uc))
444 static int __uc_init_hw(struct intel_uc *uc)
446 struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
447 struct intel_guc *guc = &uc->guc;
448 struct intel_huc *huc = &uc->huc;
451 GEM_BUG_ON(!intel_uc_supports_guc(uc));
452 GEM_BUG_ON(!intel_uc_wants_guc(uc));
454 if (!intel_uc_fw_is_loadable(&guc->fw)) {
455 ret = __uc_check_hw(uc) ||
456 intel_uc_fw_is_overridden(&guc->fw) ||
457 intel_uc_wants_guc_submission(uc) ?
458 intel_uc_fw_status_to_error(guc->fw.status) : 0;
462 ret = uc_init_wopcm(uc);
466 guc_reset_interrupts(guc);
468 /* WaEnableuKernelHeaderValidFix:skl */
469 /* WaEnableGuCBootHashCheckNotSet:skl,bxt,kbl */
470 if (GRAPHICS_VER(i915) == 9)
477 * Always reset the GuC just before (re)loading, so
478 * that the state and timing are fairly predictable
480 ret = __uc_sanitize(uc);
484 intel_huc_fw_upload(huc);
485 intel_guc_ads_reset(guc);
486 intel_guc_write_params(guc);
487 ret = intel_guc_fw_upload(guc);
491 DRM_DEBUG_DRIVER("GuC fw load failed: %d; will reset and "
492 "retry %d more time(s)\n", ret, attempts);
495 /* Did we succeded or run out of retries? */
497 goto err_log_capture;
499 ret = guc_enable_communication(guc);
501 goto err_log_capture;
505 if (intel_uc_uses_guc_submission(uc))
506 intel_guc_submission_enable(guc);
508 drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n",
509 intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_GUC), guc->fw.path,
510 guc->fw.major_ver_found, guc->fw.minor_ver_found,
512 enableddisabled(intel_uc_uses_guc_submission(uc)));
514 if (intel_uc_uses_huc(uc)) {
515 drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n",
516 intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_HUC),
518 huc->fw.major_ver_found, huc->fw.minor_ver_found,
520 yesno(intel_huc_is_authenticated(huc)));
526 * We've failed to load the firmware :(
529 __uc_capture_load_err_log(uc);
534 drm_notice(&i915->drm, "GuC is uninitialized\n");
535 /* We want to run without GuC submission */
539 i915_probe_error(i915, "GuC initialization failed %d\n", ret);
541 /* We want to keep KMS alive */
545 static void __uc_fini_hw(struct intel_uc *uc)
547 struct intel_guc *guc = &uc->guc;
549 if (!intel_guc_is_fw_running(guc))
552 if (intel_uc_uses_guc_submission(uc))
553 intel_guc_submission_disable(guc);
559 * intel_uc_reset_prepare - Prepare for reset
560 * @uc: the intel_uc structure
562 * Preparing for full gpu reset.
564 void intel_uc_reset_prepare(struct intel_uc *uc)
566 struct intel_guc *guc = &uc->guc;
568 if (!intel_guc_is_ready(guc))
574 void intel_uc_runtime_suspend(struct intel_uc *uc)
576 struct intel_guc *guc = &uc->guc;
579 if (!intel_guc_is_ready(guc))
582 err = intel_guc_suspend(guc);
584 DRM_DEBUG_DRIVER("Failed to suspend GuC, err=%d", err);
586 guc_disable_communication(guc);
589 void intel_uc_suspend(struct intel_uc *uc)
591 struct intel_guc *guc = &uc->guc;
592 intel_wakeref_t wakeref;
594 if (!intel_guc_is_ready(guc))
597 with_intel_runtime_pm(uc_to_gt(uc)->uncore->rpm, wakeref)
598 intel_uc_runtime_suspend(uc);
601 static int __uc_resume(struct intel_uc *uc, bool enable_communication)
603 struct intel_guc *guc = &uc->guc;
606 if (!intel_guc_is_fw_running(guc))
609 /* Make sure we enable communication if and only if it's disabled */
610 GEM_BUG_ON(enable_communication == guc_communication_enabled(guc));
612 if (enable_communication)
613 guc_enable_communication(guc);
615 err = intel_guc_resume(guc);
617 DRM_DEBUG_DRIVER("Failed to resume GuC, err=%d", err);
624 int intel_uc_resume(struct intel_uc *uc)
627 * When coming out of S3/S4 we sanitize and re-init the HW, so
628 * communication is already re-enabled at this point.
630 return __uc_resume(uc, false);
633 int intel_uc_runtime_resume(struct intel_uc *uc)
636 * During runtime resume we don't sanitize, so we need to re-init
637 * communication as well.
639 return __uc_resume(uc, true);
642 static const struct intel_uc_ops uc_ops_off = {
643 .init_hw = __uc_check_hw,
646 static const struct intel_uc_ops uc_ops_on = {
647 .sanitize = __uc_sanitize,
649 .init_fw = __uc_fetch_firmwares,
650 .fini_fw = __uc_cleanup_firmwares,
655 .init_hw = __uc_init_hw,
656 .fini_hw = __uc_fini_hw,