1 // SPDX-License-Identifier: MIT
3 * Copyright © 2014-2019 Intel Corporation
6 #include <linux/bsearch.h>
8 #include "gt/intel_engine_regs.h"
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_regs.h"
11 #include "gt/intel_lrc.h"
12 #include "gt/shmem_utils.h"
13 #include "intel_guc_ads.h"
14 #include "intel_guc_fwif.h"
19 * The Additional Data Struct (ADS) has pointers for different buffers used by
20 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
21 * all the extra buffers indirectly linked via the ADS struct's entries.
23 * Layout of the ADS blob allocated for the GuC:
25 * +---------------------------------------+ <== base
27 * +---------------------------------------+
29 * +---------------------------------------+
30 * | guc_gt_system_info |
31 * +---------------------------------------+
32 * | guc_engine_usage |
33 * +---------------------------------------+ <== static
34 * | guc_mmio_reg[countA] (engine 0.0) |
35 * | guc_mmio_reg[countB] (engine 0.1) |
36 * | guc_mmio_reg[countC] (engine 1.0) |
38 * +---------------------------------------+ <== dynamic
40 * +---------------------------------------+ <== 4K aligned
42 * +---------------------------------------+
44 * +---------------------------------------+ <== 4K aligned
46 * +---------------------------------------+
48 * +---------------------------------------+ <== 4K aligned
50 * +---------------------------------------+
52 * +---------------------------------------+ <== 4K aligned
54 struct __guc_ads_blob {
56 struct guc_policies policies;
57 struct guc_gt_system_info system_info;
58 struct guc_engine_usage engine_usage;
59 /* From here on, location is dynamic! Refer to above diagram. */
60 struct guc_mmio_reg regset[0];
63 #define ads_blob_read(guc_, field_) \
64 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
66 #define ads_blob_write(guc_, field_, val_) \
67 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \
70 #define info_map_write(map_, field_, val_) \
71 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
73 #define info_map_read(map_, field_) \
74 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
76 static u32 guc_ads_regset_size(struct intel_guc *guc)
78 GEM_BUG_ON(!guc->ads_regset_size);
79 return guc->ads_regset_size;
82 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
84 return PAGE_ALIGN(guc->ads_golden_ctxt_size);
87 static u32 guc_ads_capture_size(struct intel_guc *guc)
89 /* FIXME: Allocate a proper capture list */
90 return PAGE_ALIGN(PAGE_SIZE);
93 static u32 guc_ads_private_data_size(struct intel_guc *guc)
95 return PAGE_ALIGN(guc->fw.private_data_size);
98 static u32 guc_ads_regset_offset(struct intel_guc *guc)
100 return offsetof(struct __guc_ads_blob, regset);
103 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
107 offset = guc_ads_regset_offset(guc) +
108 guc_ads_regset_size(guc);
110 return PAGE_ALIGN(offset);
113 static u32 guc_ads_capture_offset(struct intel_guc *guc)
117 offset = guc_ads_golden_ctxt_offset(guc) +
118 guc_ads_golden_ctxt_size(guc);
120 return PAGE_ALIGN(offset);
123 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
127 offset = guc_ads_capture_offset(guc) +
128 guc_ads_capture_size(guc);
130 return PAGE_ALIGN(offset);
133 static u32 guc_ads_blob_size(struct intel_guc *guc)
135 return guc_ads_private_data_offset(guc) +
136 guc_ads_private_data_size(guc);
139 static void guc_policies_init(struct intel_guc *guc)
141 struct intel_gt *gt = guc_to_gt(guc);
142 struct drm_i915_private *i915 = gt->i915;
143 u32 global_flags = 0;
145 ads_blob_write(guc, policies.dpc_promote_time,
146 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
147 ads_blob_write(guc, policies.max_num_work_items,
148 GLOBAL_POLICY_MAX_NUM_WI);
150 if (i915->params.reset < 2)
151 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
153 ads_blob_write(guc, policies.global_flags, global_flags);
154 ads_blob_write(guc, policies.is_valid, 1);
157 void intel_guc_ads_print_policy_info(struct intel_guc *guc,
158 struct drm_printer *dp)
160 if (unlikely(iosys_map_is_null(&guc->ads_map)))
163 drm_printf(dp, "Global scheduling policies:\n");
164 drm_printf(dp, " DPC promote time = %u\n",
165 ads_blob_read(guc, policies.dpc_promote_time));
166 drm_printf(dp, " Max num work items = %u\n",
167 ads_blob_read(guc, policies.max_num_work_items));
168 drm_printf(dp, " Flags = %u\n",
169 ads_blob_read(guc, policies.global_flags));
172 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
175 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
179 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
182 int intel_guc_global_policies_update(struct intel_guc *guc)
184 struct intel_gt *gt = guc_to_gt(guc);
185 u32 scheduler_policies;
186 intel_wakeref_t wakeref;
189 if (iosys_map_is_null(&guc->ads_map))
192 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
193 GEM_BUG_ON(!scheduler_policies);
195 guc_policies_init(guc);
197 if (!intel_guc_is_ready(guc))
200 with_intel_runtime_pm(>->i915->runtime_pm, wakeref)
201 ret = guc_action_policies_update(guc, scheduler_policies);
206 static void guc_mapping_table_init(struct intel_gt *gt,
207 struct iosys_map *info_map)
210 struct intel_engine_cs *engine;
211 enum intel_engine_id id;
213 /* Table must be set to invalid values for entries not used */
214 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
215 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
216 info_map_write(info_map, mapping_table[i][j],
217 GUC_MAX_INSTANCES_PER_CLASS);
219 for_each_engine(engine, gt, id) {
220 u8 guc_class = engine_class_to_guc_class(engine->class);
222 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
228 * The save/restore register list must be pre-calculated to a temporary
229 * buffer before it can be copied inside the ADS.
233 * ptr to the section of the storage for the engine currently being
236 struct guc_mmio_reg *registers;
237 /* ptr to the base of the allocated storage for all engines */
238 struct guc_mmio_reg *storage;
243 static int guc_mmio_reg_cmp(const void *a, const void *b)
245 const struct guc_mmio_reg *ra = a;
246 const struct guc_mmio_reg *rb = b;
248 return (int)ra->offset - (int)rb->offset;
251 static struct guc_mmio_reg * __must_check
252 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
254 u32 pos = regset->storage_used;
255 struct guc_mmio_reg *slot;
257 if (pos >= regset->storage_max) {
258 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
259 struct guc_mmio_reg *r = krealloc(regset->storage,
262 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
264 return ERR_PTR(-ENOMEM);
267 regset->registers = r + (regset->registers - regset->storage);
269 regset->storage_max = size / sizeof(*slot);
272 slot = ®set->storage[pos];
273 regset->storage_used++;
279 static long __must_check guc_mmio_reg_add(struct temp_regset *regset,
280 u32 offset, u32 flags)
282 u32 count = regset->storage_used - (regset->registers - regset->storage);
283 struct guc_mmio_reg reg = {
287 struct guc_mmio_reg *slot;
290 * The mmio list is built using separate lists within the driver.
291 * It's possible that at some point we may attempt to add the same
292 * register more than once. Do not consider this an error; silently
293 * move on if the register is already in the list.
295 if (bsearch(®, regset->registers, count,
296 sizeof(reg), guc_mmio_reg_cmp))
299 slot = __mmio_reg_add(regset, ®);
301 return PTR_ERR(slot);
303 while (slot-- > regset->registers) {
304 GEM_BUG_ON(slot[0].offset == slot[1].offset);
305 if (slot[1].offset > slot[0].offset)
308 swap(slot[1], slot[0]);
314 #define GUC_MMIO_REG_ADD(regset, reg, masked) \
315 guc_mmio_reg_add(regset, \
316 i915_mmio_reg_offset((reg)), \
317 (masked) ? GUC_REGSET_MASKED : 0)
319 static int guc_mmio_regset_init(struct temp_regset *regset,
320 struct intel_engine_cs *engine)
322 const u32 base = engine->mmio_base;
323 struct i915_wa_list *wal = &engine->wa_list;
329 * Each engine's registers point to a new start relative to
332 regset->registers = regset->storage + regset->storage_used;
334 ret |= GUC_MMIO_REG_ADD(regset, RING_MODE_GEN7(base), true);
335 ret |= GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false);
336 ret |= GUC_MMIO_REG_ADD(regset, RING_IMR(base), false);
338 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
339 ret |= GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg);
341 /* Be extra paranoid and include all whitelist registers. */
342 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
343 ret |= GUC_MMIO_REG_ADD(regset,
344 RING_FORCE_TO_NONPRIV(base, i),
347 /* add in local MOCS registers */
348 for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++)
349 ret |= GUC_MMIO_REG_ADD(regset, GEN9_LNCFCMOCS(i), false);
354 static long guc_mmio_reg_state_create(struct intel_guc *guc)
356 struct intel_gt *gt = guc_to_gt(guc);
357 struct intel_engine_cs *engine;
358 enum intel_engine_id id;
359 struct temp_regset temp_set = {};
363 for_each_engine(engine, gt, id) {
364 u32 used = temp_set.storage_used;
366 ret = guc_mmio_regset_init(&temp_set, engine);
368 goto fail_regset_init;
370 guc->ads_regset_count[id] = temp_set.storage_used - used;
371 total += guc->ads_regset_count[id];
374 guc->ads_regset = temp_set.storage;
376 drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n",
377 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
379 return total * sizeof(struct guc_mmio_reg);
382 kfree(temp_set.storage);
386 static void guc_mmio_reg_state_init(struct intel_guc *guc)
388 struct intel_gt *gt = guc_to_gt(guc);
389 struct intel_engine_cs *engine;
390 enum intel_engine_id id;
391 u32 addr_ggtt, offset;
393 offset = guc_ads_regset_offset(guc);
394 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
396 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
397 guc->ads_regset_size);
399 for_each_engine(engine, gt, id) {
400 u32 count = guc->ads_regset_count[id];
403 /* Class index is checked in class converter */
404 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
406 guc_class = engine_class_to_guc_class(engine->class);
410 ads.reg_state_list[guc_class][engine->instance].address,
413 ads.reg_state_list[guc_class][engine->instance].count,
419 ads.reg_state_list[guc_class][engine->instance].address,
422 ads.reg_state_list[guc_class][engine->instance].count,
425 addr_ggtt += count * sizeof(struct guc_mmio_reg);
429 static void fill_engine_enable_masks(struct intel_gt *gt,
430 struct iosys_map *info_map)
432 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], 1);
433 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], 1);
434 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
435 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
438 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
439 #define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE)
440 static int guc_prep_golden_context(struct intel_guc *guc)
442 struct intel_gt *gt = guc_to_gt(guc);
443 u32 addr_ggtt, offset;
444 u32 total_size = 0, alloc_size, real_size;
445 u8 engine_class, guc_class;
446 struct guc_gt_system_info local_info;
447 struct iosys_map info_map;
450 * Reserve the memory for the golden contexts and point GuC at it but
451 * leave it empty for now. The context data will be filled in later
452 * once there is something available to put there.
454 * Note that the HWSP and ring context are not included.
456 * Note also that the storage must be pinned in the GGTT, so that the
457 * address won't change after GuC has been told where to find it. The
458 * GuC will also validate that the LRC base + size fall within the
459 * allowed GGTT range.
461 if (!iosys_map_is_null(&guc->ads_map)) {
462 offset = guc_ads_golden_ctxt_offset(guc);
463 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
464 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
465 offsetof(struct __guc_ads_blob, system_info));
467 memset(&local_info, 0, sizeof(local_info));
468 iosys_map_set_vaddr(&info_map, &local_info);
469 fill_engine_enable_masks(gt, &info_map);
472 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
473 if (engine_class == OTHER_CLASS)
476 guc_class = engine_class_to_guc_class(engine_class);
478 if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
481 real_size = intel_engine_context_size(gt, engine_class);
482 alloc_size = PAGE_ALIGN(real_size);
483 total_size += alloc_size;
485 if (iosys_map_is_null(&guc->ads_map))
489 * This interface is slightly confusing. We need to pass the
490 * base address of the full golden context and the size of just
491 * the engine state, which is the section of the context image
492 * that starts after the execlists context. This is required to
493 * allow the GuC to restore just the engine state when a
494 * watchdog reset occurs.
495 * We calculate the engine state size by removing the size of
496 * what comes before it in the context image (which is identical
499 ads_blob_write(guc, ads.eng_state_size[guc_class],
500 real_size - LRC_SKIP_SIZE);
501 ads_blob_write(guc, ads.golden_context_lrca[guc_class],
504 addr_ggtt += alloc_size;
507 /* Make sure current size matches what we calculated previously */
508 if (guc->ads_golden_ctxt_size)
509 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
514 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
516 struct intel_engine_cs *engine;
517 enum intel_engine_id id;
519 for_each_engine(engine, gt, id) {
520 if (engine->class != engine_class)
523 if (!engine->default_state)
532 static void guc_init_golden_context(struct intel_guc *guc)
534 struct intel_engine_cs *engine;
535 struct intel_gt *gt = guc_to_gt(guc);
536 unsigned long offset;
537 u32 addr_ggtt, total_size = 0, alloc_size, real_size;
538 u8 engine_class, guc_class;
540 if (!intel_uc_uses_guc_submission(>->uc))
543 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
546 * Go back and fill in the golden context data now that it is
549 offset = guc_ads_golden_ctxt_offset(guc);
550 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
552 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
553 if (engine_class == OTHER_CLASS)
556 guc_class = engine_class_to_guc_class(engine_class);
557 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
560 real_size = intel_engine_context_size(gt, engine_class);
561 alloc_size = PAGE_ALIGN(real_size);
562 total_size += alloc_size;
564 engine = find_engine_state(gt, engine_class);
566 drm_err(>->i915->drm, "No engine state recorded for class %d!\n",
568 ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
569 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
573 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
574 real_size - LRC_SKIP_SIZE);
575 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
577 addr_ggtt += alloc_size;
579 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
581 offset += alloc_size;
584 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
587 static void guc_capture_list_init(struct intel_guc *guc)
590 u32 addr_ggtt, offset;
592 offset = guc_ads_capture_offset(guc);
593 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
595 /* FIXME: Populate a proper capture list */
597 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
598 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
599 ads_blob_write(guc, ads.capture_instance[i][j], addr_ggtt);
600 ads_blob_write(guc, ads.capture_class[i][j], addr_ggtt);
603 ads_blob_write(guc, ads.capture_global[i], addr_ggtt);
607 static void __guc_ads_init(struct intel_guc *guc)
609 struct intel_gt *gt = guc_to_gt(guc);
610 struct drm_i915_private *i915 = gt->i915;
611 struct __guc_ads_blob *blob = guc->ads_blob;
612 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
613 offsetof(struct __guc_ads_blob, system_info));
616 /* GuC scheduling policies */
617 guc_policies_init(guc);
620 fill_engine_enable_masks(gt, &info_map);
622 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] =
623 hweight8(gt->info.sseu.slice_mask);
624 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] =
625 gt->info.vdbox_sfc_access;
627 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
628 u32 distdbreg = intel_uncore_read(gt->uncore,
629 GEN12_DIST_DBS_POPULATED);
630 blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] =
631 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) &
632 GEN12_DOORBELLS_PER_SQIDI) + 1;
635 /* Golden contexts for re-initialising after a watchdog reset */
636 guc_prep_golden_context(guc);
638 guc_mapping_table_init(guc_to_gt(guc), &info_map);
640 base = intel_guc_ggtt_offset(guc, guc->ads_vma);
642 /* Capture list for hang debug */
643 guc_capture_list_init(guc);
646 blob->ads.scheduler_policies = base + ptr_offset(blob, policies);
647 blob->ads.gt_system_info = base + ptr_offset(blob, system_info);
649 /* MMIO save/restore list */
650 guc_mmio_reg_state_init(guc);
653 blob->ads.private_data = base + guc_ads_private_data_offset(guc);
655 i915_gem_object_flush_map(guc->ads_vma->obj);
659 * intel_guc_ads_create() - allocates and initializes GuC ADS.
660 * @guc: intel_guc struct
662 * GuC needs memory block (Additional Data Struct), where it will store
663 * some data. Allocate and initialize such memory block for GuC use.
665 int intel_guc_ads_create(struct intel_guc *guc)
670 GEM_BUG_ON(guc->ads_vma);
673 * Create reg state size dynamically on system memory to be copied to
674 * the final ads blob on gt init/reset
676 ret = guc_mmio_reg_state_create(guc);
679 guc->ads_regset_size = ret;
681 /* Likewise the golden contexts: */
682 ret = guc_prep_golden_context(guc);
685 guc->ads_golden_ctxt_size = ret;
687 /* Now the total size can be determined: */
688 size = guc_ads_blob_size(guc);
690 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
691 (void **)&guc->ads_blob);
695 if (i915_gem_object_is_lmem(guc->ads_vma->obj))
696 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)guc->ads_blob);
698 iosys_map_set_vaddr(&guc->ads_map, guc->ads_blob);
705 void intel_guc_ads_init_late(struct intel_guc *guc)
708 * The golden context setup requires the saved engine state from
709 * __engines_record_defaults(). However, that requires engines to be
710 * operational which means the ADS must already have been configured.
711 * Fortunately, the golden context state is not needed until a hang
712 * occurs, so it can be filled in during this late init phase.
714 guc_init_golden_context(guc);
717 void intel_guc_ads_destroy(struct intel_guc *guc)
719 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
720 guc->ads_blob = NULL;
721 iosys_map_clear(&guc->ads_map);
722 kfree(guc->ads_regset);
725 static void guc_ads_private_data_reset(struct intel_guc *guc)
729 size = guc_ads_private_data_size(guc);
733 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
738 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
739 * @guc: intel_guc struct
741 * GuC stores some data in ADS, which might be stale after a reset.
742 * Reinitialize whole ADS in case any part of it was corrupted during
745 void intel_guc_ads_reset(struct intel_guc *guc)
752 guc_ads_private_data_reset(guc);
755 u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
757 return intel_guc_ggtt_offset(guc, guc->ads_vma) +
758 offsetof(struct __guc_ads_blob, engine_usage);
761 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
763 struct intel_guc *guc = &engine->gt->uc.guc;
764 u8 guc_class = engine_class_to_guc_class(engine->class);
765 size_t offset = offsetof(struct __guc_ads_blob,
766 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
768 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);