1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2021 HabanaLabs, Ltd.
8 #include <linux/vmalloc.h>
9 #include <uapi/misc/habanalabs.h>
10 #include "habanalabs.h"
13 * hl_format_as_binary - helper function, format an integer as binary
14 * using supplied scratch buffer
15 * @buf: the buffer to use
16 * @buf_len: buffer capacity
17 * @n: number to format
19 * Returns pointer to buffer
21 char *hl_format_as_binary(char *buf, size_t buf_len, u32 n)
28 if (buf_len > 0 && buf_len < 3) {
36 /* Remove 3 characters from length for '0b' and '\0' termination */
39 for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {
40 /* Writing bit calculation in one line would cause a false
41 * positive static code analysis error, so splitting.
43 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));
58 * resize_to_fit - helper function, resize buffer to fit given amount of data
59 * @buf: destination buffer double pointer
60 * @size: pointer to the size container
61 * @desired_size: size the buffer must contain
63 * Returns 0 on success or error code on failure.
64 * On success, the size of buffer is at least desired_size. Buffer is allocated
65 * via vmalloc and must be freed with vfree.
67 static int resize_to_fit(char **buf, size_t *size, size_t desired_size)
72 if (*size >= desired_size)
75 /* Not enough space to print all, have to resize */
76 new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));
77 resized_buf = vmalloc(new_size);
80 memcpy(resized_buf, *buf, *size);
89 * hl_snprintf_resize() - print formatted data to buffer, resize as needed
90 * @buf: buffer double pointer, to be written to and resized, must be either
91 * NULL or allocated with vmalloc.
92 * @size: current size of the buffer
93 * @offset: current offset to write to
94 * @format: format of the data
96 * This function will write formatted data into the buffer. If buffer is not
97 * large enough, it will be resized using vmalloc. Size may be modified if the
98 * buffer was resized, offset will be advanced by the number of bytes written
99 * not including the terminating character
101 * Returns 0 on success or error code on failure
103 * Note that the buffer has to be manually released using vfree.
105 int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
106 const char *format, ...)
112 if (*buf == NULL && (*size != 0 || *offset != 0))
115 va_start(args, format);
116 length = vsnprintf(*buf + *offset, *size - *offset, format, args);
119 rc = resize_to_fit(buf, size, *offset + length + 1);
123 /* Resize was needed, write again */
124 va_start(args, format);
125 length = vsnprintf(*buf + *offset, *size - *offset, format,
136 * hl_sync_engine_to_string - convert engine type enum to string literal
137 * @engine_type: engine type (TPC/MME/DMA)
139 * Return the resolved string literal
141 const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)
143 switch (engine_type) {
151 return "Invalid Engine Type";
155 * hl_print_resize_sync_engine - helper function, format engine name and ID
156 * using hl_snprintf_resize
157 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
158 * @size: pointer to the size container
159 * @offset: pointer to the offset container
160 * @engine_type: engine type (TPC/MME/DMA)
161 * @engine_id: engine numerical id
163 * Returns 0 on success or error code on failure
165 static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset,
166 enum hl_sync_engine_type engine_type,
169 return hl_snprintf_resize(buf, size, offset, "%s%u",
170 hl_sync_engine_to_string(engine_type), engine_id);
174 * hl_state_dump_get_sync_name - transform sync object id to name if available
175 * @hdev: pointer to the device
176 * @sync_id: sync object id
178 * Returns a name literal or NULL if not resolved.
179 * Note: returning NULL shall not be considered as a failure, as not all
180 * sync objects are named.
182 const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id)
184 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
185 struct hl_hw_obj_name_entry *entry;
187 hash_for_each_possible(sds->so_id_to_str_tb, entry,
189 if (sync_id == entry->id)
196 * hl_state_dump_get_monitor_name - transform monitor object dump to monitor
198 * @hdev: pointer to the device
199 * @mon: monitor state dump
201 * Returns a name literal or NULL if not resolved.
202 * Note: returning NULL shall not be considered as a failure, as not all
203 * monitors are named.
205 const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
206 struct hl_mon_state_dump *mon)
208 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
209 struct hl_hw_obj_name_entry *entry;
211 hash_for_each_possible(sds->monitor_id_to_str_tb,
212 entry, node, mon->id)
213 if (mon->id == entry->id)
220 * hl_state_dump_free_sync_to_engine_map - free sync object to engine map
221 * @map: sync object to engine map
223 * Note: generic free implementation, the allocation is implemented per ASIC.
225 void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)
227 struct hl_sync_to_engine_map_entry *entry;
228 struct hlist_node *tmp_node;
231 hash_for_each_safe(map->tb, i, tmp_node, entry, node) {
232 hash_del(&entry->node);
238 * hl_state_dump_get_sync_to_engine - transform sync_id to
239 * hl_sync_to_engine_map_entry if available for current id
240 * @map: sync object to engine map
241 * @sync_id: sync object id
243 * Returns the translation entry if found or NULL if not.
244 * Note, returned NULL shall not be considered as a failure as the map
245 * does not cover all possible, it is a best effort sync ids.
247 static struct hl_sync_to_engine_map_entry *
248 hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)
250 struct hl_sync_to_engine_map_entry *entry;
252 hash_for_each_possible(map->tb, entry, node, sync_id)
253 if (entry->sync_id == sync_id)
259 * hl_state_dump_read_sync_objects - read sync objects array
260 * @hdev: pointer to the device
261 * @index: sync manager block index starting with E_N
263 * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure
265 static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index)
267 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
269 s64 base_addr; /* Base addr can be negative */
272 base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
273 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
275 sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));
279 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)
280 sync_objects[i] = RREG32(base_addr + i * sizeof(u32));
286 * hl_state_dump_free_sync_objects - free sync objects array allocated by
287 * hl_state_dump_read_sync_objects
288 * @sync_objects: sync objects array
290 static void hl_state_dump_free_sync_objects(u32 *sync_objects)
297 * hl_state_dump_print_syncs_single_block - print active sync objects on a
299 * @hdev: pointer to the device
300 * @index: sync manager block index starting with E_N
301 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
302 * @size: pointer to the size container
303 * @offset: pointer to the offset container
304 * @map: sync engines names map
306 * Returns 0 on success or error code on failure
309 hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,
310 char **buf, size_t *size, size_t *offset,
311 struct hl_sync_to_engine_map *map)
313 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
314 const char *sync_name;
315 u32 *sync_objects = NULL;
318 if (sds->sync_namager_names) {
319 rc = hl_snprintf_resize(
320 buf, size, offset, "%s\n",
321 sds->sync_namager_names[index]);
326 sync_objects = hl_state_dump_read_sync_objects(hdev, index);
332 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {
333 struct hl_sync_to_engine_map_entry *entry;
334 u64 sync_object_addr;
336 if (!sync_objects[i])
339 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
340 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +
343 rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i);
345 goto free_sync_objects;
346 sync_name = hl_state_dump_get_sync_name(hdev, i);
348 rc = hl_snprintf_resize(buf, size, offset, " %s",
351 goto free_sync_objects;
353 rc = hl_snprintf_resize(buf, size, offset, ", value: %u",
356 goto free_sync_objects;
358 /* Append engine string */
359 entry = hl_state_dump_get_sync_to_engine(map,
360 (u32)sync_object_addr);
362 rc = hl_snprintf_resize(buf, size, offset,
365 goto free_sync_objects;
366 rc = hl_print_resize_sync_engine(buf, size, offset,
370 goto free_sync_objects;
373 rc = hl_snprintf_resize(buf, size, offset, "\n");
375 goto free_sync_objects;
379 hl_state_dump_free_sync_objects(sync_objects);
385 * hl_state_dump_print_syncs - print active sync objects
386 * @hdev: pointer to the device
387 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
388 * @size: pointer to the size container
389 * @offset: pointer to the offset container
391 * Returns 0 on success or error code on failure
393 static int hl_state_dump_print_syncs(struct hl_device *hdev,
394 char **buf, size_t *size,
398 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
399 struct hl_sync_to_engine_map *map;
403 map = kzalloc(sizeof(*map), GFP_KERNEL);
407 rc = sds->funcs.gen_sync_to_engine_map(hdev, map);
411 rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n");
415 if (sds->sync_namager_names) {
416 for (index = 0; sds->sync_namager_names[index]; ++index) {
417 rc = hl_state_dump_print_syncs_single_block(
418 hdev, index, buf, size, offset, map);
423 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
424 rc = hl_state_dump_print_syncs_single_block(
425 hdev, index, buf, size, offset, map);
432 hl_state_dump_free_sync_to_engine_map(map);
440 * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific
442 * @hdev: pointer to the device
443 * @index: sync manager block index starting with E_N
445 * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL
448 static struct hl_mon_state_dump *
449 hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)
451 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
452 struct hl_mon_state_dump *monitors;
453 s64 base_addr; /* Base addr can be negative */
456 monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *
457 sizeof(struct hl_mon_state_dump));
461 base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
463 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
465 monitors[i].wr_addr_low =
466 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +
469 monitors[i].wr_addr_high =
470 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +
473 monitors[i].wr_data =
474 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +
477 monitors[i].arm_data =
478 RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +
482 RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +
490 * hl_state_dump_free_monitors - free the monitors structure
491 * @monitors: monitors array created with
492 * hl_state_dump_alloc_read_sm_block_monitors
494 static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)
500 * hl_state_dump_print_monitors_single_block - print active monitors on a
502 * @hdev: pointer to the device
503 * @index: sync manager block index starting with E_N
504 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
505 * @size: pointer to the size container
506 * @offset: pointer to the offset container
508 * Returns 0 on success or error code on failure
510 static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,
512 char **buf, size_t *size,
515 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
516 struct hl_mon_state_dump *monitors = NULL;
519 if (sds->sync_namager_names) {
520 rc = hl_snprintf_resize(
521 buf, size, offset, "%s\n",
522 sds->sync_namager_names[index]);
527 monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);
533 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
534 if (!(sds->funcs.monitor_valid(&monitors[i])))
537 /* Monitor is valid, dump it */
538 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,
543 hl_snprintf_resize(buf, size, offset, "\n");
547 hl_state_dump_free_monitors(monitors);
553 * hl_state_dump_print_monitors - print active monitors
554 * @hdev: pointer to the device
555 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
556 * @size: pointer to the size container
557 * @offset: pointer to the offset container
559 * Returns 0 on success or error code on failure
561 static int hl_state_dump_print_monitors(struct hl_device *hdev,
562 char **buf, size_t *size,
565 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
569 rc = hl_snprintf_resize(buf, size, offset,
570 "Valid (armed) monitor objects:\n");
574 if (sds->sync_namager_names) {
575 for (index = 0; sds->sync_namager_names[index]; ++index) {
576 rc = hl_state_dump_print_monitors_single_block(
577 hdev, index, buf, size, offset);
582 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
583 rc = hl_state_dump_print_monitors_single_block(
584 hdev, index, buf, size, offset);
595 * hl_state_dump_print_engine_fences - print active fences for a specific
597 * @hdev: pointer to the device
598 * @engine_type: engine type to use
599 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
600 * @size: pointer to the size container
601 * @offset: pointer to the offset container
604 hl_state_dump_print_engine_fences(struct hl_device *hdev,
605 enum hl_sync_engine_type engine_type,
606 char **buf, size_t *size, size_t *offset)
608 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
609 int rc = 0, i, n_fences;
610 u64 base_addr, next_fence;
612 switch (engine_type) {
614 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];
615 base_addr = sds->props[SP_TPC0_CMDQ];
616 next_fence = sds->props[SP_NEXT_TPC];
619 n_fences = sds->props[SP_NUM_OF_MME_ENGINES];
620 base_addr = sds->props[SP_MME_CMDQ];
621 next_fence = sds->props[SP_NEXT_MME];
624 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];
625 base_addr = sds->props[SP_DMA_CMDQ];
626 next_fence = sds->props[SP_DMA_QUEUES_OFFSET];
631 for (i = 0; i < n_fences; ++i) {
632 rc = sds->funcs.print_fences_single_engine(
634 base_addr + next_fence * i +
635 sds->props[SP_FENCE0_CNT_OFFSET],
636 base_addr + next_fence * i +
637 sds->props[SP_CP_STS_OFFSET],
638 engine_type, i, buf, size, offset);
647 * hl_state_dump_print_fences - print active fences
648 * @hdev: pointer to the device
649 * @buf: destination buffer double pointer to be used with hl_snprintf_resize
650 * @size: pointer to the size container
651 * @offset: pointer to the offset container
653 static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf,
654 size_t *size, size_t *offset)
658 rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n");
662 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset);
666 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset);
670 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset);
679 * hl_state_dump() - dump system state
680 * @hdev: pointer to device structure
682 int hl_state_dump(struct hl_device *hdev)
685 size_t offset = 0, size = 0;
688 rc = hl_snprintf_resize(&buf, &size, &offset,
689 "Timestamp taken on: %llu\n\n",
690 ktime_to_ns(ktime_get()));
694 rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset);
698 hl_snprintf_resize(&buf, &size, &offset, "\n");
700 rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset);
704 hl_snprintf_resize(&buf, &size, &offset, "\n");
706 rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset);
710 hl_snprintf_resize(&buf, &size, &offset, "\n");
712 hl_debugfs_set_state_dump(hdev, buf, size);