drivers/misc/habanalabs/common/state_dump.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2021 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <linux/vmalloc.h>
   9 #include <uapi/misc/habanalabs.h>
  10 #include "habanalabs.h"
  11
  12 /**
  13  * hl_format_as_binary - helper function, format an integer as binary
  14  *                       using supplied scratch buffer
  15  * @buf: the buffer to use
  16  * @buf_len: buffer capacity
  17  * @n: number to format
  18  *
  19  * Returns pointer to buffer
  20  */
  21 char *hl_format_as_binary(char *buf, size_t buf_len, u32 n)
  22 {
  23         int i;
  24         u32 bit;
  25         bool leading0 = true;
  26         char *wrptr = buf;
  27
  28         if (buf_len > 0 && buf_len < 3) {
  29                 *wrptr = '\0';
  30                 return buf;
  31         }
  32
  33         wrptr[0] = '0';
  34         wrptr[1] = 'b';
  35         wrptr += 2;
  36         /* Remove 3 characters from length for '0b' and '\0' termination */
  37         buf_len -= 3;
  38
  39         for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {
  40                 /* Writing bit calculation in one line would cause a false
  41                  * positive static code analysis error, so splitting.
  42                  */
  43                 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));
  44                 bit = !!bit;
  45                 leading0 &= !bit;
  46                 if (!leading0) {
  47                         *wrptr = '0' + bit;
  48                         ++wrptr;
  49                 }
  50         }
  51
  52         *wrptr = '\0';
  53
  54         return buf;
  55 }
  56
  57 /**
  58  * resize_to_fit - helper function, resize buffer to fit given amount of data
  59  * @buf: destination buffer double pointer
  60  * @size: pointer to the size container
  61  * @desired_size: size the buffer must contain
  62  *
  63  * Returns 0 on success or error code on failure.
  64  * On success, the size of buffer is at least desired_size. Buffer is allocated
  65  * via vmalloc and must be freed with vfree.
  66  */
  67 static int resize_to_fit(char **buf, size_t *size, size_t desired_size)
  68 {
  69         char *resized_buf;
  70         size_t new_size;
  71
  72         if (*size >= desired_size)
  73                 return 0;
  74
  75         /* Not enough space to print all, have to resize */
  76         new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));
  77         resized_buf = vmalloc(new_size);
  78         if (!resized_buf)
  79                 return -ENOMEM;
  80         memcpy(resized_buf, *buf, *size);
  81         vfree(*buf);
  82         *buf = resized_buf;
  83         *size = new_size;
  84
  85         return 1;
  86 }
  87
  88 /**
  89  * hl_snprintf_resize() - print formatted data to buffer, resize as needed
  90  * @buf: buffer double pointer, to be written to and resized, must be either
  91  *       NULL or allocated with vmalloc.
  92  * @size: current size of the buffer
  93  * @offset: current offset to write to
  94  * @format: format of the data
  95  *
  96  * This function will write formatted data into the buffer. If buffer is not
  97  * large enough, it will be resized using vmalloc. Size may be modified if the
  98  * buffer was resized, offset will be advanced by the number of bytes written
  99  * not including the terminating character
 100  *
 101  * Returns 0 on success or error code on failure
 102  *
 103  * Note that the buffer has to be manually released using vfree.
 104  */
 105 int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
 106                            const char *format, ...)
 107 {
 108         va_list args;
 109         size_t length;
 110         int rc;
 111
 112         if (*buf == NULL && (*size != 0 || *offset != 0))
 113                 return -EINVAL;
 114
 115         va_start(args, format);
 116         length = vsnprintf(*buf + *offset, *size - *offset, format, args);
 117         va_end(args);
 118
 119         rc = resize_to_fit(buf, size, *offset + length + 1);
 120         if (rc < 0)
 121                 return rc;
 122         else if (rc > 0) {
 123                 /* Resize was needed, write again */
 124                 va_start(args, format);
 125                 length = vsnprintf(*buf + *offset, *size - *offset, format,
 126                                    args);
 127                 va_end(args);
 128         }
 129
 130         *offset += length;
 131
 132         return 0;
 133 }
 134
 135 /**
 136  * hl_sync_engine_to_string - convert engine type enum to string literal
 137  * @engine_type: engine type (TPC/MME/DMA)
 138  *
 139  * Return the resolved string literal
 140  */
 141 const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)
 142 {
 143         switch (engine_type) {
 144         case ENGINE_DMA:
 145                 return "DMA";
 146         case ENGINE_MME:
 147                 return "MME";
 148         case ENGINE_TPC:
 149                 return "TPC";
 150         }
 151         return "Invalid Engine Type";
 152 }
 153
 154 /**
 155  * hl_print_resize_sync_engine - helper function, format engine name and ID
 156  * using hl_snprintf_resize
 157  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 158  * @size: pointer to the size container
 159  * @offset: pointer to the offset container
 160  * @engine_type: engine type (TPC/MME/DMA)
 161  * @engine_id: engine numerical id
 162  *
 163  * Returns 0 on success or error code on failure
 164  */
 165 static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset,
 166                                 enum hl_sync_engine_type engine_type,
 167                                 u32 engine_id)
 168 {
 169         return hl_snprintf_resize(buf, size, offset, "%s%u",
 170                         hl_sync_engine_to_string(engine_type), engine_id);
 171 }
 172
 173 /**
 174  * hl_state_dump_get_sync_name - transform sync object id to name if available
 175  * @hdev: pointer to the device
 176  * @sync_id: sync object id
 177  *
 178  * Returns a name literal or NULL if not resolved.
 179  * Note: returning NULL shall not be considered as a failure, as not all
 180  * sync objects are named.
 181  */
 182 const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id)
 183 {
 184         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 185         struct hl_hw_obj_name_entry *entry;
 186
 187         hash_for_each_possible(sds->so_id_to_str_tb, entry,
 188                                 node, sync_id)
 189                 if (sync_id == entry->id)
 190                         return entry->name;
 191
 192         return NULL;
 193 }
 194
 195 /**
 196  * hl_state_dump_get_monitor_name - transform monitor object dump to monitor
 197  * name if available
 198  * @hdev: pointer to the device
 199  * @mon: monitor state dump
 200  *
 201  * Returns a name literal or NULL if not resolved.
 202  * Note: returning NULL shall not be considered as a failure, as not all
 203  * monitors are named.
 204  */
 205 const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
 206                                         struct hl_mon_state_dump *mon)
 207 {
 208         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 209         struct hl_hw_obj_name_entry *entry;
 210
 211         hash_for_each_possible(sds->monitor_id_to_str_tb,
 212                                 entry, node, mon->id)
 213                 if (mon->id == entry->id)
 214                         return entry->name;
 215
 216         return NULL;
 217 }
 218
 219 /**
 220  * hl_state_dump_free_sync_to_engine_map - free sync object to engine map
 221  * @map: sync object to engine map
 222  *
 223  * Note: generic free implementation, the allocation is implemented per ASIC.
 224  */
 225 void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)
 226 {
 227         struct hl_sync_to_engine_map_entry *entry;
 228         struct hlist_node *tmp_node;
 229         int i;
 230
 231         hash_for_each_safe(map->tb, i, tmp_node, entry, node) {
 232                 hash_del(&entry->node);
 233                 kfree(entry);
 234         }
 235 }
 236
 237 /**
 238  * hl_state_dump_get_sync_to_engine - transform sync_id to
 239  * hl_sync_to_engine_map_entry if available for current id
 240  * @map: sync object to engine map
 241  * @sync_id: sync object id
 242  *
 243  * Returns the translation entry if found or NULL if not.
 244  * Note, returned NULL shall not be considered as a failure as the map
 245  * does not cover all possible, it is a best effort sync ids.
 246  */
 247 static struct hl_sync_to_engine_map_entry *
 248 hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)
 249 {
 250         struct hl_sync_to_engine_map_entry *entry;
 251
 252         hash_for_each_possible(map->tb, entry, node, sync_id)
 253                 if (entry->sync_id == sync_id)
 254                         return entry;
 255         return NULL;
 256 }
 257
 258 /**
 259  * hl_state_dump_read_sync_objects - read sync objects array
 260  * @hdev: pointer to the device
 261  * @index: sync manager block index starting with E_N
 262  *
 263  * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure
 264  */
 265 static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index)
 266 {
 267         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 268         u32 *sync_objects;
 269         s64 base_addr; /* Base addr can be negative */
 270         int i;
 271
 272         base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
 273                         sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
 274
 275         sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));
 276         if (!sync_objects)
 277                 return NULL;
 278
 279         for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)
 280                 sync_objects[i] = RREG32(base_addr + i * sizeof(u32));
 281
 282         return sync_objects;
 283 }
 284
 285 /**
 286  * hl_state_dump_free_sync_objects - free sync objects array allocated by
 287  * hl_state_dump_read_sync_objects
 288  * @sync_objects: sync objects array
 289  */
 290 static void hl_state_dump_free_sync_objects(u32 *sync_objects)
 291 {
 292         vfree(sync_objects);
 293 }
 294
 295
 296 /**
 297  * hl_state_dump_print_syncs_single_block - print active sync objects on a
 298  * single block
 299  * @hdev: pointer to the device
 300  * @index: sync manager block index starting with E_N
 301  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 302  * @size: pointer to the size container
 303  * @offset: pointer to the offset container
 304  * @map: sync engines names map
 305  *
 306  * Returns 0 on success or error code on failure
 307  */
 308 static int
 309 hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,
 310                                 char **buf, size_t *size, size_t *offset,
 311                                 struct hl_sync_to_engine_map *map)
 312 {
 313         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 314         const char *sync_name;
 315         u32 *sync_objects = NULL;
 316         int rc = 0, i;
 317
 318         if (sds->sync_namager_names) {
 319                 rc = hl_snprintf_resize(
 320                         buf, size, offset, "%s\n",
 321                         sds->sync_namager_names[index]);
 322                 if (rc)
 323                         goto out;
 324         }
 325
 326         sync_objects = hl_state_dump_read_sync_objects(hdev, index);
 327         if (!sync_objects) {
 328                 rc = -ENOMEM;
 329                 goto out;
 330         }
 331
 332         for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {
 333                 struct hl_sync_to_engine_map_entry *entry;
 334                 u64 sync_object_addr;
 335
 336                 if (!sync_objects[i])
 337                         continue;
 338
 339                 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
 340                                 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +
 341                                 i * sizeof(u32);
 342
 343                 rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i);
 344                 if (rc)
 345                         goto free_sync_objects;
 346                 sync_name = hl_state_dump_get_sync_name(hdev, i);
 347                 if (sync_name) {
 348                         rc = hl_snprintf_resize(buf, size, offset, " %s",
 349                                                 sync_name);
 350                         if (rc)
 351                                 goto free_sync_objects;
 352                 }
 353                 rc = hl_snprintf_resize(buf, size, offset, ", value: %u",
 354                                         sync_objects[i]);
 355                 if (rc)
 356                         goto free_sync_objects;
 357
 358                 /* Append engine string */
 359                 entry = hl_state_dump_get_sync_to_engine(map,
 360                         (u32)sync_object_addr);
 361                 if (entry) {
 362                         rc = hl_snprintf_resize(buf, size, offset,
 363                                                 ", Engine: ");
 364                         if (rc)
 365                                 goto free_sync_objects;
 366                         rc = hl_print_resize_sync_engine(buf, size, offset,
 367                                                 entry->engine_type,
 368                                                 entry->engine_id);
 369                         if (rc)
 370                                 goto free_sync_objects;
 371                 }
 372
 373                 rc = hl_snprintf_resize(buf, size, offset, "\n");
 374                 if (rc)
 375                         goto free_sync_objects;
 376         }
 377
 378 free_sync_objects:
 379         hl_state_dump_free_sync_objects(sync_objects);
 380 out:
 381         return rc;
 382 }
 383
 384 /**
 385  * hl_state_dump_print_syncs - print active sync objects
 386  * @hdev: pointer to the device
 387  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 388  * @size: pointer to the size container
 389  * @offset: pointer to the offset container
 390  *
 391  * Returns 0 on success or error code on failure
 392  */
 393 static int hl_state_dump_print_syncs(struct hl_device *hdev,
 394                                         char **buf, size_t *size,
 395                                         size_t *offset)
 396
 397 {
 398         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 399         struct hl_sync_to_engine_map *map;
 400         u32 index;
 401         int rc = 0;
 402
 403         map = kzalloc(sizeof(*map), GFP_KERNEL);
 404         if (!map)
 405                 return -ENOMEM;
 406
 407         rc = sds->funcs.gen_sync_to_engine_map(hdev, map);
 408         if (rc)
 409                 goto free_map_mem;
 410
 411         rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n");
 412         if (rc)
 413                 goto out;
 414
 415         if (sds->sync_namager_names) {
 416                 for (index = 0; sds->sync_namager_names[index]; ++index) {
 417                         rc = hl_state_dump_print_syncs_single_block(
 418                                 hdev, index, buf, size, offset, map);
 419                         if (rc)
 420                                 goto out;
 421                 }
 422         } else {
 423                 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
 424                         rc = hl_state_dump_print_syncs_single_block(
 425                                 hdev, index, buf, size, offset, map);
 426                         if (rc)
 427                                 goto out;
 428                 }
 429         }
 430
 431 out:
 432         hl_state_dump_free_sync_to_engine_map(map);
 433 free_map_mem:
 434         kfree(map);
 435
 436         return rc;
 437 }
 438
 439 /**
 440  * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific
 441  * block
 442  * @hdev: pointer to the device
 443  * @index: sync manager block index starting with E_N
 444  *
 445  * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL
 446  * on error
 447  */
 448 static struct hl_mon_state_dump *
 449 hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)
 450 {
 451         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 452         struct hl_mon_state_dump *monitors;
 453         s64 base_addr; /* Base addr can be negative */
 454         int i;
 455
 456         monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *
 457                            sizeof(struct hl_mon_state_dump));
 458         if (!monitors)
 459                 return NULL;
 460
 461         base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
 462
 463         for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
 464                 monitors[i].id = i;
 465                 monitors[i].wr_addr_low =
 466                         RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +
 467                                 i * sizeof(u32));
 468
 469                 monitors[i].wr_addr_high =
 470                         RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +
 471                                 i * sizeof(u32));
 472
 473                 monitors[i].wr_data =
 474                         RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +
 475                                 i * sizeof(u32));
 476
 477                 monitors[i].arm_data =
 478                         RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +
 479                                 i * sizeof(u32));
 480
 481                 monitors[i].status =
 482                         RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +
 483                                 i * sizeof(u32));
 484         }
 485
 486         return monitors;
 487 }
 488
 489 /**
 490  * hl_state_dump_free_monitors - free the monitors structure
 491  * @monitors: monitors array created with
 492  *            hl_state_dump_alloc_read_sm_block_monitors
 493  */
 494 static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)
 495 {
 496         vfree(monitors);
 497 }
 498
 499 /**
 500  * hl_state_dump_print_monitors_single_block - print active monitors on a
 501  * single block
 502  * @hdev: pointer to the device
 503  * @index: sync manager block index starting with E_N
 504  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 505  * @size: pointer to the size container
 506  * @offset: pointer to the offset container
 507  *
 508  * Returns 0 on success or error code on failure
 509  */
 510 static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,
 511                                                 u32 index,
 512                                                 char **buf, size_t *size,
 513                                                 size_t *offset)
 514 {
 515         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 516         struct hl_mon_state_dump *monitors = NULL;
 517         int rc = 0, i;
 518
 519         if (sds->sync_namager_names) {
 520                 rc = hl_snprintf_resize(
 521                         buf, size, offset, "%s\n",
 522                         sds->sync_namager_names[index]);
 523                 if (rc)
 524                         goto out;
 525         }
 526
 527         monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);
 528         if (!monitors) {
 529                 rc = -ENOMEM;
 530                 goto out;
 531         }
 532
 533         for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
 534                 if (!(sds->funcs.monitor_valid(&monitors[i])))
 535                         continue;
 536
 537                 /* Monitor is valid, dump it */
 538                 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,
 539                                                         &monitors[i]);
 540                 if (rc)
 541                         goto free_monitors;
 542
 543                 hl_snprintf_resize(buf, size, offset, "\n");
 544         }
 545
 546 free_monitors:
 547         hl_state_dump_free_monitors(monitors);
 548 out:
 549         return rc;
 550 }
 551
 552 /**
 553  * hl_state_dump_print_monitors - print active monitors
 554  * @hdev: pointer to the device
 555  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 556  * @size: pointer to the size container
 557  * @offset: pointer to the offset container
 558  *
 559  * Returns 0 on success or error code on failure
 560  */
 561 static int hl_state_dump_print_monitors(struct hl_device *hdev,
 562                                         char **buf, size_t *size,
 563                                         size_t *offset)
 564 {
 565         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 566         u32 index;
 567         int rc = 0;
 568
 569         rc = hl_snprintf_resize(buf, size, offset,
 570                 "Valid (armed) monitor objects:\n");
 571         if (rc)
 572                 goto out;
 573
 574         if (sds->sync_namager_names) {
 575                 for (index = 0; sds->sync_namager_names[index]; ++index) {
 576                         rc = hl_state_dump_print_monitors_single_block(
 577                                 hdev, index, buf, size, offset);
 578                         if (rc)
 579                                 goto out;
 580                 }
 581         } else {
 582                 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
 583                         rc = hl_state_dump_print_monitors_single_block(
 584                                 hdev, index, buf, size, offset);
 585                         if (rc)
 586                                 goto out;
 587                 }
 588         }
 589
 590 out:
 591         return rc;
 592 }
 593
 594 /**
 595  * hl_state_dump_print_engine_fences - print active fences for a specific
 596  * engine
 597  * @hdev: pointer to the device
 598  * @engine_type: engine type to use
 599  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 600  * @size: pointer to the size container
 601  * @offset: pointer to the offset container
 602  */
 603 static int
 604 hl_state_dump_print_engine_fences(struct hl_device *hdev,
 605                                   enum hl_sync_engine_type engine_type,
 606                                   char **buf, size_t *size, size_t *offset)
 607 {
 608         struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
 609         int rc = 0, i, n_fences;
 610         u64 base_addr, next_fence;
 611
 612         switch (engine_type) {
 613         case ENGINE_TPC:
 614                 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];
 615                 base_addr = sds->props[SP_TPC0_CMDQ];
 616                 next_fence = sds->props[SP_NEXT_TPC];
 617                 break;
 618         case ENGINE_MME:
 619                 n_fences = sds->props[SP_NUM_OF_MME_ENGINES];
 620                 base_addr = sds->props[SP_MME_CMDQ];
 621                 next_fence = sds->props[SP_NEXT_MME];
 622                 break;
 623         case ENGINE_DMA:
 624                 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];
 625                 base_addr = sds->props[SP_DMA_CMDQ];
 626                 next_fence = sds->props[SP_DMA_QUEUES_OFFSET];
 627                 break;
 628         default:
 629                 return -EINVAL;
 630         }
 631         for (i = 0; i < n_fences; ++i) {
 632                 rc = sds->funcs.print_fences_single_engine(
 633                         hdev,
 634                         base_addr + next_fence * i +
 635                                 sds->props[SP_FENCE0_CNT_OFFSET],
 636                         base_addr + next_fence * i +
 637                                 sds->props[SP_CP_STS_OFFSET],
 638                         engine_type, i, buf, size, offset);
 639                 if (rc)
 640                         goto out;
 641         }
 642 out:
 643         return rc;
 644 }
 645
 646 /**
 647  * hl_state_dump_print_fences - print active fences
 648  * @hdev: pointer to the device
 649  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
 650  * @size: pointer to the size container
 651  * @offset: pointer to the offset container
 652  */
 653 static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf,
 654                                       size_t *size, size_t *offset)
 655 {
 656         int rc = 0;
 657
 658         rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n");
 659         if (rc)
 660                 goto out;
 661
 662         rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset);
 663         if (rc)
 664                 goto out;
 665
 666         rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset);
 667         if (rc)
 668                 goto out;
 669
 670         rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset);
 671         if (rc)
 672                 goto out;
 673
 674 out:
 675         return rc;
 676 }
 677
 678 /**
 679  * hl_state_dump() - dump system state
 680  * @hdev: pointer to device structure
 681  */
 682 int hl_state_dump(struct hl_device *hdev)
 683 {
 684         char *buf = NULL;
 685         size_t offset = 0, size = 0;
 686         int rc;
 687
 688         rc = hl_snprintf_resize(&buf, &size, &offset,
 689                                 "Timestamp taken on: %llu\n\n",
 690                                 ktime_to_ns(ktime_get()));
 691         if (rc)
 692                 goto err;
 693
 694         rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset);
 695         if (rc)
 696                 goto err;
 697
 698         hl_snprintf_resize(&buf, &size, &offset, "\n");
 699
 700         rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset);
 701         if (rc)
 702                 goto err;
 703
 704         hl_snprintf_resize(&buf, &size, &offset, "\n");
 705
 706         rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset);
 707         if (rc)
 708                 goto err;
 709
 710         hl_snprintf_resize(&buf, &size, &offset, "\n");
 711
 712         hl_debugfs_set_state_dump(hdev, buf, size);
 713
 714         return 0;
 715 err:
 716         vfree(buf);
 717         return rc;
 718 }