1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #define pr_fmt(fmt) "habanalabs: " fmt
10 #include <uapi/misc/habanalabs.h>
11 #include "habanalabs.h"
13 #include <linux/kernel.h>
15 #include <linux/uaccess.h>
16 #include <linux/slab.h>
18 static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
19 [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
20 [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
21 [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
22 [HL_DEBUG_OP_FUNNEL] = 0,
23 [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
24 [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
25 [HL_DEBUG_OP_TIMESTAMP] = 0
29 static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
31 struct hl_info_device_status dev_stat = {0};
32 u32 size = args->return_size;
33 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
35 if ((!size) || (!out))
38 dev_stat.status = hl_device_status(hdev);
40 return copy_to_user(out, &dev_stat,
41 min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
44 static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
46 struct hl_info_hw_ip_info hw_ip = {0};
47 u32 size = args->return_size;
48 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
49 struct asic_fixed_properties *prop = &hdev->asic_prop;
50 u64 sram_kmd_size, dram_kmd_size;
52 if ((!size) || (!out))
55 sram_kmd_size = (prop->sram_user_base_address -
56 prop->sram_base_address);
57 dram_kmd_size = (prop->dram_user_base_address -
58 prop->dram_base_address);
60 hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
61 hw_ip.sram_base_address = prop->sram_user_base_address;
62 hw_ip.dram_base_address =
63 hdev->mmu_enable && prop->dram_supports_virtual_memory ?
64 prop->dmmu.start_addr : prop->dram_user_base_address;
65 hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
66 hw_ip.sram_size = prop->sram_size - sram_kmd_size;
70 DIV_ROUND_DOWN_ULL(prop->dram_size - dram_kmd_size,
71 prop->dram_page_size) *
74 hw_ip.dram_size = prop->dram_size - dram_kmd_size;
76 if (hw_ip.dram_size > PAGE_SIZE)
77 hw_ip.dram_enabled = 1;
78 hw_ip.dram_page_size = prop->dram_page_size;
79 hw_ip.num_of_events = prop->num_of_events;
81 memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version,
82 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
84 memcpy(hw_ip.card_name, prop->cpucp_info.card_name,
85 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
87 hw_ip.cpld_version = le32_to_cpu(prop->cpucp_info.cpld_version);
88 hw_ip.module_id = le32_to_cpu(prop->cpucp_info.card_location);
90 hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
91 hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
92 hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
93 hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
95 hw_ip.first_available_interrupt_id =
96 prop->first_available_user_msix_interrupt;
97 hw_ip.server_type = prop->server_type;
99 return copy_to_user(out, &hw_ip,
100 min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;
103 static int hw_events_info(struct hl_device *hdev, bool aggregate,
104 struct hl_info_args *args)
106 u32 size, max_size = args->return_size;
107 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
110 if ((!max_size) || (!out))
113 arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
115 return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
118 static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
120 struct hl_device *hdev = hpriv->hdev;
121 struct hl_info_dram_usage dram_usage = {0};
122 u32 max_size = args->return_size;
123 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
124 struct asic_fixed_properties *prop = &hdev->asic_prop;
127 if ((!max_size) || (!out))
130 dram_kmd_size = (prop->dram_user_base_address -
131 prop->dram_base_address);
132 dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
133 atomic64_read(&hdev->dram_used_mem);
135 dram_usage.ctx_dram_mem =
136 atomic64_read(&hpriv->ctx->dram_phys_mem);
138 return copy_to_user(out, &dram_usage,
139 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
142 static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
144 struct hl_info_hw_idle hw_idle = {0};
145 u32 max_size = args->return_size;
146 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
148 if ((!max_size) || (!out))
151 hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
152 hw_idle.busy_engines_mask_ext,
153 HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL);
154 hw_idle.busy_engines_mask =
155 lower_32_bits(hw_idle.busy_engines_mask_ext[0]);
157 return copy_to_user(out, &hw_idle,
158 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
161 static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
163 struct hl_debug_params *params;
164 void *input = NULL, *output = NULL;
167 params = kzalloc(sizeof(*params), GFP_KERNEL);
171 params->reg_idx = args->reg_idx;
172 params->enable = args->enable;
173 params->op = args->op;
175 if (args->input_ptr && args->input_size) {
176 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
182 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
185 dev_err(hdev->dev, "failed to copy input debug data\n");
189 params->input = input;
192 if (args->output_ptr && args->output_size) {
193 output = kzalloc(args->output_size, GFP_KERNEL);
199 params->output = output;
200 params->output_size = args->output_size;
203 rc = hdev->asic_funcs->debug_coresight(hdev, params);
206 "debug coresight operation failed %d\n", rc);
210 if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr,
211 output, args->output_size)) {
212 dev_err(hdev->dev, "copy to user failed in debug ioctl\n");
226 static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
228 struct hl_info_device_utilization device_util = {0};
229 u32 max_size = args->return_size;
230 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
233 if ((!max_size) || (!out))
236 rc = hl_device_utilization(hdev, &device_util.utilization);
240 return copy_to_user(out, &device_util,
241 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
244 static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args)
246 struct hl_info_clk_rate clk_rate = {0};
247 u32 max_size = args->return_size;
248 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
251 if ((!max_size) || (!out))
254 rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz,
255 &clk_rate.max_clk_rate_mhz);
259 return copy_to_user(out, &clk_rate,
260 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0;
263 static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
265 struct hl_info_reset_count reset_count = {0};
266 u32 max_size = args->return_size;
267 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
269 if ((!max_size) || (!out))
272 reset_count.hard_reset_cnt = hdev->hard_reset_cnt;
273 reset_count.soft_reset_cnt = hdev->soft_reset_cnt;
275 return copy_to_user(out, &reset_count,
276 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
279 static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
281 struct hl_info_time_sync time_sync = {0};
282 u32 max_size = args->return_size;
283 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
285 if ((!max_size) || (!out))
288 time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
289 time_sync.host_time = ktime_get_raw_ns();
291 return copy_to_user(out, &time_sync,
292 min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
295 static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
297 struct hl_device *hdev = hpriv->hdev;
298 struct hl_info_pci_counters pci_counters = {0};
299 u32 max_size = args->return_size;
300 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
303 if ((!max_size) || (!out))
306 rc = hl_fw_cpucp_pci_counters_get(hdev, &pci_counters);
310 return copy_to_user(out, &pci_counters,
311 min((size_t) max_size, sizeof(pci_counters))) ? -EFAULT : 0;
314 static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
316 struct hl_device *hdev = hpriv->hdev;
317 struct hl_info_clk_throttle clk_throttle = {0};
318 u32 max_size = args->return_size;
319 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
321 if ((!max_size) || (!out))
324 clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason;
326 return copy_to_user(out, &clk_throttle,
327 min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0;
330 static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
332 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
333 struct hl_info_cs_counters cs_counters = {0};
334 struct hl_device *hdev = hpriv->hdev;
335 struct hl_cs_counters_atomic *cntr;
336 u32 max_size = args->return_size;
338 cntr = &hdev->aggregated_cs_counters;
340 if ((!max_size) || (!out))
343 cs_counters.total_out_of_mem_drop_cnt =
344 atomic64_read(&cntr->out_of_mem_drop_cnt);
345 cs_counters.total_parsing_drop_cnt =
346 atomic64_read(&cntr->parsing_drop_cnt);
347 cs_counters.total_queue_full_drop_cnt =
348 atomic64_read(&cntr->queue_full_drop_cnt);
349 cs_counters.total_device_in_reset_drop_cnt =
350 atomic64_read(&cntr->device_in_reset_drop_cnt);
351 cs_counters.total_max_cs_in_flight_drop_cnt =
352 atomic64_read(&cntr->max_cs_in_flight_drop_cnt);
353 cs_counters.total_validation_drop_cnt =
354 atomic64_read(&cntr->validation_drop_cnt);
357 cs_counters.ctx_out_of_mem_drop_cnt =
359 &hpriv->ctx->cs_counters.out_of_mem_drop_cnt);
360 cs_counters.ctx_parsing_drop_cnt =
362 &hpriv->ctx->cs_counters.parsing_drop_cnt);
363 cs_counters.ctx_queue_full_drop_cnt =
365 &hpriv->ctx->cs_counters.queue_full_drop_cnt);
366 cs_counters.ctx_device_in_reset_drop_cnt =
368 &hpriv->ctx->cs_counters.device_in_reset_drop_cnt);
369 cs_counters.ctx_max_cs_in_flight_drop_cnt =
371 &hpriv->ctx->cs_counters.max_cs_in_flight_drop_cnt);
372 cs_counters.ctx_validation_drop_cnt =
374 &hpriv->ctx->cs_counters.validation_drop_cnt);
377 return copy_to_user(out, &cs_counters,
378 min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0;
381 static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
383 struct hl_device *hdev = hpriv->hdev;
384 struct asic_fixed_properties *prop = &hdev->asic_prop;
385 struct hl_info_sync_manager sm_info = {0};
386 u32 max_size = args->return_size;
387 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
389 if ((!max_size) || (!out))
392 if (args->dcore_id >= HL_MAX_DCORES)
395 sm_info.first_available_sync_object =
396 prop->first_available_user_sob[args->dcore_id];
397 sm_info.first_available_monitor =
398 prop->first_available_user_mon[args->dcore_id];
399 sm_info.first_available_cq =
400 prop->first_available_cq[args->dcore_id];
402 return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size,
403 sizeof(sm_info))) ? -EFAULT : 0;
406 static int total_energy_consumption_info(struct hl_fpriv *hpriv,
407 struct hl_info_args *args)
409 struct hl_device *hdev = hpriv->hdev;
410 struct hl_info_energy total_energy = {0};
411 u32 max_size = args->return_size;
412 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
415 if ((!max_size) || (!out))
418 rc = hl_fw_cpucp_total_energy_get(hdev,
419 &total_energy.total_energy_consumption);
423 return copy_to_user(out, &total_energy,
424 min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0;
427 static int pll_frequency_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
429 struct hl_device *hdev = hpriv->hdev;
430 struct hl_pll_frequency_info freq_info = { {0} };
431 u32 max_size = args->return_size;
432 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
435 if ((!max_size) || (!out))
438 rc = hl_fw_cpucp_pll_info_get(hdev, args->pll_index, freq_info.output);
442 return copy_to_user(out, &freq_info,
443 min((size_t) max_size, sizeof(freq_info))) ? -EFAULT : 0;
446 static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
448 struct hl_device *hdev = hpriv->hdev;
449 u32 max_size = args->return_size;
450 struct hl_power_info power_info = {0};
451 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
454 if ((!max_size) || (!out))
457 rc = hl_fw_cpucp_power_get(hdev, &power_info.power);
461 return copy_to_user(out, &power_info,
462 min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0;
465 static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
467 struct hl_device *hdev = hpriv->hdev;
468 u32 max_size = args->return_size;
469 struct hl_open_stats_info open_stats_info = {0};
470 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
472 if ((!max_size) || (!out))
475 open_stats_info.last_open_period_ms = jiffies64_to_msecs(
476 hdev->last_open_session_duration_jif);
477 open_stats_info.open_counter = hdev->open_counter;
479 return copy_to_user(out, &open_stats_info,
480 min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0;
483 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
486 enum hl_device_status status;
487 struct hl_info_args *args = data;
488 struct hl_device *hdev = hpriv->hdev;
493 * Information is returned for the following opcodes even if the device
494 * is disabled or in reset.
497 case HL_INFO_HW_IP_INFO:
498 return hw_ip_info(hdev, args);
500 case HL_INFO_DEVICE_STATUS:
501 return device_status_info(hdev, args);
503 case HL_INFO_RESET_COUNT:
504 return get_reset_count(hdev, args);
510 if (!hl_device_operational(hdev, &status)) {
511 dev_warn_ratelimited(dev,
512 "Device is %s. Can't execute INFO IOCTL\n",
513 hdev->status[status]);
518 case HL_INFO_HW_EVENTS:
519 rc = hw_events_info(hdev, false, args);
522 case HL_INFO_DRAM_USAGE:
523 rc = dram_usage_info(hpriv, args);
526 case HL_INFO_HW_IDLE:
527 rc = hw_idle(hdev, args);
530 case HL_INFO_DEVICE_UTILIZATION:
531 rc = device_utilization(hdev, args);
534 case HL_INFO_HW_EVENTS_AGGREGATE:
535 rc = hw_events_info(hdev, true, args);
538 case HL_INFO_CLK_RATE:
539 rc = get_clk_rate(hdev, args);
542 case HL_INFO_TIME_SYNC:
543 return time_sync_info(hdev, args);
545 case HL_INFO_CS_COUNTERS:
546 return cs_counters_info(hpriv, args);
548 case HL_INFO_PCI_COUNTERS:
549 return pci_counters_info(hpriv, args);
551 case HL_INFO_CLK_THROTTLE_REASON:
552 return clk_throttle_info(hpriv, args);
554 case HL_INFO_SYNC_MANAGER:
555 return sync_manager_info(hpriv, args);
557 case HL_INFO_TOTAL_ENERGY:
558 return total_energy_consumption_info(hpriv, args);
560 case HL_INFO_PLL_FREQUENCY:
561 return pll_frequency_info(hpriv, args);
564 return power_info(hpriv, args);
566 case HL_INFO_OPEN_STATS:
567 return open_stats_info(hpriv, args);
570 dev_err(dev, "Invalid request %d\n", args->op);
578 static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
580 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
583 static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
585 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
588 static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
590 struct hl_debug_args *args = data;
591 struct hl_device *hdev = hpriv->hdev;
592 enum hl_device_status status;
596 if (!hl_device_operational(hdev, &status)) {
597 dev_warn_ratelimited(hdev->dev,
598 "Device is %s. Can't execute DEBUG IOCTL\n",
599 hdev->status[status]);
604 case HL_DEBUG_OP_ETR:
605 case HL_DEBUG_OP_ETF:
606 case HL_DEBUG_OP_STM:
607 case HL_DEBUG_OP_FUNNEL:
608 case HL_DEBUG_OP_BMON:
609 case HL_DEBUG_OP_SPMU:
610 case HL_DEBUG_OP_TIMESTAMP:
611 if (!hdev->in_debug) {
612 dev_err_ratelimited(hdev->dev,
613 "Rejecting debug configuration request because device not in debug mode\n");
617 min(args->input_size, hl_debug_struct_size[args->op]);
618 rc = debug_coresight(hdev, args);
620 case HL_DEBUG_OP_SET_MODE:
621 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
624 dev_err(hdev->dev, "Invalid request %d\n", args->op);
632 #define HL_IOCTL_DEF(ioctl, _func) \
633 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
635 static const struct hl_ioctl_desc hl_ioctls[] = {
636 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
637 HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
638 HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
639 HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_wait_ioctl),
640 HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
641 HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
644 static const struct hl_ioctl_desc hl_ioctls_control[] = {
645 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
648 static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
649 const struct hl_ioctl_desc *ioctl, struct device *dev)
651 struct hl_fpriv *hpriv = filep->private_data;
652 struct hl_device *hdev = hpriv->hdev;
653 unsigned int nr = _IOC_NR(cmd);
654 char stack_kdata[128] = {0};
656 unsigned int usize, asize;
661 if (hdev->hard_reset_pending) {
662 dev_crit_ratelimited(dev,
663 "Device HARD reset pending! Please close FD\n");
667 /* Do not trust userspace, use our own definition */
670 if (unlikely(!func)) {
671 dev_dbg(dev, "no function\n");
676 hl_size = _IOC_SIZE(ioctl->cmd);
677 usize = asize = _IOC_SIZE(cmd);
683 if (cmd & (IOC_IN | IOC_OUT)) {
684 if (asize <= sizeof(stack_kdata)) {
687 kdata = kzalloc(asize, GFP_KERNEL);
696 if (copy_from_user(kdata, (void __user *)arg, usize)) {
700 } else if (cmd & IOC_OUT) {
701 memset(kdata, 0, usize);
704 retcode = func(hpriv, kdata);
706 if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize))
711 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
712 task_pid_nr(current), cmd, nr);
714 if (kdata != stack_kdata)
720 long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
722 struct hl_fpriv *hpriv = filep->private_data;
723 struct hl_device *hdev = hpriv->hdev;
724 const struct hl_ioctl_desc *ioctl = NULL;
725 unsigned int nr = _IOC_NR(cmd);
728 pr_err_ratelimited("Sending ioctl after device was removed! Please close FD\n");
732 if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
733 ioctl = &hl_ioctls[nr];
735 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
736 task_pid_nr(current), nr);
740 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
743 long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
745 struct hl_fpriv *hpriv = filep->private_data;
746 struct hl_device *hdev = hpriv->hdev;
747 const struct hl_ioctl_desc *ioctl = NULL;
748 unsigned int nr = _IOC_NR(cmd);
751 pr_err_ratelimited("Sending ioctl after device was removed! Please close FD\n");
755 if (nr == _IOC_NR(HL_IOCTL_INFO)) {
756 ioctl = &hl_ioctls_control[nr];
758 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
759 task_pid_nr(current), nr);
763 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);