drivers/misc/habanalabs/common/habanalabs_ioctl.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #define pr_fmt(fmt)     "habanalabs: " fmt
   9
  10 #include <uapi/misc/habanalabs.h>
  11 #include "habanalabs.h"
  12
  13 #include <linux/kernel.h>
  14 #include <linux/fs.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/slab.h>
  17
  18 static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
  19         [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
  20         [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
  21         [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
  22         [HL_DEBUG_OP_FUNNEL] = 0,
  23         [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
  24         [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
  25         [HL_DEBUG_OP_TIMESTAMP] = 0
  26
  27 };
  28
  29 static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
  30 {
  31         struct hl_info_device_status dev_stat = {0};
  32         u32 size = args->return_size;
  33         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  34
  35         if ((!size) || (!out))
  36                 return -EINVAL;
  37
  38         dev_stat.status = hl_device_status(hdev);
  39
  40         return copy_to_user(out, &dev_stat,
  41                         min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
  42 }
  43
  44 static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
  45 {
  46         struct hl_info_hw_ip_info hw_ip = {0};
  47         u32 size = args->return_size;
  48         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  49         struct asic_fixed_properties *prop = &hdev->asic_prop;
  50         u64 sram_kmd_size, dram_kmd_size;
  51
  52         if ((!size) || (!out))
  53                 return -EINVAL;
  54
  55         sram_kmd_size = (prop->sram_user_base_address -
  56                                 prop->sram_base_address);
  57         dram_kmd_size = (prop->dram_user_base_address -
  58                                 prop->dram_base_address);
  59
  60         hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
  61         hw_ip.sram_base_address = prop->sram_user_base_address;
  62         hw_ip.dram_base_address =
  63                         hdev->mmu_enable && prop->dram_supports_virtual_memory ?
  64                         prop->dmmu.start_addr : prop->dram_user_base_address;
  65         hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
  66         hw_ip.sram_size = prop->sram_size - sram_kmd_size;
  67
  68         if (hdev->mmu_enable)
  69                 hw_ip.dram_size =
  70                         DIV_ROUND_DOWN_ULL(prop->dram_size - dram_kmd_size,
  71                                                 prop->dram_page_size) *
  72                                                         prop->dram_page_size;
  73         else
  74                 hw_ip.dram_size = prop->dram_size - dram_kmd_size;
  75
  76         if (hw_ip.dram_size > PAGE_SIZE)
  77                 hw_ip.dram_enabled = 1;
  78         hw_ip.dram_page_size = prop->dram_page_size;
  79         hw_ip.num_of_events = prop->num_of_events;
  80
  81         memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version,
  82                 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
  83
  84         memcpy(hw_ip.card_name, prop->cpucp_info.card_name,
  85                 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
  86
  87         hw_ip.cpld_version = le32_to_cpu(prop->cpucp_info.cpld_version);
  88         hw_ip.module_id = le32_to_cpu(prop->cpucp_info.card_location);
  89
  90         hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
  91         hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
  92         hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
  93         hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
  94
  95         hw_ip.first_available_interrupt_id =
  96                         prop->first_available_user_msix_interrupt;
  97         hw_ip.server_type = prop->server_type;
  98
  99         return copy_to_user(out, &hw_ip,
 100                 min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;
 101 }
 102
 103 static int hw_events_info(struct hl_device *hdev, bool aggregate,
 104                         struct hl_info_args *args)
 105 {
 106         u32 size, max_size = args->return_size;
 107         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 108         void *arr;
 109
 110         if ((!max_size) || (!out))
 111                 return -EINVAL;
 112
 113         arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
 114
 115         return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
 116 }
 117
 118 static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 119 {
 120         struct hl_device *hdev = hpriv->hdev;
 121         struct hl_info_dram_usage dram_usage = {0};
 122         u32 max_size = args->return_size;
 123         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 124         struct asic_fixed_properties *prop = &hdev->asic_prop;
 125         u64 dram_kmd_size;
 126
 127         if ((!max_size) || (!out))
 128                 return -EINVAL;
 129
 130         dram_kmd_size = (prop->dram_user_base_address -
 131                                 prop->dram_base_address);
 132         dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
 133                                         atomic64_read(&hdev->dram_used_mem);
 134         if (hpriv->ctx)
 135                 dram_usage.ctx_dram_mem =
 136                         atomic64_read(&hpriv->ctx->dram_phys_mem);
 137
 138         return copy_to_user(out, &dram_usage,
 139                 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
 140 }
 141
 142 static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
 143 {
 144         struct hl_info_hw_idle hw_idle = {0};
 145         u32 max_size = args->return_size;
 146         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 147
 148         if ((!max_size) || (!out))
 149                 return -EINVAL;
 150
 151         hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
 152                                         hw_idle.busy_engines_mask_ext,
 153                                         HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL);
 154         hw_idle.busy_engines_mask =
 155                         lower_32_bits(hw_idle.busy_engines_mask_ext[0]);
 156
 157         return copy_to_user(out, &hw_idle,
 158                 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
 159 }
 160
 161 static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
 162 {
 163         struct hl_debug_params *params;
 164         void *input = NULL, *output = NULL;
 165         int rc;
 166
 167         params = kzalloc(sizeof(*params), GFP_KERNEL);
 168         if (!params)
 169                 return -ENOMEM;
 170
 171         params->reg_idx = args->reg_idx;
 172         params->enable = args->enable;
 173         params->op = args->op;
 174
 175         if (args->input_ptr && args->input_size) {
 176                 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
 177                 if (!input) {
 178                         rc = -ENOMEM;
 179                         goto out;
 180                 }
 181
 182                 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
 183                                         args->input_size)) {
 184                         rc = -EFAULT;
 185                         dev_err(hdev->dev, "failed to copy input debug data\n");
 186                         goto out;
 187                 }
 188
 189                 params->input = input;
 190         }
 191
 192         if (args->output_ptr && args->output_size) {
 193                 output = kzalloc(args->output_size, GFP_KERNEL);
 194                 if (!output) {
 195                         rc = -ENOMEM;
 196                         goto out;
 197                 }
 198
 199                 params->output = output;
 200                 params->output_size = args->output_size;
 201         }
 202
 203         rc = hdev->asic_funcs->debug_coresight(hdev, params);
 204         if (rc) {
 205                 dev_err(hdev->dev,
 206                         "debug coresight operation failed %d\n", rc);
 207                 goto out;
 208         }
 209
 210         if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr,
 211                                         output, args->output_size)) {
 212                 dev_err(hdev->dev, "copy to user failed in debug ioctl\n");
 213                 rc = -EFAULT;
 214                 goto out;
 215         }
 216
 217
 218 out:
 219         kfree(params);
 220         kfree(output);
 221         kfree(input);
 222
 223         return rc;
 224 }
 225
 226 static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
 227 {
 228         struct hl_info_device_utilization device_util = {0};
 229         u32 max_size = args->return_size;
 230         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 231         int rc;
 232
 233         if ((!max_size) || (!out))
 234                 return -EINVAL;
 235
 236         rc = hl_device_utilization(hdev, &device_util.utilization);
 237         if (rc)
 238                 return -EINVAL;
 239
 240         return copy_to_user(out, &device_util,
 241                 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
 242 }
 243
 244 static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args)
 245 {
 246         struct hl_info_clk_rate clk_rate = {0};
 247         u32 max_size = args->return_size;
 248         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 249         int rc;
 250
 251         if ((!max_size) || (!out))
 252                 return -EINVAL;
 253
 254         rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz,
 255                                                 &clk_rate.max_clk_rate_mhz);
 256         if (rc)
 257                 return rc;
 258
 259         return copy_to_user(out, &clk_rate,
 260                 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0;
 261 }
 262
 263 static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
 264 {
 265         struct hl_info_reset_count reset_count = {0};
 266         u32 max_size = args->return_size;
 267         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 268
 269         if ((!max_size) || (!out))
 270                 return -EINVAL;
 271
 272         reset_count.hard_reset_cnt = hdev->hard_reset_cnt;
 273         reset_count.soft_reset_cnt = hdev->soft_reset_cnt;
 274
 275         return copy_to_user(out, &reset_count,
 276                 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
 277 }
 278
 279 static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
 280 {
 281         struct hl_info_time_sync time_sync = {0};
 282         u32 max_size = args->return_size;
 283         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 284
 285         if ((!max_size) || (!out))
 286                 return -EINVAL;
 287
 288         time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
 289         time_sync.host_time = ktime_get_raw_ns();
 290
 291         return copy_to_user(out, &time_sync,
 292                 min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
 293 }
 294
 295 static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 296 {
 297         struct hl_device *hdev = hpriv->hdev;
 298         struct hl_info_pci_counters pci_counters = {0};
 299         u32 max_size = args->return_size;
 300         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 301         int rc;
 302
 303         if ((!max_size) || (!out))
 304                 return -EINVAL;
 305
 306         rc = hl_fw_cpucp_pci_counters_get(hdev, &pci_counters);
 307         if (rc)
 308                 return rc;
 309
 310         return copy_to_user(out, &pci_counters,
 311                 min((size_t) max_size, sizeof(pci_counters))) ? -EFAULT : 0;
 312 }
 313
 314 static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 315 {
 316         struct hl_device *hdev = hpriv->hdev;
 317         struct hl_info_clk_throttle clk_throttle = {0};
 318         u32 max_size = args->return_size;
 319         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 320
 321         if ((!max_size) || (!out))
 322                 return -EINVAL;
 323
 324         clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason;
 325
 326         return copy_to_user(out, &clk_throttle,
 327                 min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0;
 328 }
 329
 330 static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 331 {
 332         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 333         struct hl_info_cs_counters cs_counters = {0};
 334         struct hl_device *hdev = hpriv->hdev;
 335         struct hl_cs_counters_atomic *cntr;
 336         u32 max_size = args->return_size;
 337
 338         cntr = &hdev->aggregated_cs_counters;
 339
 340         if ((!max_size) || (!out))
 341                 return -EINVAL;
 342
 343         cs_counters.total_out_of_mem_drop_cnt =
 344                         atomic64_read(&cntr->out_of_mem_drop_cnt);
 345         cs_counters.total_parsing_drop_cnt =
 346                         atomic64_read(&cntr->parsing_drop_cnt);
 347         cs_counters.total_queue_full_drop_cnt =
 348                         atomic64_read(&cntr->queue_full_drop_cnt);
 349         cs_counters.total_device_in_reset_drop_cnt =
 350                         atomic64_read(&cntr->device_in_reset_drop_cnt);
 351         cs_counters.total_max_cs_in_flight_drop_cnt =
 352                         atomic64_read(&cntr->max_cs_in_flight_drop_cnt);
 353         cs_counters.total_validation_drop_cnt =
 354                         atomic64_read(&cntr->validation_drop_cnt);
 355
 356         if (hpriv->ctx) {
 357                 cs_counters.ctx_out_of_mem_drop_cnt =
 358                                 atomic64_read(
 359                                 &hpriv->ctx->cs_counters.out_of_mem_drop_cnt);
 360                 cs_counters.ctx_parsing_drop_cnt =
 361                                 atomic64_read(
 362                                 &hpriv->ctx->cs_counters.parsing_drop_cnt);
 363                 cs_counters.ctx_queue_full_drop_cnt =
 364                                 atomic64_read(
 365                                 &hpriv->ctx->cs_counters.queue_full_drop_cnt);
 366                 cs_counters.ctx_device_in_reset_drop_cnt =
 367                                 atomic64_read(
 368                         &hpriv->ctx->cs_counters.device_in_reset_drop_cnt);
 369                 cs_counters.ctx_max_cs_in_flight_drop_cnt =
 370                                 atomic64_read(
 371                         &hpriv->ctx->cs_counters.max_cs_in_flight_drop_cnt);
 372                 cs_counters.ctx_validation_drop_cnt =
 373                                 atomic64_read(
 374                                 &hpriv->ctx->cs_counters.validation_drop_cnt);
 375         }
 376
 377         return copy_to_user(out, &cs_counters,
 378                 min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0;
 379 }
 380
 381 static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 382 {
 383         struct hl_device *hdev = hpriv->hdev;
 384         struct asic_fixed_properties *prop = &hdev->asic_prop;
 385         struct hl_info_sync_manager sm_info = {0};
 386         u32 max_size = args->return_size;
 387         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 388
 389         if ((!max_size) || (!out))
 390                 return -EINVAL;
 391
 392         if (args->dcore_id >= HL_MAX_DCORES)
 393                 return -EINVAL;
 394
 395         sm_info.first_available_sync_object =
 396                         prop->first_available_user_sob[args->dcore_id];
 397         sm_info.first_available_monitor =
 398                         prop->first_available_user_mon[args->dcore_id];
 399         sm_info.first_available_cq =
 400                         prop->first_available_cq[args->dcore_id];
 401
 402         return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size,
 403                         sizeof(sm_info))) ? -EFAULT : 0;
 404 }
 405
 406 static int total_energy_consumption_info(struct hl_fpriv *hpriv,
 407                         struct hl_info_args *args)
 408 {
 409         struct hl_device *hdev = hpriv->hdev;
 410         struct hl_info_energy total_energy = {0};
 411         u32 max_size = args->return_size;
 412         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 413         int rc;
 414
 415         if ((!max_size) || (!out))
 416                 return -EINVAL;
 417
 418         rc = hl_fw_cpucp_total_energy_get(hdev,
 419                         &total_energy.total_energy_consumption);
 420         if (rc)
 421                 return rc;
 422
 423         return copy_to_user(out, &total_energy,
 424                 min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0;
 425 }
 426
 427 static int pll_frequency_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 428 {
 429         struct hl_device *hdev = hpriv->hdev;
 430         struct hl_pll_frequency_info freq_info = { {0} };
 431         u32 max_size = args->return_size;
 432         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 433         int rc;
 434
 435         if ((!max_size) || (!out))
 436                 return -EINVAL;
 437
 438         rc = hl_fw_cpucp_pll_info_get(hdev, args->pll_index, freq_info.output);
 439         if (rc)
 440                 return rc;
 441
 442         return copy_to_user(out, &freq_info,
 443                 min((size_t) max_size, sizeof(freq_info))) ? -EFAULT : 0;
 444 }
 445
 446 static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 447 {
 448         struct hl_device *hdev = hpriv->hdev;
 449         u32 max_size = args->return_size;
 450         struct hl_power_info power_info = {0};
 451         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 452         int rc;
 453
 454         if ((!max_size) || (!out))
 455                 return -EINVAL;
 456
 457         rc = hl_fw_cpucp_power_get(hdev, &power_info.power);
 458         if (rc)
 459                 return rc;
 460
 461         return copy_to_user(out, &power_info,
 462                 min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0;
 463 }
 464
 465 static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 466 {
 467         struct hl_device *hdev = hpriv->hdev;
 468         u32 max_size = args->return_size;
 469         struct hl_open_stats_info open_stats_info = {0};
 470         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 471
 472         if ((!max_size) || (!out))
 473                 return -EINVAL;
 474
 475         open_stats_info.last_open_period_ms = jiffies64_to_msecs(
 476                 hdev->last_open_session_duration_jif);
 477         open_stats_info.open_counter = hdev->open_counter;
 478
 479         return copy_to_user(out, &open_stats_info,
 480                 min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0;
 481 }
 482
 483 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 484                                 struct device *dev)
 485 {
 486         enum hl_device_status status;
 487         struct hl_info_args *args = data;
 488         struct hl_device *hdev = hpriv->hdev;
 489
 490         int rc;
 491
 492         /*
 493          * Information is returned for the following opcodes even if the device
 494          * is disabled or in reset.
 495          */
 496         switch (args->op) {
 497         case HL_INFO_HW_IP_INFO:
 498                 return hw_ip_info(hdev, args);
 499
 500         case HL_INFO_DEVICE_STATUS:
 501                 return device_status_info(hdev, args);
 502
 503         case HL_INFO_RESET_COUNT:
 504                 return get_reset_count(hdev, args);
 505
 506         default:
 507                 break;
 508         }
 509
 510         if (!hl_device_operational(hdev, &status)) {
 511                 dev_warn_ratelimited(dev,
 512                         "Device is %s. Can't execute INFO IOCTL\n",
 513                         hdev->status[status]);
 514                 return -EBUSY;
 515         }
 516
 517         switch (args->op) {
 518         case HL_INFO_HW_EVENTS:
 519                 rc = hw_events_info(hdev, false, args);
 520                 break;
 521
 522         case HL_INFO_DRAM_USAGE:
 523                 rc = dram_usage_info(hpriv, args);
 524                 break;
 525
 526         case HL_INFO_HW_IDLE:
 527                 rc = hw_idle(hdev, args);
 528                 break;
 529
 530         case HL_INFO_DEVICE_UTILIZATION:
 531                 rc = device_utilization(hdev, args);
 532                 break;
 533
 534         case HL_INFO_HW_EVENTS_AGGREGATE:
 535                 rc = hw_events_info(hdev, true, args);
 536                 break;
 537
 538         case HL_INFO_CLK_RATE:
 539                 rc = get_clk_rate(hdev, args);
 540                 break;
 541
 542         case HL_INFO_TIME_SYNC:
 543                 return time_sync_info(hdev, args);
 544
 545         case HL_INFO_CS_COUNTERS:
 546                 return cs_counters_info(hpriv, args);
 547
 548         case HL_INFO_PCI_COUNTERS:
 549                 return pci_counters_info(hpriv, args);
 550
 551         case HL_INFO_CLK_THROTTLE_REASON:
 552                 return clk_throttle_info(hpriv, args);
 553
 554         case HL_INFO_SYNC_MANAGER:
 555                 return sync_manager_info(hpriv, args);
 556
 557         case HL_INFO_TOTAL_ENERGY:
 558                 return total_energy_consumption_info(hpriv, args);
 559
 560         case HL_INFO_PLL_FREQUENCY:
 561                 return pll_frequency_info(hpriv, args);
 562
 563         case HL_INFO_POWER:
 564                 return power_info(hpriv, args);
 565
 566         case HL_INFO_OPEN_STATS:
 567                 return open_stats_info(hpriv, args);
 568
 569         default:
 570                 dev_err(dev, "Invalid request %d\n", args->op);
 571                 rc = -ENOTTY;
 572                 break;
 573         }
 574
 575         return rc;
 576 }
 577
 578 static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
 579 {
 580         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
 581 }
 582
 583 static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
 584 {
 585         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
 586 }
 587
 588 static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
 589 {
 590         struct hl_debug_args *args = data;
 591         struct hl_device *hdev = hpriv->hdev;
 592         enum hl_device_status status;
 593
 594         int rc = 0;
 595
 596         if (!hl_device_operational(hdev, &status)) {
 597                 dev_warn_ratelimited(hdev->dev,
 598                         "Device is %s. Can't execute DEBUG IOCTL\n",
 599                         hdev->status[status]);
 600                 return -EBUSY;
 601         }
 602
 603         switch (args->op) {
 604         case HL_DEBUG_OP_ETR:
 605         case HL_DEBUG_OP_ETF:
 606         case HL_DEBUG_OP_STM:
 607         case HL_DEBUG_OP_FUNNEL:
 608         case HL_DEBUG_OP_BMON:
 609         case HL_DEBUG_OP_SPMU:
 610         case HL_DEBUG_OP_TIMESTAMP:
 611                 if (!hdev->in_debug) {
 612                         dev_err_ratelimited(hdev->dev,
 613                                 "Rejecting debug configuration request because device not in debug mode\n");
 614                         return -EFAULT;
 615                 }
 616                 args->input_size =
 617                         min(args->input_size, hl_debug_struct_size[args->op]);
 618                 rc = debug_coresight(hdev, args);
 619                 break;
 620         case HL_DEBUG_OP_SET_MODE:
 621                 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
 622                 break;
 623         default:
 624                 dev_err(hdev->dev, "Invalid request %d\n", args->op);
 625                 rc = -ENOTTY;
 626                 break;
 627         }
 628
 629         return rc;
 630 }
 631
 632 #define HL_IOCTL_DEF(ioctl, _func) \
 633         [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
 634
 635 static const struct hl_ioctl_desc hl_ioctls[] = {
 636         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
 637         HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
 638         HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
 639         HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_wait_ioctl),
 640         HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
 641         HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
 642 };
 643
 644 static const struct hl_ioctl_desc hl_ioctls_control[] = {
 645         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
 646 };
 647
 648 static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
 649                 const struct hl_ioctl_desc *ioctl, struct device *dev)
 650 {
 651         struct hl_fpriv *hpriv = filep->private_data;
 652         struct hl_device *hdev = hpriv->hdev;
 653         unsigned int nr = _IOC_NR(cmd);
 654         char stack_kdata[128] = {0};
 655         char *kdata = NULL;
 656         unsigned int usize, asize;
 657         hl_ioctl_t *func;
 658         u32 hl_size;
 659         int retcode;
 660
 661         if (hdev->hard_reset_pending) {
 662                 dev_crit_ratelimited(dev,
 663                         "Device HARD reset pending! Please close FD\n");
 664                 return -ENODEV;
 665         }
 666
 667         /* Do not trust userspace, use our own definition */
 668         func = ioctl->func;
 669
 670         if (unlikely(!func)) {
 671                 dev_dbg(dev, "no function\n");
 672                 retcode = -ENOTTY;
 673                 goto out_err;
 674         }
 675
 676         hl_size = _IOC_SIZE(ioctl->cmd);
 677         usize = asize = _IOC_SIZE(cmd);
 678         if (hl_size > asize)
 679                 asize = hl_size;
 680
 681         cmd = ioctl->cmd;
 682
 683         if (cmd & (IOC_IN | IOC_OUT)) {
 684                 if (asize <= sizeof(stack_kdata)) {
 685                         kdata = stack_kdata;
 686                 } else {
 687                         kdata = kzalloc(asize, GFP_KERNEL);
 688                         if (!kdata) {
 689                                 retcode = -ENOMEM;
 690                                 goto out_err;
 691                         }
 692                 }
 693         }
 694
 695         if (cmd & IOC_IN) {
 696                 if (copy_from_user(kdata, (void __user *)arg, usize)) {
 697                         retcode = -EFAULT;
 698                         goto out_err;
 699                 }
 700         } else if (cmd & IOC_OUT) {
 701                 memset(kdata, 0, usize);
 702         }
 703
 704         retcode = func(hpriv, kdata);
 705
 706         if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize))
 707                 retcode = -EFAULT;
 708
 709 out_err:
 710         if (retcode)
 711                 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
 712                           task_pid_nr(current), cmd, nr);
 713
 714         if (kdata != stack_kdata)
 715                 kfree(kdata);
 716
 717         return retcode;
 718 }
 719
 720 long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 721 {
 722         struct hl_fpriv *hpriv = filep->private_data;
 723         struct hl_device *hdev = hpriv->hdev;
 724         const struct hl_ioctl_desc *ioctl = NULL;
 725         unsigned int nr = _IOC_NR(cmd);
 726
 727         if (!hdev) {
 728                 pr_err_ratelimited("Sending ioctl after device was removed! Please close FD\n");
 729                 return -ENODEV;
 730         }
 731
 732         if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
 733                 ioctl = &hl_ioctls[nr];
 734         } else {
 735                 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
 736                         task_pid_nr(current), nr);
 737                 return -ENOTTY;
 738         }
 739
 740         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
 741 }
 742
 743 long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
 744 {
 745         struct hl_fpriv *hpriv = filep->private_data;
 746         struct hl_device *hdev = hpriv->hdev;
 747         const struct hl_ioctl_desc *ioctl = NULL;
 748         unsigned int nr = _IOC_NR(cmd);
 749
 750         if (!hdev) {
 751                 pr_err_ratelimited("Sending ioctl after device was removed! Please close FD\n");
 752                 return -ENODEV;
 753         }
 754
 755         if (nr == _IOC_NR(HL_IOCTL_INFO)) {
 756                 ioctl = &hl_ioctls_control[nr];
 757         } else {
 758                 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
 759                         task_pid_nr(current), nr);
 760                 return -ENOTTY;
 761         }
 762
 763         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
 764 }