drivers/misc/habanalabs/common/habanalabs_ioctl.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/kernel.h>
  12 #include <linux/fs.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/slab.h>
  15
  16 static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
  17         [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
  18         [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
  19         [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
  20         [HL_DEBUG_OP_FUNNEL] = 0,
  21         [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
  22         [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
  23         [HL_DEBUG_OP_TIMESTAMP] = 0
  24
  25 };
  26
  27 static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
  28 {
  29         struct hl_info_device_status dev_stat = {0};
  30         u32 size = args->return_size;
  31         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  32
  33         if ((!size) || (!out))
  34                 return -EINVAL;
  35
  36         dev_stat.status = hl_device_status(hdev);
  37
  38         return copy_to_user(out, &dev_stat,
  39                         min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
  40 }
  41
  42 static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
  43 {
  44         struct hl_info_hw_ip_info hw_ip = {0};
  45         u32 size = args->return_size;
  46         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  47         struct asic_fixed_properties *prop = &hdev->asic_prop;
  48         u64 sram_kmd_size, dram_kmd_size;
  49
  50         if ((!size) || (!out))
  51                 return -EINVAL;
  52
  53         sram_kmd_size = (prop->sram_user_base_address -
  54                                 prop->sram_base_address);
  55         dram_kmd_size = (prop->dram_user_base_address -
  56                                 prop->dram_base_address);
  57
  58         hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
  59         hw_ip.sram_base_address = prop->sram_user_base_address;
  60         hw_ip.dram_base_address =
  61                         hdev->mmu_enable && prop->dram_supports_virtual_memory ?
  62                         prop->dmmu.start_addr : prop->dram_user_base_address;
  63         hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
  64         hw_ip.sram_size = prop->sram_size - sram_kmd_size;
  65
  66         if (hdev->mmu_enable)
  67                 hw_ip.dram_size =
  68                         DIV_ROUND_DOWN_ULL(prop->dram_size - dram_kmd_size,
  69                                                 prop->dram_page_size) *
  70                                                         prop->dram_page_size;
  71         else
  72                 hw_ip.dram_size = prop->dram_size - dram_kmd_size;
  73
  74         if (hw_ip.dram_size > PAGE_SIZE)
  75                 hw_ip.dram_enabled = 1;
  76         hw_ip.dram_page_size = prop->dram_page_size;
  77         hw_ip.num_of_events = prop->num_of_events;
  78
  79         memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version,
  80                 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
  81
  82         memcpy(hw_ip.card_name, prop->cpucp_info.card_name,
  83                 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
  84
  85         hw_ip.cpld_version = le32_to_cpu(prop->cpucp_info.cpld_version);
  86         hw_ip.module_id = le32_to_cpu(prop->cpucp_info.card_location);
  87
  88         hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
  89         hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
  90         hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
  91         hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
  92
  93         hw_ip.first_available_interrupt_id =
  94                         prop->first_available_user_msix_interrupt;
  95         return copy_to_user(out, &hw_ip,
  96                 min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
  97 }
  98
  99 static int hw_events_info(struct hl_device *hdev, bool aggregate,
 100                         struct hl_info_args *args)
 101 {
 102         u32 size, max_size = args->return_size;
 103         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 104         void *arr;
 105
 106         if ((!max_size) || (!out))
 107                 return -EINVAL;
 108
 109         arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
 110
 111         return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
 112 }
 113
 114 static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 115 {
 116         struct hl_device *hdev = hpriv->hdev;
 117         struct hl_info_dram_usage dram_usage = {0};
 118         u32 max_size = args->return_size;
 119         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 120         struct asic_fixed_properties *prop = &hdev->asic_prop;
 121         u64 dram_kmd_size;
 122
 123         if ((!max_size) || (!out))
 124                 return -EINVAL;
 125
 126         dram_kmd_size = (prop->dram_user_base_address -
 127                                 prop->dram_base_address);
 128         dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
 129                                         atomic64_read(&hdev->dram_used_mem);
 130         if (hpriv->ctx)
 131                 dram_usage.ctx_dram_mem =
 132                         atomic64_read(&hpriv->ctx->dram_phys_mem);
 133
 134         return copy_to_user(out, &dram_usage,
 135                 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
 136 }
 137
 138 static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
 139 {
 140         struct hl_info_hw_idle hw_idle = {0};
 141         u32 max_size = args->return_size;
 142         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 143
 144         if ((!max_size) || (!out))
 145                 return -EINVAL;
 146
 147         hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
 148                                         hw_idle.busy_engines_mask_ext,
 149                                         HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL);
 150         hw_idle.busy_engines_mask =
 151                         lower_32_bits(hw_idle.busy_engines_mask_ext[0]);
 152
 153         return copy_to_user(out, &hw_idle,
 154                 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
 155 }
 156
 157 static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
 158 {
 159         struct hl_debug_params *params;
 160         void *input = NULL, *output = NULL;
 161         int rc;
 162
 163         params = kzalloc(sizeof(*params), GFP_KERNEL);
 164         if (!params)
 165                 return -ENOMEM;
 166
 167         params->reg_idx = args->reg_idx;
 168         params->enable = args->enable;
 169         params->op = args->op;
 170
 171         if (args->input_ptr && args->input_size) {
 172                 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
 173                 if (!input) {
 174                         rc = -ENOMEM;
 175                         goto out;
 176                 }
 177
 178                 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
 179                                         args->input_size)) {
 180                         rc = -EFAULT;
 181                         dev_err(hdev->dev, "failed to copy input debug data\n");
 182                         goto out;
 183                 }
 184
 185                 params->input = input;
 186         }
 187
 188         if (args->output_ptr && args->output_size) {
 189                 output = kzalloc(args->output_size, GFP_KERNEL);
 190                 if (!output) {
 191                         rc = -ENOMEM;
 192                         goto out;
 193                 }
 194
 195                 params->output = output;
 196                 params->output_size = args->output_size;
 197         }
 198
 199         rc = hdev->asic_funcs->debug_coresight(hdev, params);
 200         if (rc) {
 201                 dev_err(hdev->dev,
 202                         "debug coresight operation failed %d\n", rc);
 203                 goto out;
 204         }
 205
 206         if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr,
 207                                         output, args->output_size)) {
 208                 dev_err(hdev->dev, "copy to user failed in debug ioctl\n");
 209                 rc = -EFAULT;
 210                 goto out;
 211         }
 212
 213
 214 out:
 215         kfree(params);
 216         kfree(output);
 217         kfree(input);
 218
 219         return rc;
 220 }
 221
 222 static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
 223 {
 224         struct hl_info_device_utilization device_util = {0};
 225         u32 max_size = args->return_size;
 226         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 227
 228         if ((!max_size) || (!out))
 229                 return -EINVAL;
 230
 231         if ((args->period_ms < 100) || (args->period_ms > 1000) ||
 232                 (args->period_ms % 100)) {
 233                 dev_err(hdev->dev,
 234                         "period %u must be between 100 - 1000 and must be divisible by 100\n",
 235                         args->period_ms);
 236                 return -EINVAL;
 237         }
 238
 239         device_util.utilization = hl_device_utilization(hdev, args->period_ms);
 240
 241         return copy_to_user(out, &device_util,
 242                 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
 243 }
 244
 245 static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args)
 246 {
 247         struct hl_info_clk_rate clk_rate = {0};
 248         u32 max_size = args->return_size;
 249         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 250         int rc;
 251
 252         if ((!max_size) || (!out))
 253                 return -EINVAL;
 254
 255         rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz,
 256                                                 &clk_rate.max_clk_rate_mhz);
 257         if (rc)
 258                 return rc;
 259
 260         return copy_to_user(out, &clk_rate,
 261                 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0;
 262 }
 263
 264 static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
 265 {
 266         struct hl_info_reset_count reset_count = {0};
 267         u32 max_size = args->return_size;
 268         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 269
 270         if ((!max_size) || (!out))
 271                 return -EINVAL;
 272
 273         reset_count.hard_reset_cnt = hdev->hard_reset_cnt;
 274         reset_count.soft_reset_cnt = hdev->soft_reset_cnt;
 275
 276         return copy_to_user(out, &reset_count,
 277                 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
 278 }
 279
 280 static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
 281 {
 282         struct hl_info_time_sync time_sync = {0};
 283         u32 max_size = args->return_size;
 284         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 285
 286         if ((!max_size) || (!out))
 287                 return -EINVAL;
 288
 289         time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
 290         time_sync.host_time = ktime_get_raw_ns();
 291
 292         return copy_to_user(out, &time_sync,
 293                 min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
 294 }
 295
 296 static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 297 {
 298         struct hl_device *hdev = hpriv->hdev;
 299         struct hl_info_pci_counters pci_counters = {0};
 300         u32 max_size = args->return_size;
 301         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 302         int rc;
 303
 304         if ((!max_size) || (!out))
 305                 return -EINVAL;
 306
 307         rc = hl_fw_cpucp_pci_counters_get(hdev, &pci_counters);
 308         if (rc)
 309                 return rc;
 310
 311         return copy_to_user(out, &pci_counters,
 312                 min((size_t) max_size, sizeof(pci_counters))) ? -EFAULT : 0;
 313 }
 314
 315 static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 316 {
 317         struct hl_device *hdev = hpriv->hdev;
 318         struct hl_info_clk_throttle clk_throttle = {0};
 319         u32 max_size = args->return_size;
 320         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 321
 322         if ((!max_size) || (!out))
 323                 return -EINVAL;
 324
 325         clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason;
 326
 327         return copy_to_user(out, &clk_throttle,
 328                 min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0;
 329 }
 330
 331 static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 332 {
 333         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 334         struct hl_info_cs_counters cs_counters = {0};
 335         struct hl_device *hdev = hpriv->hdev;
 336         struct hl_cs_counters_atomic *cntr;
 337         u32 max_size = args->return_size;
 338
 339         cntr = &hdev->aggregated_cs_counters;
 340
 341         if ((!max_size) || (!out))
 342                 return -EINVAL;
 343
 344         cs_counters.total_out_of_mem_drop_cnt =
 345                         atomic64_read(&cntr->out_of_mem_drop_cnt);
 346         cs_counters.total_parsing_drop_cnt =
 347                         atomic64_read(&cntr->parsing_drop_cnt);
 348         cs_counters.total_queue_full_drop_cnt =
 349                         atomic64_read(&cntr->queue_full_drop_cnt);
 350         cs_counters.total_device_in_reset_drop_cnt =
 351                         atomic64_read(&cntr->device_in_reset_drop_cnt);
 352         cs_counters.total_max_cs_in_flight_drop_cnt =
 353                         atomic64_read(&cntr->max_cs_in_flight_drop_cnt);
 354         cs_counters.total_validation_drop_cnt =
 355                         atomic64_read(&cntr->validation_drop_cnt);
 356
 357         if (hpriv->ctx) {
 358                 cs_counters.ctx_out_of_mem_drop_cnt =
 359                                 atomic64_read(
 360                                 &hpriv->ctx->cs_counters.out_of_mem_drop_cnt);
 361                 cs_counters.ctx_parsing_drop_cnt =
 362                                 atomic64_read(
 363                                 &hpriv->ctx->cs_counters.parsing_drop_cnt);
 364                 cs_counters.ctx_queue_full_drop_cnt =
 365                                 atomic64_read(
 366                                 &hpriv->ctx->cs_counters.queue_full_drop_cnt);
 367                 cs_counters.ctx_device_in_reset_drop_cnt =
 368                                 atomic64_read(
 369                         &hpriv->ctx->cs_counters.device_in_reset_drop_cnt);
 370                 cs_counters.ctx_max_cs_in_flight_drop_cnt =
 371                                 atomic64_read(
 372                         &hpriv->ctx->cs_counters.max_cs_in_flight_drop_cnt);
 373                 cs_counters.ctx_validation_drop_cnt =
 374                                 atomic64_read(
 375                                 &hpriv->ctx->cs_counters.validation_drop_cnt);
 376         }
 377
 378         return copy_to_user(out, &cs_counters,
 379                 min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0;
 380 }
 381
 382 static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 383 {
 384         struct hl_device *hdev = hpriv->hdev;
 385         struct asic_fixed_properties *prop = &hdev->asic_prop;
 386         struct hl_info_sync_manager sm_info = {0};
 387         u32 max_size = args->return_size;
 388         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 389
 390         if ((!max_size) || (!out))
 391                 return -EINVAL;
 392
 393         if (args->dcore_id >= HL_MAX_DCORES)
 394                 return -EINVAL;
 395
 396         sm_info.first_available_sync_object =
 397                         prop->first_available_user_sob[args->dcore_id];
 398         sm_info.first_available_monitor =
 399                         prop->first_available_user_mon[args->dcore_id];
 400         sm_info.first_available_cq =
 401                         prop->first_available_cq[args->dcore_id];
 402
 403         return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size,
 404                         sizeof(sm_info))) ? -EFAULT : 0;
 405 }
 406
 407 static int total_energy_consumption_info(struct hl_fpriv *hpriv,
 408                         struct hl_info_args *args)
 409 {
 410         struct hl_device *hdev = hpriv->hdev;
 411         struct hl_info_energy total_energy = {0};
 412         u32 max_size = args->return_size;
 413         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 414         int rc;
 415
 416         if ((!max_size) || (!out))
 417                 return -EINVAL;
 418
 419         rc = hl_fw_cpucp_total_energy_get(hdev,
 420                         &total_energy.total_energy_consumption);
 421         if (rc)
 422                 return rc;
 423
 424         return copy_to_user(out, &total_energy,
 425                 min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0;
 426 }
 427
 428 static int pll_frequency_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 429 {
 430         struct hl_device *hdev = hpriv->hdev;
 431         struct hl_pll_frequency_info freq_info = { {0} };
 432         u32 max_size = args->return_size;
 433         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 434         int rc;
 435
 436         if ((!max_size) || (!out))
 437                 return -EINVAL;
 438
 439         rc = hl_fw_cpucp_pll_info_get(hdev, args->pll_index, freq_info.output);
 440         if (rc)
 441                 return rc;
 442
 443         return copy_to_user(out, &freq_info,
 444                 min((size_t) max_size, sizeof(freq_info))) ? -EFAULT : 0;
 445 }
 446
 447 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 448                                 struct device *dev)
 449 {
 450         enum hl_device_status status;
 451         struct hl_info_args *args = data;
 452         struct hl_device *hdev = hpriv->hdev;
 453
 454         int rc;
 455
 456         /*
 457          * Information is returned for the following opcodes even if the device
 458          * is disabled or in reset.
 459          */
 460         switch (args->op) {
 461         case HL_INFO_HW_IP_INFO:
 462                 return hw_ip_info(hdev, args);
 463
 464         case HL_INFO_DEVICE_STATUS:
 465                 return device_status_info(hdev, args);
 466
 467         case HL_INFO_RESET_COUNT:
 468                 return get_reset_count(hdev, args);
 469
 470         default:
 471                 break;
 472         }
 473
 474         if (!hl_device_operational(hdev, &status)) {
 475                 dev_warn_ratelimited(dev,
 476                         "Device is %s. Can't execute INFO IOCTL\n",
 477                         hdev->status[status]);
 478                 return -EBUSY;
 479         }
 480
 481         switch (args->op) {
 482         case HL_INFO_HW_EVENTS:
 483                 rc = hw_events_info(hdev, false, args);
 484                 break;
 485
 486         case HL_INFO_DRAM_USAGE:
 487                 rc = dram_usage_info(hpriv, args);
 488                 break;
 489
 490         case HL_INFO_HW_IDLE:
 491                 rc = hw_idle(hdev, args);
 492                 break;
 493
 494         case HL_INFO_DEVICE_UTILIZATION:
 495                 rc = device_utilization(hdev, args);
 496                 break;
 497
 498         case HL_INFO_HW_EVENTS_AGGREGATE:
 499                 rc = hw_events_info(hdev, true, args);
 500                 break;
 501
 502         case HL_INFO_CLK_RATE:
 503                 rc = get_clk_rate(hdev, args);
 504                 break;
 505
 506         case HL_INFO_TIME_SYNC:
 507                 return time_sync_info(hdev, args);
 508
 509         case HL_INFO_CS_COUNTERS:
 510                 return cs_counters_info(hpriv, args);
 511
 512         case HL_INFO_PCI_COUNTERS:
 513                 return pci_counters_info(hpriv, args);
 514
 515         case HL_INFO_CLK_THROTTLE_REASON:
 516                 return clk_throttle_info(hpriv, args);
 517
 518         case HL_INFO_SYNC_MANAGER:
 519                 return sync_manager_info(hpriv, args);
 520
 521         case HL_INFO_TOTAL_ENERGY:
 522                 return total_energy_consumption_info(hpriv, args);
 523
 524         case HL_INFO_PLL_FREQUENCY:
 525                 return pll_frequency_info(hpriv, args);
 526
 527         default:
 528                 dev_err(dev, "Invalid request %d\n", args->op);
 529                 rc = -ENOTTY;
 530                 break;
 531         }
 532
 533         return rc;
 534 }
 535
 536 static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
 537 {
 538         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
 539 }
 540
 541 static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
 542 {
 543         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
 544 }
 545
 546 static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
 547 {
 548         struct hl_debug_args *args = data;
 549         struct hl_device *hdev = hpriv->hdev;
 550         enum hl_device_status status;
 551
 552         int rc = 0;
 553
 554         if (!hl_device_operational(hdev, &status)) {
 555                 dev_warn_ratelimited(hdev->dev,
 556                         "Device is %s. Can't execute DEBUG IOCTL\n",
 557                         hdev->status[status]);
 558                 return -EBUSY;
 559         }
 560
 561         switch (args->op) {
 562         case HL_DEBUG_OP_ETR:
 563         case HL_DEBUG_OP_ETF:
 564         case HL_DEBUG_OP_STM:
 565         case HL_DEBUG_OP_FUNNEL:
 566         case HL_DEBUG_OP_BMON:
 567         case HL_DEBUG_OP_SPMU:
 568         case HL_DEBUG_OP_TIMESTAMP:
 569                 if (!hdev->in_debug) {
 570                         dev_err_ratelimited(hdev->dev,
 571                                 "Rejecting debug configuration request because device not in debug mode\n");
 572                         return -EFAULT;
 573                 }
 574                 args->input_size =
 575                         min(args->input_size, hl_debug_struct_size[args->op]);
 576                 rc = debug_coresight(hdev, args);
 577                 break;
 578         case HL_DEBUG_OP_SET_MODE:
 579                 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
 580                 break;
 581         default:
 582                 dev_err(hdev->dev, "Invalid request %d\n", args->op);
 583                 rc = -ENOTTY;
 584                 break;
 585         }
 586
 587         return rc;
 588 }
 589
 590 #define HL_IOCTL_DEF(ioctl, _func) \
 591         [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
 592
 593 static const struct hl_ioctl_desc hl_ioctls[] = {
 594         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
 595         HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
 596         HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
 597         HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
 598         HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
 599         HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
 600 };
 601
 602 static const struct hl_ioctl_desc hl_ioctls_control[] = {
 603         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
 604 };
 605
 606 static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
 607                 const struct hl_ioctl_desc *ioctl, struct device *dev)
 608 {
 609         struct hl_fpriv *hpriv = filep->private_data;
 610         struct hl_device *hdev = hpriv->hdev;
 611         unsigned int nr = _IOC_NR(cmd);
 612         char stack_kdata[128] = {0};
 613         char *kdata = NULL;
 614         unsigned int usize, asize;
 615         hl_ioctl_t *func;
 616         u32 hl_size;
 617         int retcode;
 618
 619         if (hdev->hard_reset_pending) {
 620                 dev_crit_ratelimited(dev,
 621                         "Device HARD reset pending! Please close FD\n");
 622                 return -ENODEV;
 623         }
 624
 625         /* Do not trust userspace, use our own definition */
 626         func = ioctl->func;
 627
 628         if (unlikely(!func)) {
 629                 dev_dbg(dev, "no function\n");
 630                 retcode = -ENOTTY;
 631                 goto out_err;
 632         }
 633
 634         hl_size = _IOC_SIZE(ioctl->cmd);
 635         usize = asize = _IOC_SIZE(cmd);
 636         if (hl_size > asize)
 637                 asize = hl_size;
 638
 639         cmd = ioctl->cmd;
 640
 641         if (cmd & (IOC_IN | IOC_OUT)) {
 642                 if (asize <= sizeof(stack_kdata)) {
 643                         kdata = stack_kdata;
 644                 } else {
 645                         kdata = kzalloc(asize, GFP_KERNEL);
 646                         if (!kdata) {
 647                                 retcode = -ENOMEM;
 648                                 goto out_err;
 649                         }
 650                 }
 651         }
 652
 653         if (cmd & IOC_IN) {
 654                 if (copy_from_user(kdata, (void __user *)arg, usize)) {
 655                         retcode = -EFAULT;
 656                         goto out_err;
 657                 }
 658         } else if (cmd & IOC_OUT) {
 659                 memset(kdata, 0, usize);
 660         }
 661
 662         retcode = func(hpriv, kdata);
 663
 664         if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize))
 665                 retcode = -EFAULT;
 666
 667 out_err:
 668         if (retcode)
 669                 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
 670                           task_pid_nr(current), cmd, nr);
 671
 672         if (kdata != stack_kdata)
 673                 kfree(kdata);
 674
 675         return retcode;
 676 }
 677
 678 long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 679 {
 680         struct hl_fpriv *hpriv = filep->private_data;
 681         struct hl_device *hdev = hpriv->hdev;
 682         const struct hl_ioctl_desc *ioctl = NULL;
 683         unsigned int nr = _IOC_NR(cmd);
 684
 685         if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
 686                 ioctl = &hl_ioctls[nr];
 687         } else {
 688                 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
 689                         task_pid_nr(current), nr);
 690                 return -ENOTTY;
 691         }
 692
 693         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
 694 }
 695
 696 long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
 697 {
 698         struct hl_fpriv *hpriv = filep->private_data;
 699         struct hl_device *hdev = hpriv->hdev;
 700         const struct hl_ioctl_desc *ioctl = NULL;
 701         unsigned int nr = _IOC_NR(cmd);
 702
 703         if (nr == _IOC_NR(HL_IOCTL_INFO)) {
 704                 ioctl = &hl_ioctls_control[nr];
 705         } else {
 706                 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
 707                         task_pid_nr(current), nr);
 708                 return -ENOTTY;
 709         }
 710
 711         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
 712 }