drivers/misc/habanalabs/common/habanalabs_ioctl.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/kernel.h>
  12 #include <linux/fs.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/slab.h>
  15
  16 static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
  17         [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
  18         [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
  19         [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
  20         [HL_DEBUG_OP_FUNNEL] = 0,
  21         [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
  22         [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
  23         [HL_DEBUG_OP_TIMESTAMP] = 0
  24
  25 };
  26
  27 static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
  28 {
  29         struct hl_info_device_status dev_stat = {0};
  30         u32 size = args->return_size;
  31         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  32
  33         if ((!size) || (!out))
  34                 return -EINVAL;
  35
  36         dev_stat.status = hl_device_status(hdev);
  37
  38         return copy_to_user(out, &dev_stat,
  39                         min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
  40 }
  41
  42 static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
  43 {
  44         struct hl_info_hw_ip_info hw_ip = {0};
  45         u32 size = args->return_size;
  46         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  47         struct asic_fixed_properties *prop = &hdev->asic_prop;
  48         u64 sram_kmd_size, dram_kmd_size;
  49
  50         if ((!size) || (!out))
  51                 return -EINVAL;
  52
  53         sram_kmd_size = (prop->sram_user_base_address -
  54                                 prop->sram_base_address);
  55         dram_kmd_size = (prop->dram_user_base_address -
  56                                 prop->dram_base_address);
  57
  58         hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
  59         hw_ip.sram_base_address = prop->sram_user_base_address;
  60         hw_ip.dram_base_address = prop->dram_user_base_address;
  61         hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
  62         hw_ip.sram_size = prop->sram_size - sram_kmd_size;
  63         hw_ip.dram_size = prop->dram_size - dram_kmd_size;
  64         if (hw_ip.dram_size > PAGE_SIZE)
  65                 hw_ip.dram_enabled = 1;
  66         hw_ip.num_of_events = prop->num_of_events;
  67
  68         memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version,
  69                 min(VERSION_MAX_LEN, HL_INFO_VERSION_MAX_LEN));
  70
  71         memcpy(hw_ip.card_name, prop->cpucp_info.card_name,
  72                 min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
  73
  74         hw_ip.cpld_version = le32_to_cpu(prop->cpucp_info.cpld_version);
  75         hw_ip.module_id = le32_to_cpu(prop->cpucp_info.card_location);
  76
  77         hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
  78         hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
  79         hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
  80         hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
  81
  82         return copy_to_user(out, &hw_ip,
  83                 min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
  84 }
  85
  86 static int hw_events_info(struct hl_device *hdev, bool aggregate,
  87                         struct hl_info_args *args)
  88 {
  89         u32 size, max_size = args->return_size;
  90         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
  91         void *arr;
  92
  93         if ((!max_size) || (!out))
  94                 return -EINVAL;
  95
  96         arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
  97
  98         return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
  99 }
 100
 101 static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 102 {
 103         struct hl_device *hdev = hpriv->hdev;
 104         struct hl_info_dram_usage dram_usage = {0};
 105         u32 max_size = args->return_size;
 106         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 107         struct asic_fixed_properties *prop = &hdev->asic_prop;
 108         u64 dram_kmd_size;
 109
 110         if ((!max_size) || (!out))
 111                 return -EINVAL;
 112
 113         dram_kmd_size = (prop->dram_user_base_address -
 114                                 prop->dram_base_address);
 115         dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
 116                                         atomic64_read(&hdev->dram_used_mem);
 117         if (hpriv->ctx)
 118                 dram_usage.ctx_dram_mem =
 119                         atomic64_read(&hpriv->ctx->dram_phys_mem);
 120
 121         return copy_to_user(out, &dram_usage,
 122                 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
 123 }
 124
 125 static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
 126 {
 127         struct hl_info_hw_idle hw_idle = {0};
 128         u32 max_size = args->return_size;
 129         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 130
 131         if ((!max_size) || (!out))
 132                 return -EINVAL;
 133
 134         hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
 135                                         &hw_idle.busy_engines_mask_ext, NULL);
 136         hw_idle.busy_engines_mask =
 137                         lower_32_bits(hw_idle.busy_engines_mask_ext);
 138
 139         return copy_to_user(out, &hw_idle,
 140                 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
 141 }
 142
 143 static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
 144 {
 145         struct hl_debug_params *params;
 146         void *input = NULL, *output = NULL;
 147         int rc;
 148
 149         params = kzalloc(sizeof(*params), GFP_KERNEL);
 150         if (!params)
 151                 return -ENOMEM;
 152
 153         params->reg_idx = args->reg_idx;
 154         params->enable = args->enable;
 155         params->op = args->op;
 156
 157         if (args->input_ptr && args->input_size) {
 158                 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
 159                 if (!input) {
 160                         rc = -ENOMEM;
 161                         goto out;
 162                 }
 163
 164                 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
 165                                         args->input_size)) {
 166                         rc = -EFAULT;
 167                         dev_err(hdev->dev, "failed to copy input debug data\n");
 168                         goto out;
 169                 }
 170
 171                 params->input = input;
 172         }
 173
 174         if (args->output_ptr && args->output_size) {
 175                 output = kzalloc(args->output_size, GFP_KERNEL);
 176                 if (!output) {
 177                         rc = -ENOMEM;
 178                         goto out;
 179                 }
 180
 181                 params->output = output;
 182                 params->output_size = args->output_size;
 183         }
 184
 185         rc = hdev->asic_funcs->debug_coresight(hdev, params);
 186         if (rc) {
 187                 dev_err(hdev->dev,
 188                         "debug coresight operation failed %d\n", rc);
 189                 goto out;
 190         }
 191
 192         if (output && copy_to_user((void __user *) (uintptr_t) args->output_ptr,
 193                                         output, args->output_size)) {
 194                 dev_err(hdev->dev, "copy to user failed in debug ioctl\n");
 195                 rc = -EFAULT;
 196                 goto out;
 197         }
 198
 199
 200 out:
 201         kfree(params);
 202         kfree(output);
 203         kfree(input);
 204
 205         return rc;
 206 }
 207
 208 static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
 209 {
 210         struct hl_info_device_utilization device_util = {0};
 211         u32 max_size = args->return_size;
 212         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 213
 214         if ((!max_size) || (!out))
 215                 return -EINVAL;
 216
 217         if ((args->period_ms < 100) || (args->period_ms > 1000) ||
 218                 (args->period_ms % 100)) {
 219                 dev_err(hdev->dev,
 220                         "period %u must be between 100 - 1000 and must be divisible by 100\n",
 221                         args->period_ms);
 222                 return -EINVAL;
 223         }
 224
 225         device_util.utilization = hl_device_utilization(hdev, args->period_ms);
 226
 227         return copy_to_user(out, &device_util,
 228                 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
 229 }
 230
 231 static int get_clk_rate(struct hl_device *hdev, struct hl_info_args *args)
 232 {
 233         struct hl_info_clk_rate clk_rate = {0};
 234         u32 max_size = args->return_size;
 235         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 236         int rc;
 237
 238         if ((!max_size) || (!out))
 239                 return -EINVAL;
 240
 241         rc = hdev->asic_funcs->get_clk_rate(hdev, &clk_rate.cur_clk_rate_mhz,
 242                                                 &clk_rate.max_clk_rate_mhz);
 243         if (rc)
 244                 return rc;
 245
 246         return copy_to_user(out, &clk_rate,
 247                 min((size_t) max_size, sizeof(clk_rate))) ? -EFAULT : 0;
 248 }
 249
 250 static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
 251 {
 252         struct hl_info_reset_count reset_count = {0};
 253         u32 max_size = args->return_size;
 254         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 255
 256         if ((!max_size) || (!out))
 257                 return -EINVAL;
 258
 259         reset_count.hard_reset_cnt = hdev->hard_reset_cnt;
 260         reset_count.soft_reset_cnt = hdev->soft_reset_cnt;
 261
 262         return copy_to_user(out, &reset_count,
 263                 min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
 264 }
 265
 266 static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
 267 {
 268         struct hl_info_time_sync time_sync = {0};
 269         u32 max_size = args->return_size;
 270         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 271
 272         if ((!max_size) || (!out))
 273                 return -EINVAL;
 274
 275         time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
 276         time_sync.host_time = ktime_get_raw_ns();
 277
 278         return copy_to_user(out, &time_sync,
 279                 min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
 280 }
 281
 282 static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 283 {
 284         struct hl_device *hdev = hpriv->hdev;
 285         struct hl_info_pci_counters pci_counters = {0};
 286         u32 max_size = args->return_size;
 287         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 288         int rc;
 289
 290         if ((!max_size) || (!out))
 291                 return -EINVAL;
 292
 293         rc = hl_fw_cpucp_pci_counters_get(hdev, &pci_counters);
 294         if (rc)
 295                 return rc;
 296
 297         return copy_to_user(out, &pci_counters,
 298                 min((size_t) max_size, sizeof(pci_counters))) ? -EFAULT : 0;
 299 }
 300
 301 static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 302 {
 303         struct hl_device *hdev = hpriv->hdev;
 304         struct hl_info_clk_throttle clk_throttle = {0};
 305         u32 max_size = args->return_size;
 306         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 307
 308         if ((!max_size) || (!out))
 309                 return -EINVAL;
 310
 311         clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason;
 312
 313         return copy_to_user(out, &clk_throttle,
 314                 min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0;
 315 }
 316
 317 static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 318 {
 319         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 320         struct hl_info_cs_counters cs_counters = {0};
 321         struct hl_device *hdev = hpriv->hdev;
 322         struct hl_cs_counters_atomic *cntr;
 323         u32 max_size = args->return_size;
 324
 325         cntr = &hdev->aggregated_cs_counters;
 326
 327         if ((!max_size) || (!out))
 328                 return -EINVAL;
 329
 330         cs_counters.total_out_of_mem_drop_cnt =
 331                         atomic64_read(&cntr->out_of_mem_drop_cnt);
 332         cs_counters.total_parsing_drop_cnt =
 333                         atomic64_read(&cntr->parsing_drop_cnt);
 334         cs_counters.total_queue_full_drop_cnt =
 335                         atomic64_read(&cntr->queue_full_drop_cnt);
 336         cs_counters.total_device_in_reset_drop_cnt =
 337                         atomic64_read(&cntr->device_in_reset_drop_cnt);
 338         cs_counters.total_max_cs_in_flight_drop_cnt =
 339                         atomic64_read(&cntr->max_cs_in_flight_drop_cnt);
 340         cs_counters.total_validation_drop_cnt =
 341                         atomic64_read(&cntr->validation_drop_cnt);
 342
 343         if (hpriv->ctx) {
 344                 cs_counters.ctx_out_of_mem_drop_cnt =
 345                                 atomic64_read(
 346                                 &hpriv->ctx->cs_counters.out_of_mem_drop_cnt);
 347                 cs_counters.ctx_parsing_drop_cnt =
 348                                 atomic64_read(
 349                                 &hpriv->ctx->cs_counters.parsing_drop_cnt);
 350                 cs_counters.ctx_queue_full_drop_cnt =
 351                                 atomic64_read(
 352                                 &hpriv->ctx->cs_counters.queue_full_drop_cnt);
 353                 cs_counters.ctx_device_in_reset_drop_cnt =
 354                                 atomic64_read(
 355                         &hpriv->ctx->cs_counters.device_in_reset_drop_cnt);
 356                 cs_counters.ctx_max_cs_in_flight_drop_cnt =
 357                                 atomic64_read(
 358                         &hpriv->ctx->cs_counters.max_cs_in_flight_drop_cnt);
 359                 cs_counters.ctx_validation_drop_cnt =
 360                                 atomic64_read(
 361                                 &hpriv->ctx->cs_counters.validation_drop_cnt);
 362         }
 363
 364         return copy_to_user(out, &cs_counters,
 365                 min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0;
 366 }
 367
 368 static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 369 {
 370         struct hl_device *hdev = hpriv->hdev;
 371         struct asic_fixed_properties *prop = &hdev->asic_prop;
 372         struct hl_info_sync_manager sm_info = {0};
 373         u32 max_size = args->return_size;
 374         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 375
 376         if ((!max_size) || (!out))
 377                 return -EINVAL;
 378
 379         if (args->dcore_id >= HL_MAX_DCORES)
 380                 return -EINVAL;
 381
 382         sm_info.first_available_sync_object =
 383                         prop->first_available_user_sob[args->dcore_id];
 384         sm_info.first_available_monitor =
 385                         prop->first_available_user_mon[args->dcore_id];
 386
 387
 388         return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size,
 389                         sizeof(sm_info))) ? -EFAULT : 0;
 390 }
 391
 392 static int total_energy_consumption_info(struct hl_fpriv *hpriv,
 393                         struct hl_info_args *args)
 394 {
 395         struct hl_device *hdev = hpriv->hdev;
 396         struct hl_info_energy total_energy = {0};
 397         u32 max_size = args->return_size;
 398         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 399         int rc;
 400
 401         if ((!max_size) || (!out))
 402                 return -EINVAL;
 403
 404         rc = hl_fw_cpucp_total_energy_get(hdev,
 405                         &total_energy.total_energy_consumption);
 406         if (rc)
 407                 return rc;
 408
 409         return copy_to_user(out, &total_energy,
 410                 min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0;
 411 }
 412
 413 static int pll_frequency_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
 414 {
 415         struct hl_device *hdev = hpriv->hdev;
 416         struct hl_pll_frequency_info freq_info = { {0} };
 417         u32 max_size = args->return_size;
 418         void __user *out = (void __user *) (uintptr_t) args->return_pointer;
 419         int rc;
 420
 421         if ((!max_size) || (!out))
 422                 return -EINVAL;
 423
 424         rc = hl_fw_cpucp_pll_info_get(hdev, args->pll_index, freq_info.output);
 425         if (rc)
 426                 return rc;
 427
 428         return copy_to_user(out, &freq_info,
 429                 min((size_t) max_size, sizeof(freq_info))) ? -EFAULT : 0;
 430 }
 431
 432 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 433                                 struct device *dev)
 434 {
 435         enum hl_device_status status;
 436         struct hl_info_args *args = data;
 437         struct hl_device *hdev = hpriv->hdev;
 438
 439         int rc;
 440
 441         /*
 442          * Information is returned for the following opcodes even if the device
 443          * is disabled or in reset.
 444          */
 445         switch (args->op) {
 446         case HL_INFO_HW_IP_INFO:
 447                 return hw_ip_info(hdev, args);
 448
 449         case HL_INFO_DEVICE_STATUS:
 450                 return device_status_info(hdev, args);
 451
 452         case HL_INFO_RESET_COUNT:
 453                 return get_reset_count(hdev, args);
 454
 455         default:
 456                 break;
 457         }
 458
 459         if (!hl_device_operational(hdev, &status)) {
 460                 dev_warn_ratelimited(dev,
 461                         "Device is %s. Can't execute INFO IOCTL\n",
 462                         hdev->status[status]);
 463                 return -EBUSY;
 464         }
 465
 466         switch (args->op) {
 467         case HL_INFO_HW_EVENTS:
 468                 rc = hw_events_info(hdev, false, args);
 469                 break;
 470
 471         case HL_INFO_DRAM_USAGE:
 472                 rc = dram_usage_info(hpriv, args);
 473                 break;
 474
 475         case HL_INFO_HW_IDLE:
 476                 rc = hw_idle(hdev, args);
 477                 break;
 478
 479         case HL_INFO_DEVICE_UTILIZATION:
 480                 rc = device_utilization(hdev, args);
 481                 break;
 482
 483         case HL_INFO_HW_EVENTS_AGGREGATE:
 484                 rc = hw_events_info(hdev, true, args);
 485                 break;
 486
 487         case HL_INFO_CLK_RATE:
 488                 rc = get_clk_rate(hdev, args);
 489                 break;
 490
 491         case HL_INFO_TIME_SYNC:
 492                 return time_sync_info(hdev, args);
 493
 494         case HL_INFO_CS_COUNTERS:
 495                 return cs_counters_info(hpriv, args);
 496
 497         case HL_INFO_PCI_COUNTERS:
 498                 return pci_counters_info(hpriv, args);
 499
 500         case HL_INFO_CLK_THROTTLE_REASON:
 501                 return clk_throttle_info(hpriv, args);
 502
 503         case HL_INFO_SYNC_MANAGER:
 504                 return sync_manager_info(hpriv, args);
 505
 506         case HL_INFO_TOTAL_ENERGY:
 507                 return total_energy_consumption_info(hpriv, args);
 508
 509         case HL_INFO_PLL_FREQUENCY:
 510                 return pll_frequency_info(hpriv, args);
 511
 512         default:
 513                 dev_err(dev, "Invalid request %d\n", args->op);
 514                 rc = -ENOTTY;
 515                 break;
 516         }
 517
 518         return rc;
 519 }
 520
 521 static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
 522 {
 523         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
 524 }
 525
 526 static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
 527 {
 528         return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
 529 }
 530
 531 static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
 532 {
 533         struct hl_debug_args *args = data;
 534         struct hl_device *hdev = hpriv->hdev;
 535         enum hl_device_status status;
 536
 537         int rc = 0;
 538
 539         if (!hl_device_operational(hdev, &status)) {
 540                 dev_warn_ratelimited(hdev->dev,
 541                         "Device is %s. Can't execute DEBUG IOCTL\n",
 542                         hdev->status[status]);
 543                 return -EBUSY;
 544         }
 545
 546         switch (args->op) {
 547         case HL_DEBUG_OP_ETR:
 548         case HL_DEBUG_OP_ETF:
 549         case HL_DEBUG_OP_STM:
 550         case HL_DEBUG_OP_FUNNEL:
 551         case HL_DEBUG_OP_BMON:
 552         case HL_DEBUG_OP_SPMU:
 553         case HL_DEBUG_OP_TIMESTAMP:
 554                 if (!hdev->in_debug) {
 555                         dev_err_ratelimited(hdev->dev,
 556                                 "Rejecting debug configuration request because device not in debug mode\n");
 557                         return -EFAULT;
 558                 }
 559                 args->input_size =
 560                         min(args->input_size, hl_debug_struct_size[args->op]);
 561                 rc = debug_coresight(hdev, args);
 562                 break;
 563         case HL_DEBUG_OP_SET_MODE:
 564                 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
 565                 break;
 566         default:
 567                 dev_err(hdev->dev, "Invalid request %d\n", args->op);
 568                 rc = -ENOTTY;
 569                 break;
 570         }
 571
 572         return rc;
 573 }
 574
 575 #define HL_IOCTL_DEF(ioctl, _func) \
 576         [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
 577
 578 static const struct hl_ioctl_desc hl_ioctls[] = {
 579         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
 580         HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
 581         HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
 582         HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
 583         HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
 584         HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
 585 };
 586
 587 static const struct hl_ioctl_desc hl_ioctls_control[] = {
 588         HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
 589 };
 590
 591 static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
 592                 const struct hl_ioctl_desc *ioctl, struct device *dev)
 593 {
 594         struct hl_fpriv *hpriv = filep->private_data;
 595         struct hl_device *hdev = hpriv->hdev;
 596         unsigned int nr = _IOC_NR(cmd);
 597         char stack_kdata[128] = {0};
 598         char *kdata = NULL;
 599         unsigned int usize, asize;
 600         hl_ioctl_t *func;
 601         u32 hl_size;
 602         int retcode;
 603
 604         if (hdev->hard_reset_pending) {
 605                 dev_crit_ratelimited(dev,
 606                         "Device HARD reset pending! Please close FD\n");
 607                 return -ENODEV;
 608         }
 609
 610         /* Do not trust userspace, use our own definition */
 611         func = ioctl->func;
 612
 613         if (unlikely(!func)) {
 614                 dev_dbg(dev, "no function\n");
 615                 retcode = -ENOTTY;
 616                 goto out_err;
 617         }
 618
 619         hl_size = _IOC_SIZE(ioctl->cmd);
 620         usize = asize = _IOC_SIZE(cmd);
 621         if (hl_size > asize)
 622                 asize = hl_size;
 623
 624         cmd = ioctl->cmd;
 625
 626         if (cmd & (IOC_IN | IOC_OUT)) {
 627                 if (asize <= sizeof(stack_kdata)) {
 628                         kdata = stack_kdata;
 629                 } else {
 630                         kdata = kzalloc(asize, GFP_KERNEL);
 631                         if (!kdata) {
 632                                 retcode = -ENOMEM;
 633                                 goto out_err;
 634                         }
 635                 }
 636         }
 637
 638         if (cmd & IOC_IN) {
 639                 if (copy_from_user(kdata, (void __user *)arg, usize)) {
 640                         retcode = -EFAULT;
 641                         goto out_err;
 642                 }
 643         } else if (cmd & IOC_OUT) {
 644                 memset(kdata, 0, usize);
 645         }
 646
 647         retcode = func(hpriv, kdata);
 648
 649         if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, kdata, usize))
 650                 retcode = -EFAULT;
 651
 652 out_err:
 653         if (retcode)
 654                 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
 655                           task_pid_nr(current), cmd, nr);
 656
 657         if (kdata != stack_kdata)
 658                 kfree(kdata);
 659
 660         return retcode;
 661 }
 662
 663 long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 664 {
 665         struct hl_fpriv *hpriv = filep->private_data;
 666         struct hl_device *hdev = hpriv->hdev;
 667         const struct hl_ioctl_desc *ioctl = NULL;
 668         unsigned int nr = _IOC_NR(cmd);
 669
 670         if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
 671                 ioctl = &hl_ioctls[nr];
 672         } else {
 673                 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
 674                         task_pid_nr(current), nr);
 675                 return -ENOTTY;
 676         }
 677
 678         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
 679 }
 680
 681 long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
 682 {
 683         struct hl_fpriv *hpriv = filep->private_data;
 684         struct hl_device *hdev = hpriv->hdev;
 685         const struct hl_ioctl_desc *ioctl = NULL;
 686         unsigned int nr = _IOC_NR(cmd);
 687
 688         if (nr == _IOC_NR(HL_IOCTL_INFO)) {
 689                 ioctl = &hl_ioctls_control[nr];
 690         } else {
 691                 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
 692                         task_pid_nr(current), nr);
 693                 return -ENOTTY;
 694         }
 695
 696         return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
 697 }