drivers/misc/habanalabs/common/command_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/mm.h>
  12 #include <linux/slab.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/genalloc.h>
  15
  16 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
  17 {
  18         if (cb->is_internal)
  19                 gen_pool_free(hdev->internal_cb_pool,
  20                                 cb->kernel_address, cb->size);
  21         else
  22                 hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
  23                                 (void *) (uintptr_t) cb->kernel_address,
  24                                 cb->bus_address);
  25
  26         kfree(cb);
  27 }
  28
  29 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
  30 {
  31         if (cb->is_pool) {
  32                 spin_lock(&hdev->cb_pool_lock);
  33                 list_add(&cb->pool_list, &hdev->cb_pool);
  34                 spin_unlock(&hdev->cb_pool_lock);
  35         } else {
  36                 cb_fini(hdev, cb);
  37         }
  38 }
  39
  40 static void cb_release(struct kref *ref)
  41 {
  42         struct hl_device *hdev;
  43         struct hl_cb *cb;
  44
  45         cb = container_of(ref, struct hl_cb, refcount);
  46         hdev = cb->hdev;
  47
  48         hl_debugfs_remove_cb(cb);
  49
  50         cb_do_release(hdev, cb);
  51 }
  52
  53 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
  54                                         int ctx_id, bool internal_cb)
  55 {
  56         struct hl_cb *cb;
  57         u32 cb_offset;
  58         void *p;
  59
  60         /*
  61          * We use of GFP_ATOMIC here because this function can be called from
  62          * the latency-sensitive code path for command submission. Due to H/W
  63          * limitations in some of the ASICs, the kernel must copy the user CB
  64          * that is designated for an external queue and actually enqueue
  65          * the kernel's copy. Hence, we must never sleep in this code section
  66          * and must use GFP_ATOMIC for all memory allocations.
  67          */
  68         if (ctx_id == HL_KERNEL_ASID_ID)
  69                 cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
  70         else
  71                 cb = kzalloc(sizeof(*cb), GFP_KERNEL);
  72
  73         if (!cb)
  74                 return NULL;
  75
  76         if (internal_cb) {
  77                 p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size);
  78                 if (!p) {
  79                         kfree(cb);
  80                         return NULL;
  81                 }
  82
  83                 cb_offset = p - hdev->internal_cb_pool_virt_addr;
  84                 cb->is_internal = true;
  85                 cb->bus_address =  hdev->internal_cb_va_base + cb_offset;
  86         } else if (ctx_id == HL_KERNEL_ASID_ID) {
  87                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  88                                                 &cb->bus_address, GFP_ATOMIC);
  89         } else {
  90                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  91                                                 &cb->bus_address,
  92                                                 GFP_USER | __GFP_ZERO);
  93         }
  94
  95         if (!p) {
  96                 dev_err(hdev->dev,
  97                         "failed to allocate %d of dma memory for CB\n",
  98                         cb_size);
  99                 kfree(cb);
 100                 return NULL;
 101         }
 102
 103         cb->kernel_address = (u64) (uintptr_t) p;
 104         cb->size = cb_size;
 105
 106         return cb;
 107 }
 108
 109 int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 110                         u32 cb_size, u64 *handle, int ctx_id, bool internal_cb)
 111 {
 112         struct hl_cb *cb;
 113         bool alloc_new_cb = true;
 114         int rc;
 115
 116         /*
 117          * Can't use generic function to check this because of special case
 118          * where we create a CB as part of the reset process
 119          */
 120         if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
 121                                         (ctx_id != HL_KERNEL_ASID_ID))) {
 122                 dev_warn_ratelimited(hdev->dev,
 123                         "Device is disabled or in reset. Can't create new CBs\n");
 124                 rc = -EBUSY;
 125                 goto out_err;
 126         }
 127
 128         if (cb_size > SZ_2M) {
 129                 dev_err(hdev->dev, "CB size %d must be less than %d\n",
 130                         cb_size, SZ_2M);
 131                 rc = -EINVAL;
 132                 goto out_err;
 133         }
 134
 135         if (!internal_cb) {
 136                 /* Minimum allocation must be PAGE SIZE */
 137                 if (cb_size < PAGE_SIZE)
 138                         cb_size = PAGE_SIZE;
 139
 140                 if (ctx_id == HL_KERNEL_ASID_ID &&
 141                                 cb_size <= hdev->asic_prop.cb_pool_cb_size) {
 142
 143                         spin_lock(&hdev->cb_pool_lock);
 144                         if (!list_empty(&hdev->cb_pool)) {
 145                                 cb = list_first_entry(&hdev->cb_pool,
 146                                                 typeof(*cb), pool_list);
 147                                 list_del(&cb->pool_list);
 148                                 spin_unlock(&hdev->cb_pool_lock);
 149                                 alloc_new_cb = false;
 150                         } else {
 151                                 spin_unlock(&hdev->cb_pool_lock);
 152                                 dev_dbg(hdev->dev, "CB pool is empty\n");
 153                         }
 154                 }
 155         }
 156
 157         if (alloc_new_cb) {
 158                 cb = hl_cb_alloc(hdev, cb_size, ctx_id, internal_cb);
 159                 if (!cb) {
 160                         rc = -ENOMEM;
 161                         goto out_err;
 162                 }
 163         }
 164
 165         cb->hdev = hdev;
 166         cb->ctx_id = ctx_id;
 167
 168         spin_lock(&mgr->cb_lock);
 169         rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
 170         spin_unlock(&mgr->cb_lock);
 171
 172         if (rc < 0) {
 173                 dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
 174                 goto release_cb;
 175         }
 176
 177         cb->id = rc;
 178
 179         kref_init(&cb->refcount);
 180         spin_lock_init(&cb->lock);
 181
 182         /*
 183          * idr is 32-bit so we can safely OR it with a mask that is above
 184          * 32 bit
 185          */
 186         *handle = cb->id | HL_MMAP_CB_MASK;
 187         *handle <<= PAGE_SHIFT;
 188
 189         hl_debugfs_add_cb(cb);
 190
 191         return 0;
 192
 193 release_cb:
 194         cb_do_release(hdev, cb);
 195 out_err:
 196         *handle = 0;
 197
 198         return rc;
 199 }
 200
 201 int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
 202 {
 203         struct hl_cb *cb;
 204         u32 handle;
 205         int rc = 0;
 206
 207         /*
 208          * handle was given to user to do mmap, I need to shift it back to
 209          * how the idr module gave it to me
 210          */
 211         cb_handle >>= PAGE_SHIFT;
 212         handle = (u32) cb_handle;
 213
 214         spin_lock(&mgr->cb_lock);
 215
 216         cb = idr_find(&mgr->cb_handles, handle);
 217         if (cb) {
 218                 idr_remove(&mgr->cb_handles, handle);
 219                 spin_unlock(&mgr->cb_lock);
 220                 kref_put(&cb->refcount, cb_release);
 221         } else {
 222                 spin_unlock(&mgr->cb_lock);
 223                 dev_err(hdev->dev,
 224                         "CB destroy failed, no match to handle 0x%x\n", handle);
 225                 rc = -EINVAL;
 226         }
 227
 228         return rc;
 229 }
 230
 231 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 232 {
 233         union hl_cb_args *args = data;
 234         struct hl_device *hdev = hpriv->hdev;
 235         u64 handle = 0;
 236         int rc;
 237
 238         if (hl_device_disabled_or_in_reset(hdev)) {
 239                 dev_warn_ratelimited(hdev->dev,
 240                         "Device is %s. Can't execute CB IOCTL\n",
 241                         atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
 242                 return -EBUSY;
 243         }
 244
 245         switch (args->in.op) {
 246         case HL_CB_OP_CREATE:
 247                 if (args->in.cb_size > HL_MAX_CB_SIZE) {
 248                         dev_err(hdev->dev,
 249                                 "User requested CB size %d must be less than %d\n",
 250                                 args->in.cb_size, HL_MAX_CB_SIZE);
 251                         rc = -EINVAL;
 252                 } else {
 253                         rc = hl_cb_create(hdev, &hpriv->cb_mgr,
 254                                         args->in.cb_size, &handle,
 255                                         hpriv->ctx->asid, false);
 256                 }
 257
 258                 memset(args, 0, sizeof(*args));
 259                 args->out.cb_handle = handle;
 260                 break;
 261
 262         case HL_CB_OP_DESTROY:
 263                 rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
 264                                         args->in.cb_handle);
 265                 break;
 266
 267         default:
 268                 rc = -ENOTTY;
 269                 break;
 270         }
 271
 272         return rc;
 273 }
 274
 275 static void cb_vm_close(struct vm_area_struct *vma)
 276 {
 277         struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
 278         long new_mmap_size;
 279
 280         new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
 281
 282         if (new_mmap_size > 0) {
 283                 cb->mmap_size = new_mmap_size;
 284                 return;
 285         }
 286
 287         spin_lock(&cb->lock);
 288         cb->mmap = false;
 289         spin_unlock(&cb->lock);
 290
 291         hl_cb_put(cb);
 292         vma->vm_private_data = NULL;
 293 }
 294
 295 static const struct vm_operations_struct cb_vm_ops = {
 296         .close = cb_vm_close
 297 };
 298
 299 int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 300 {
 301         struct hl_device *hdev = hpriv->hdev;
 302         struct hl_cb *cb;
 303         phys_addr_t address;
 304         u32 handle, user_cb_size;
 305         int rc;
 306
 307         handle = vma->vm_pgoff;
 308
 309         /* reference was taken here */
 310         cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
 311         if (!cb) {
 312                 dev_err(hdev->dev,
 313                         "CB mmap failed, no match to handle 0x%x\n", handle);
 314                 return -EINVAL;
 315         }
 316
 317         /* Validation check */
 318         user_cb_size = vma->vm_end - vma->vm_start;
 319         if (user_cb_size != ALIGN(cb->size, PAGE_SIZE)) {
 320                 dev_err(hdev->dev,
 321                         "CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
 322                         vma->vm_end - vma->vm_start, cb->size);
 323                 rc = -EINVAL;
 324                 goto put_cb;
 325         }
 326
 327         if (!access_ok((void __user *) (uintptr_t) vma->vm_start,
 328                                                         user_cb_size)) {
 329                 dev_err(hdev->dev,
 330                         "user pointer is invalid - 0x%lx\n",
 331                         vma->vm_start);
 332
 333                 rc = -EINVAL;
 334                 goto put_cb;
 335         }
 336
 337         spin_lock(&cb->lock);
 338
 339         if (cb->mmap) {
 340                 dev_err(hdev->dev,
 341                         "CB mmap failed, CB already mmaped to user\n");
 342                 rc = -EINVAL;
 343                 goto release_lock;
 344         }
 345
 346         cb->mmap = true;
 347
 348         spin_unlock(&cb->lock);
 349
 350         vma->vm_ops = &cb_vm_ops;
 351
 352         /*
 353          * Note: We're transferring the cb reference to
 354          * vma->vm_private_data here.
 355          */
 356
 357         vma->vm_private_data = cb;
 358
 359         /* Calculate address for CB */
 360         address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
 361
 362         rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
 363                                         address, cb->size);
 364
 365         if (rc) {
 366                 spin_lock(&cb->lock);
 367                 cb->mmap = false;
 368                 goto release_lock;
 369         }
 370
 371         cb->mmap_size = cb->size;
 372
 373         return 0;
 374
 375 release_lock:
 376         spin_unlock(&cb->lock);
 377 put_cb:
 378         hl_cb_put(cb);
 379         return rc;
 380 }
 381
 382 struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 383                         u32 handle)
 384 {
 385         struct hl_cb *cb;
 386
 387         spin_lock(&mgr->cb_lock);
 388         cb = idr_find(&mgr->cb_handles, handle);
 389
 390         if (!cb) {
 391                 spin_unlock(&mgr->cb_lock);
 392                 dev_warn(hdev->dev,
 393                         "CB get failed, no match to handle 0x%x\n", handle);
 394                 return NULL;
 395         }
 396
 397         kref_get(&cb->refcount);
 398
 399         spin_unlock(&mgr->cb_lock);
 400
 401         return cb;
 402
 403 }
 404
 405 void hl_cb_put(struct hl_cb *cb)
 406 {
 407         kref_put(&cb->refcount, cb_release);
 408 }
 409
 410 void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
 411 {
 412         spin_lock_init(&mgr->cb_lock);
 413         idr_init(&mgr->cb_handles);
 414 }
 415
 416 void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
 417 {
 418         struct hl_cb *cb;
 419         struct idr *idp;
 420         u32 id;
 421
 422         idp = &mgr->cb_handles;
 423
 424         idr_for_each_entry(idp, cb, id) {
 425                 if (kref_put(&cb->refcount, cb_release) != 1)
 426                         dev_err(hdev->dev,
 427                                 "CB %d for CTX ID %d is still alive\n",
 428                                 id, cb->ctx_id);
 429         }
 430
 431         idr_destroy(&mgr->cb_handles);
 432 }
 433
 434 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
 435                                         bool internal_cb)
 436 {
 437         u64 cb_handle;
 438         struct hl_cb *cb;
 439         int rc;
 440
 441         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
 442                         HL_KERNEL_ASID_ID, internal_cb);
 443         if (rc) {
 444                 dev_err(hdev->dev,
 445                         "Failed to allocate CB for the kernel driver %d\n", rc);
 446                 return NULL;
 447         }
 448
 449         cb_handle >>= PAGE_SHIFT;
 450         cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
 451         /* hl_cb_get should never fail here so use kernel WARN */
 452         WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
 453         if (!cb)
 454                 goto destroy_cb;
 455
 456         return cb;
 457
 458 destroy_cb:
 459         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
 460
 461         return NULL;
 462 }
 463
 464 int hl_cb_pool_init(struct hl_device *hdev)
 465 {
 466         struct hl_cb *cb;
 467         int i;
 468
 469         INIT_LIST_HEAD(&hdev->cb_pool);
 470         spin_lock_init(&hdev->cb_pool_lock);
 471
 472         for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
 473                 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
 474                                 HL_KERNEL_ASID_ID, false);
 475                 if (cb) {
 476                         cb->is_pool = true;
 477                         list_add(&cb->pool_list, &hdev->cb_pool);
 478                 } else {
 479                         hl_cb_pool_fini(hdev);
 480                         return -ENOMEM;
 481                 }
 482         }
 483
 484         return 0;
 485 }
 486
 487 int hl_cb_pool_fini(struct hl_device *hdev)
 488 {
 489         struct hl_cb *cb, *tmp;
 490
 491         list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
 492                 list_del(&cb->pool_list);
 493                 cb_fini(hdev, cb);
 494         }
 495
 496         return 0;
 497 }