drivers/misc/habanalabs/common/command_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/mm.h>
  12 #include <linux/slab.h>
  13 #include <linux/genalloc.h>
  14
  15 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
  16 {
  17         if (cb->is_internal)
  18                 gen_pool_free(hdev->internal_cb_pool,
  19                                 cb->kernel_address, cb->size);
  20         else
  21                 hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
  22                                 (void *) (uintptr_t) cb->kernel_address,
  23                                 cb->bus_address);
  24
  25         kfree(cb);
  26 }
  27
  28 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
  29 {
  30         if (cb->is_pool) {
  31                 spin_lock(&hdev->cb_pool_lock);
  32                 list_add(&cb->pool_list, &hdev->cb_pool);
  33                 spin_unlock(&hdev->cb_pool_lock);
  34         } else {
  35                 cb_fini(hdev, cb);
  36         }
  37 }
  38
  39 static void cb_release(struct kref *ref)
  40 {
  41         struct hl_device *hdev;
  42         struct hl_cb *cb;
  43
  44         cb = container_of(ref, struct hl_cb, refcount);
  45         hdev = cb->hdev;
  46
  47         hl_debugfs_remove_cb(cb);
  48
  49         cb_do_release(hdev, cb);
  50 }
  51
  52 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
  53                                         int ctx_id, bool internal_cb)
  54 {
  55         struct hl_cb *cb;
  56         u32 cb_offset;
  57         void *p;
  58
  59         /*
  60          * We use of GFP_ATOMIC here because this function can be called from
  61          * the latency-sensitive code path for command submission. Due to H/W
  62          * limitations in some of the ASICs, the kernel must copy the user CB
  63          * that is designated for an external queue and actually enqueue
  64          * the kernel's copy. Hence, we must never sleep in this code section
  65          * and must use GFP_ATOMIC for all memory allocations.
  66          */
  67         if (ctx_id == HL_KERNEL_ASID_ID)
  68                 cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
  69         else
  70                 cb = kzalloc(sizeof(*cb), GFP_KERNEL);
  71
  72         if (!cb)
  73                 return NULL;
  74
  75         if (internal_cb) {
  76                 p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size);
  77                 if (!p) {
  78                         kfree(cb);
  79                         return NULL;
  80                 }
  81
  82                 cb_offset = p - hdev->internal_cb_pool_virt_addr;
  83                 cb->is_internal = true;
  84                 cb->bus_address =  hdev->internal_cb_va_base + cb_offset;
  85         } else if (ctx_id == HL_KERNEL_ASID_ID) {
  86                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  87                                                 &cb->bus_address, GFP_ATOMIC);
  88         } else {
  89                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  90                                                 &cb->bus_address,
  91                                                 GFP_USER | __GFP_ZERO);
  92         }
  93
  94         if (!p) {
  95                 dev_err(hdev->dev,
  96                         "failed to allocate %d of dma memory for CB\n",
  97                         cb_size);
  98                 kfree(cb);
  99                 return NULL;
 100         }
 101
 102         cb->kernel_address = (u64) (uintptr_t) p;
 103         cb->size = cb_size;
 104
 105         return cb;
 106 }
 107
 108 int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 109                         u32 cb_size, u64 *handle, int ctx_id, bool internal_cb)
 110 {
 111         struct hl_cb *cb;
 112         bool alloc_new_cb = true;
 113         int rc;
 114
 115         /*
 116          * Can't use generic function to check this because of special case
 117          * where we create a CB as part of the reset process
 118          */
 119         if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
 120                                         (ctx_id != HL_KERNEL_ASID_ID))) {
 121                 dev_warn_ratelimited(hdev->dev,
 122                         "Device is disabled or in reset. Can't create new CBs\n");
 123                 rc = -EBUSY;
 124                 goto out_err;
 125         }
 126
 127         if (cb_size > SZ_2M) {
 128                 dev_err(hdev->dev, "CB size %d must be less than %d\n",
 129                         cb_size, SZ_2M);
 130                 rc = -EINVAL;
 131                 goto out_err;
 132         }
 133
 134         if (!internal_cb) {
 135                 /* Minimum allocation must be PAGE SIZE */
 136                 if (cb_size < PAGE_SIZE)
 137                         cb_size = PAGE_SIZE;
 138
 139                 if (ctx_id == HL_KERNEL_ASID_ID &&
 140                                 cb_size <= hdev->asic_prop.cb_pool_cb_size) {
 141
 142                         spin_lock(&hdev->cb_pool_lock);
 143                         if (!list_empty(&hdev->cb_pool)) {
 144                                 cb = list_first_entry(&hdev->cb_pool,
 145                                                 typeof(*cb), pool_list);
 146                                 list_del(&cb->pool_list);
 147                                 spin_unlock(&hdev->cb_pool_lock);
 148                                 alloc_new_cb = false;
 149                         } else {
 150                                 spin_unlock(&hdev->cb_pool_lock);
 151                                 dev_dbg(hdev->dev, "CB pool is empty\n");
 152                         }
 153                 }
 154         }
 155
 156         if (alloc_new_cb) {
 157                 cb = hl_cb_alloc(hdev, cb_size, ctx_id, internal_cb);
 158                 if (!cb) {
 159                         rc = -ENOMEM;
 160                         goto out_err;
 161                 }
 162         }
 163
 164         cb->hdev = hdev;
 165         cb->ctx_id = ctx_id;
 166
 167         spin_lock(&mgr->cb_lock);
 168         rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
 169         spin_unlock(&mgr->cb_lock);
 170
 171         if (rc < 0) {
 172                 dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
 173                 goto release_cb;
 174         }
 175
 176         cb->id = rc;
 177
 178         kref_init(&cb->refcount);
 179         spin_lock_init(&cb->lock);
 180
 181         /*
 182          * idr is 32-bit so we can safely OR it with a mask that is above
 183          * 32 bit
 184          */
 185         *handle = cb->id | HL_MMAP_CB_MASK;
 186         *handle <<= PAGE_SHIFT;
 187
 188         hl_debugfs_add_cb(cb);
 189
 190         return 0;
 191
 192 release_cb:
 193         cb_do_release(hdev, cb);
 194 out_err:
 195         *handle = 0;
 196
 197         return rc;
 198 }
 199
 200 int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
 201 {
 202         struct hl_cb *cb;
 203         u32 handle;
 204         int rc = 0;
 205
 206         /*
 207          * handle was given to user to do mmap, I need to shift it back to
 208          * how the idr module gave it to me
 209          */
 210         cb_handle >>= PAGE_SHIFT;
 211         handle = (u32) cb_handle;
 212
 213         spin_lock(&mgr->cb_lock);
 214
 215         cb = idr_find(&mgr->cb_handles, handle);
 216         if (cb) {
 217                 idr_remove(&mgr->cb_handles, handle);
 218                 spin_unlock(&mgr->cb_lock);
 219                 kref_put(&cb->refcount, cb_release);
 220         } else {
 221                 spin_unlock(&mgr->cb_lock);
 222                 dev_err(hdev->dev,
 223                         "CB destroy failed, no match to handle 0x%x\n", handle);
 224                 rc = -EINVAL;
 225         }
 226
 227         return rc;
 228 }
 229
 230 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 231 {
 232         union hl_cb_args *args = data;
 233         struct hl_device *hdev = hpriv->hdev;
 234         u64 handle = 0;
 235         int rc;
 236
 237         if (hl_device_disabled_or_in_reset(hdev)) {
 238                 dev_warn_ratelimited(hdev->dev,
 239                         "Device is %s. Can't execute CB IOCTL\n",
 240                         atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
 241                 return -EBUSY;
 242         }
 243
 244         switch (args->in.op) {
 245         case HL_CB_OP_CREATE:
 246                 if (args->in.cb_size > HL_MAX_CB_SIZE) {
 247                         dev_err(hdev->dev,
 248                                 "User requested CB size %d must be less than %d\n",
 249                                 args->in.cb_size, HL_MAX_CB_SIZE);
 250                         rc = -EINVAL;
 251                 } else {
 252                         rc = hl_cb_create(hdev, &hpriv->cb_mgr,
 253                                         args->in.cb_size, &handle,
 254                                         hpriv->ctx->asid, false);
 255                 }
 256
 257                 memset(args, 0, sizeof(*args));
 258                 args->out.cb_handle = handle;
 259                 break;
 260
 261         case HL_CB_OP_DESTROY:
 262                 rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
 263                                         args->in.cb_handle);
 264                 break;
 265
 266         default:
 267                 rc = -ENOTTY;
 268                 break;
 269         }
 270
 271         return rc;
 272 }
 273
 274 static void cb_vm_close(struct vm_area_struct *vma)
 275 {
 276         struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
 277         long new_mmap_size;
 278
 279         new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
 280
 281         if (new_mmap_size > 0) {
 282                 cb->mmap_size = new_mmap_size;
 283                 return;
 284         }
 285
 286         spin_lock(&cb->lock);
 287         cb->mmap = false;
 288         spin_unlock(&cb->lock);
 289
 290         hl_cb_put(cb);
 291         vma->vm_private_data = NULL;
 292 }
 293
 294 static const struct vm_operations_struct cb_vm_ops = {
 295         .close = cb_vm_close
 296 };
 297
 298 int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 299 {
 300         struct hl_device *hdev = hpriv->hdev;
 301         struct hl_cb *cb;
 302         phys_addr_t address;
 303         u32 handle;
 304         int rc;
 305
 306         handle = vma->vm_pgoff;
 307
 308         /* reference was taken here */
 309         cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
 310         if (!cb) {
 311                 dev_err(hdev->dev,
 312                         "CB mmap failed, no match to handle 0x%x\n", handle);
 313                 return -EINVAL;
 314         }
 315
 316         /* Validation check */
 317         if ((vma->vm_end - vma->vm_start) != ALIGN(cb->size, PAGE_SIZE)) {
 318                 dev_err(hdev->dev,
 319                         "CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
 320                         vma->vm_end - vma->vm_start, cb->size);
 321                 rc = -EINVAL;
 322                 goto put_cb;
 323         }
 324
 325         spin_lock(&cb->lock);
 326
 327         if (cb->mmap) {
 328                 dev_err(hdev->dev,
 329                         "CB mmap failed, CB already mmaped to user\n");
 330                 rc = -EINVAL;
 331                 goto release_lock;
 332         }
 333
 334         cb->mmap = true;
 335
 336         spin_unlock(&cb->lock);
 337
 338         vma->vm_ops = &cb_vm_ops;
 339
 340         /*
 341          * Note: We're transferring the cb reference to
 342          * vma->vm_private_data here.
 343          */
 344
 345         vma->vm_private_data = cb;
 346
 347         /* Calculate address for CB */
 348         address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
 349
 350         rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
 351                                         address, cb->size);
 352
 353         if (rc) {
 354                 spin_lock(&cb->lock);
 355                 cb->mmap = false;
 356                 goto release_lock;
 357         }
 358
 359         cb->mmap_size = cb->size;
 360
 361         return 0;
 362
 363 release_lock:
 364         spin_unlock(&cb->lock);
 365 put_cb:
 366         hl_cb_put(cb);
 367         return rc;
 368 }
 369
 370 struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 371                         u32 handle)
 372 {
 373         struct hl_cb *cb;
 374
 375         spin_lock(&mgr->cb_lock);
 376         cb = idr_find(&mgr->cb_handles, handle);
 377
 378         if (!cb) {
 379                 spin_unlock(&mgr->cb_lock);
 380                 dev_warn(hdev->dev,
 381                         "CB get failed, no match to handle 0x%x\n", handle);
 382                 return NULL;
 383         }
 384
 385         kref_get(&cb->refcount);
 386
 387         spin_unlock(&mgr->cb_lock);
 388
 389         return cb;
 390
 391 }
 392
 393 void hl_cb_put(struct hl_cb *cb)
 394 {
 395         kref_put(&cb->refcount, cb_release);
 396 }
 397
 398 void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
 399 {
 400         spin_lock_init(&mgr->cb_lock);
 401         idr_init(&mgr->cb_handles);
 402 }
 403
 404 void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
 405 {
 406         struct hl_cb *cb;
 407         struct idr *idp;
 408         u32 id;
 409
 410         idp = &mgr->cb_handles;
 411
 412         idr_for_each_entry(idp, cb, id) {
 413                 if (kref_put(&cb->refcount, cb_release) != 1)
 414                         dev_err(hdev->dev,
 415                                 "CB %d for CTX ID %d is still alive\n",
 416                                 id, cb->ctx_id);
 417         }
 418
 419         idr_destroy(&mgr->cb_handles);
 420 }
 421
 422 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
 423                                         bool internal_cb)
 424 {
 425         u64 cb_handle;
 426         struct hl_cb *cb;
 427         int rc;
 428
 429         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
 430                         HL_KERNEL_ASID_ID, internal_cb);
 431         if (rc) {
 432                 dev_err(hdev->dev,
 433                         "Failed to allocate CB for the kernel driver %d\n", rc);
 434                 return NULL;
 435         }
 436
 437         cb_handle >>= PAGE_SHIFT;
 438         cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
 439         /* hl_cb_get should never fail here so use kernel WARN */
 440         WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
 441         if (!cb)
 442                 goto destroy_cb;
 443
 444         return cb;
 445
 446 destroy_cb:
 447         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
 448
 449         return NULL;
 450 }
 451
 452 int hl_cb_pool_init(struct hl_device *hdev)
 453 {
 454         struct hl_cb *cb;
 455         int i;
 456
 457         INIT_LIST_HEAD(&hdev->cb_pool);
 458         spin_lock_init(&hdev->cb_pool_lock);
 459
 460         for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
 461                 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
 462                                 HL_KERNEL_ASID_ID, false);
 463                 if (cb) {
 464                         cb->is_pool = true;
 465                         list_add(&cb->pool_list, &hdev->cb_pool);
 466                 } else {
 467                         hl_cb_pool_fini(hdev);
 468                         return -ENOMEM;
 469                 }
 470         }
 471
 472         return 0;
 473 }
 474
 475 int hl_cb_pool_fini(struct hl_device *hdev)
 476 {
 477         struct hl_cb *cb, *tmp;
 478
 479         list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
 480                 list_del(&cb->pool_list);
 481                 cb_fini(hdev, cb);
 482         }
 483
 484         return 0;
 485 }