drivers/misc/habanalabs/command_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include <uapi/misc/habanalabs.h>
   9 #include "habanalabs.h"
  10
  11 #include <linux/mm.h>
  12 #include <linux/slab.h>
  13
  14 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
  15 {
  16         hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
  17                         (void *) (uintptr_t) cb->kernel_address,
  18                         cb->bus_address);
  19         kfree(cb);
  20 }
  21
  22 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
  23 {
  24         if (cb->is_pool) {
  25                 spin_lock(&hdev->cb_pool_lock);
  26                 list_add(&cb->pool_list, &hdev->cb_pool);
  27                 spin_unlock(&hdev->cb_pool_lock);
  28         } else {
  29                 cb_fini(hdev, cb);
  30         }
  31 }
  32
  33 static void cb_release(struct kref *ref)
  34 {
  35         struct hl_device *hdev;
  36         struct hl_cb *cb;
  37
  38         cb = container_of(ref, struct hl_cb, refcount);
  39         hdev = cb->hdev;
  40
  41         hl_debugfs_remove_cb(cb);
  42
  43         cb_do_release(hdev, cb);
  44 }
  45
  46 static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
  47                                         int ctx_id)
  48 {
  49         struct hl_cb *cb;
  50         void *p;
  51
  52         /*
  53          * We use of GFP_ATOMIC here because this function can be called from
  54          * the latency-sensitive code path for command submission. Due to H/W
  55          * limitations in some of the ASICs, the kernel must copy the user CB
  56          * that is designated for an external queue and actually enqueue
  57          * the kernel's copy. Hence, we must never sleep in this code section
  58          * and must use GFP_ATOMIC for all memory allocations.
  59          */
  60         if (ctx_id == HL_KERNEL_ASID_ID)
  61                 cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
  62         else
  63                 cb = kzalloc(sizeof(*cb), GFP_KERNEL);
  64
  65         if (!cb)
  66                 return NULL;
  67
  68         if (ctx_id == HL_KERNEL_ASID_ID)
  69                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  70                                                 &cb->bus_address, GFP_ATOMIC);
  71         else
  72                 p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
  73                                                 &cb->bus_address,
  74                                                 GFP_USER | __GFP_ZERO);
  75         if (!p) {
  76                 dev_err(hdev->dev,
  77                         "failed to allocate %d of dma memory for CB\n",
  78                         cb_size);
  79                 kfree(cb);
  80                 return NULL;
  81         }
  82
  83         cb->kernel_address = (u64) (uintptr_t) p;
  84         cb->size = cb_size;
  85
  86         return cb;
  87 }
  88
  89 int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
  90                         u32 cb_size, u64 *handle, int ctx_id)
  91 {
  92         struct hl_cb *cb;
  93         bool alloc_new_cb = true;
  94         int rc;
  95
  96         /*
  97          * Can't use generic function to check this because of special case
  98          * where we create a CB as part of the reset process
  99          */
 100         if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
 101                                         (ctx_id != HL_KERNEL_ASID_ID))) {
 102                 dev_warn_ratelimited(hdev->dev,
 103                         "Device is disabled or in reset. Can't create new CBs\n");
 104                 rc = -EBUSY;
 105                 goto out_err;
 106         }
 107
 108         if (cb_size > SZ_2M) {
 109                 dev_err(hdev->dev, "CB size %d must be less than %d\n",
 110                         cb_size, SZ_2M);
 111                 rc = -EINVAL;
 112                 goto out_err;
 113         }
 114
 115         /* Minimum allocation must be PAGE SIZE */
 116         if (cb_size < PAGE_SIZE)
 117                 cb_size = PAGE_SIZE;
 118
 119         if (ctx_id == HL_KERNEL_ASID_ID &&
 120                         cb_size <= hdev->asic_prop.cb_pool_cb_size) {
 121
 122                 spin_lock(&hdev->cb_pool_lock);
 123                 if (!list_empty(&hdev->cb_pool)) {
 124                         cb = list_first_entry(&hdev->cb_pool, typeof(*cb),
 125                                         pool_list);
 126                         list_del(&cb->pool_list);
 127                         spin_unlock(&hdev->cb_pool_lock);
 128                         alloc_new_cb = false;
 129                 } else {
 130                         spin_unlock(&hdev->cb_pool_lock);
 131                         dev_dbg(hdev->dev, "CB pool is empty\n");
 132                 }
 133         }
 134
 135         if (alloc_new_cb) {
 136                 cb = hl_cb_alloc(hdev, cb_size, ctx_id);
 137                 if (!cb) {
 138                         rc = -ENOMEM;
 139                         goto out_err;
 140                 }
 141         }
 142
 143         cb->hdev = hdev;
 144         cb->ctx_id = ctx_id;
 145
 146         spin_lock(&mgr->cb_lock);
 147         rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
 148         spin_unlock(&mgr->cb_lock);
 149
 150         if (rc < 0) {
 151                 dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
 152                 goto release_cb;
 153         }
 154
 155         cb->id = rc;
 156
 157         kref_init(&cb->refcount);
 158         spin_lock_init(&cb->lock);
 159
 160         /*
 161          * idr is 32-bit so we can safely OR it with a mask that is above
 162          * 32 bit
 163          */
 164         *handle = cb->id | HL_MMAP_CB_MASK;
 165         *handle <<= PAGE_SHIFT;
 166
 167         hl_debugfs_add_cb(cb);
 168
 169         return 0;
 170
 171 release_cb:
 172         cb_do_release(hdev, cb);
 173 out_err:
 174         *handle = 0;
 175
 176         return rc;
 177 }
 178
 179 int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
 180 {
 181         struct hl_cb *cb;
 182         u32 handle;
 183         int rc = 0;
 184
 185         /*
 186          * handle was given to user to do mmap, I need to shift it back to
 187          * how the idr module gave it to me
 188          */
 189         cb_handle >>= PAGE_SHIFT;
 190         handle = (u32) cb_handle;
 191
 192         spin_lock(&mgr->cb_lock);
 193
 194         cb = idr_find(&mgr->cb_handles, handle);
 195         if (cb) {
 196                 idr_remove(&mgr->cb_handles, handle);
 197                 spin_unlock(&mgr->cb_lock);
 198                 kref_put(&cb->refcount, cb_release);
 199         } else {
 200                 spin_unlock(&mgr->cb_lock);
 201                 dev_err(hdev->dev,
 202                         "CB destroy failed, no match to handle 0x%x\n", handle);
 203                 rc = -EINVAL;
 204         }
 205
 206         return rc;
 207 }
 208
 209 int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 210 {
 211         union hl_cb_args *args = data;
 212         struct hl_device *hdev = hpriv->hdev;
 213         u64 handle = 0;
 214         int rc;
 215
 216         if (hl_device_disabled_or_in_reset(hdev)) {
 217                 dev_warn_ratelimited(hdev->dev,
 218                         "Device is %s. Can't execute CB IOCTL\n",
 219                         atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
 220                 return -EBUSY;
 221         }
 222
 223         switch (args->in.op) {
 224         case HL_CB_OP_CREATE:
 225                 if (args->in.cb_size > HL_MAX_CB_SIZE) {
 226                         dev_err(hdev->dev,
 227                                 "User requested CB size %d must be less than %d\n",
 228                                 args->in.cb_size, HL_MAX_CB_SIZE);
 229                         rc = -EINVAL;
 230                 } else {
 231                         rc = hl_cb_create(hdev, &hpriv->cb_mgr,
 232                                                 args->in.cb_size, &handle,
 233                                                 hpriv->ctx->asid);
 234                 }
 235
 236                 memset(args, 0, sizeof(*args));
 237                 args->out.cb_handle = handle;
 238                 break;
 239
 240         case HL_CB_OP_DESTROY:
 241                 rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
 242                                         args->in.cb_handle);
 243                 break;
 244
 245         default:
 246                 rc = -ENOTTY;
 247                 break;
 248         }
 249
 250         return rc;
 251 }
 252
 253 static void cb_vm_close(struct vm_area_struct *vma)
 254 {
 255         struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
 256         long new_mmap_size;
 257
 258         new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
 259
 260         if (new_mmap_size > 0) {
 261                 cb->mmap_size = new_mmap_size;
 262                 return;
 263         }
 264
 265         spin_lock(&cb->lock);
 266         cb->mmap = false;
 267         spin_unlock(&cb->lock);
 268
 269         hl_cb_put(cb);
 270         vma->vm_private_data = NULL;
 271 }
 272
 273 static const struct vm_operations_struct cb_vm_ops = {
 274         .close = cb_vm_close
 275 };
 276
 277 int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 278 {
 279         struct hl_device *hdev = hpriv->hdev;
 280         struct hl_cb *cb;
 281         phys_addr_t address;
 282         u32 handle;
 283         int rc;
 284
 285         handle = vma->vm_pgoff;
 286
 287         /* reference was taken here */
 288         cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
 289         if (!cb) {
 290                 dev_err(hdev->dev,
 291                         "CB mmap failed, no match to handle 0x%x\n", handle);
 292                 return -EINVAL;
 293         }
 294
 295         /* Validation check */
 296         if ((vma->vm_end - vma->vm_start) != ALIGN(cb->size, PAGE_SIZE)) {
 297                 dev_err(hdev->dev,
 298                         "CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
 299                         vma->vm_end - vma->vm_start, cb->size);
 300                 rc = -EINVAL;
 301                 goto put_cb;
 302         }
 303
 304         spin_lock(&cb->lock);
 305
 306         if (cb->mmap) {
 307                 dev_err(hdev->dev,
 308                         "CB mmap failed, CB already mmaped to user\n");
 309                 rc = -EINVAL;
 310                 goto release_lock;
 311         }
 312
 313         cb->mmap = true;
 314
 315         spin_unlock(&cb->lock);
 316
 317         vma->vm_ops = &cb_vm_ops;
 318
 319         /*
 320          * Note: We're transferring the cb reference to
 321          * vma->vm_private_data here.
 322          */
 323
 324         vma->vm_private_data = cb;
 325
 326         /* Calculate address for CB */
 327         address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
 328
 329         rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
 330                                         address, cb->size);
 331
 332         if (rc) {
 333                 spin_lock(&cb->lock);
 334                 cb->mmap = false;
 335                 goto release_lock;
 336         }
 337
 338         cb->mmap_size = cb->size;
 339
 340         return 0;
 341
 342 release_lock:
 343         spin_unlock(&cb->lock);
 344 put_cb:
 345         hl_cb_put(cb);
 346         return rc;
 347 }
 348
 349 struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 350                         u32 handle)
 351 {
 352         struct hl_cb *cb;
 353
 354         spin_lock(&mgr->cb_lock);
 355         cb = idr_find(&mgr->cb_handles, handle);
 356
 357         if (!cb) {
 358                 spin_unlock(&mgr->cb_lock);
 359                 dev_warn(hdev->dev,
 360                         "CB get failed, no match to handle 0x%x\n", handle);
 361                 return NULL;
 362         }
 363
 364         kref_get(&cb->refcount);
 365
 366         spin_unlock(&mgr->cb_lock);
 367
 368         return cb;
 369
 370 }
 371
 372 void hl_cb_put(struct hl_cb *cb)
 373 {
 374         kref_put(&cb->refcount, cb_release);
 375 }
 376
 377 void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
 378 {
 379         spin_lock_init(&mgr->cb_lock);
 380         idr_init(&mgr->cb_handles);
 381 }
 382
 383 void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
 384 {
 385         struct hl_cb *cb;
 386         struct idr *idp;
 387         u32 id;
 388
 389         idp = &mgr->cb_handles;
 390
 391         idr_for_each_entry(idp, cb, id) {
 392                 if (kref_put(&cb->refcount, cb_release) != 1)
 393                         dev_err(hdev->dev,
 394                                 "CB %d for CTX ID %d is still alive\n",
 395                                 id, cb->ctx_id);
 396         }
 397
 398         idr_destroy(&mgr->cb_handles);
 399 }
 400
 401 struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size)
 402 {
 403         u64 cb_handle;
 404         struct hl_cb *cb;
 405         int rc;
 406
 407         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
 408                         HL_KERNEL_ASID_ID);
 409         if (rc) {
 410                 dev_err(hdev->dev,
 411                         "Failed to allocate CB for the kernel driver %d\n", rc);
 412                 return NULL;
 413         }
 414
 415         cb_handle >>= PAGE_SHIFT;
 416         cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
 417         /* hl_cb_get should never fail here so use kernel WARN */
 418         WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
 419         if (!cb)
 420                 goto destroy_cb;
 421
 422         return cb;
 423
 424 destroy_cb:
 425         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
 426
 427         return NULL;
 428 }
 429
 430 int hl_cb_pool_init(struct hl_device *hdev)
 431 {
 432         struct hl_cb *cb;
 433         int i;
 434
 435         INIT_LIST_HEAD(&hdev->cb_pool);
 436         spin_lock_init(&hdev->cb_pool_lock);
 437
 438         for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
 439                 cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
 440                                 HL_KERNEL_ASID_ID);
 441                 if (cb) {
 442                         cb->is_pool = true;
 443                         list_add(&cb->pool_list, &hdev->cb_pool);
 444                 } else {
 445                         hl_cb_pool_fini(hdev);
 446                         return -ENOMEM;
 447                 }
 448         }
 449
 450         return 0;
 451 }
 452
 453 int hl_cb_pool_fini(struct hl_device *hdev)
 454 {
 455         struct hl_cb *cb, *tmp;
 456
 457         list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
 458                 list_del(&cb->pool_list);
 459                 cb_fini(hdev, cb);
 460         }
 461
 462         return 0;
 463 }