drivers/gpu/drm/lima/lima_sched.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
   3
   4 #include <linux/kthread.h>
   5 #include <linux/slab.h>
   6 #include <linux/vmalloc.h>
   7 #include <linux/pm_runtime.h>
   8
   9 #include "lima_devfreq.h"
  10 #include "lima_drv.h"
  11 #include "lima_sched.h"
  12 #include "lima_vm.h"
  13 #include "lima_mmu.h"
  14 #include "lima_l2_cache.h"
  15 #include "lima_gem.h"
  16 #include "lima_trace.h"
  17
  18 struct lima_fence {
  19         struct dma_fence base;
  20         struct lima_sched_pipe *pipe;
  21 };
  22
  23 static struct kmem_cache *lima_fence_slab;
  24 static int lima_fence_slab_refcnt;
  25
  26 int lima_sched_slab_init(void)
  27 {
  28         if (!lima_fence_slab) {
  29                 lima_fence_slab = kmem_cache_create(
  30                         "lima_fence", sizeof(struct lima_fence), 0,
  31                         SLAB_HWCACHE_ALIGN, NULL);
  32                 if (!lima_fence_slab)
  33                         return -ENOMEM;
  34         }
  35
  36         lima_fence_slab_refcnt++;
  37         return 0;
  38 }
  39
  40 void lima_sched_slab_fini(void)
  41 {
  42         if (!--lima_fence_slab_refcnt) {
  43                 kmem_cache_destroy(lima_fence_slab);
  44                 lima_fence_slab = NULL;
  45         }
  46 }
  47
  48 static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
  49 {
  50         return container_of(fence, struct lima_fence, base);
  51 }
  52
  53 static const char *lima_fence_get_driver_name(struct dma_fence *fence)
  54 {
  55         return "lima";
  56 }
  57
  58 static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
  59 {
  60         struct lima_fence *f = to_lima_fence(fence);
  61
  62         return f->pipe->base.name;
  63 }
  64
  65 static void lima_fence_release_rcu(struct rcu_head *rcu)
  66 {
  67         struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
  68         struct lima_fence *fence = to_lima_fence(f);
  69
  70         kmem_cache_free(lima_fence_slab, fence);
  71 }
  72
  73 static void lima_fence_release(struct dma_fence *fence)
  74 {
  75         struct lima_fence *f = to_lima_fence(fence);
  76
  77         call_rcu(&f->base.rcu, lima_fence_release_rcu);
  78 }
  79
  80 static const struct dma_fence_ops lima_fence_ops = {
  81         .get_driver_name = lima_fence_get_driver_name,
  82         .get_timeline_name = lima_fence_get_timeline_name,
  83         .release = lima_fence_release,
  84 };
  85
  86 static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
  87 {
  88         struct lima_fence *fence;
  89
  90         fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
  91         if (!fence)
  92                 return NULL;
  93
  94         fence->pipe = pipe;
  95         dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
  96                        pipe->fence_context, ++pipe->fence_seqno);
  97
  98         return fence;
  99 }
 100
 101 static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
 102 {
 103         return container_of(job, struct lima_sched_task, base);
 104 }
 105
 106 static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
 107 {
 108         return container_of(sched, struct lima_sched_pipe, base);
 109 }
 110
 111 int lima_sched_task_init(struct lima_sched_task *task,
 112                          struct lima_sched_context *context,
 113                          struct lima_bo **bos, int num_bos,
 114                          struct lima_vm *vm)
 115 {
 116         int err, i;
 117
 118         task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
 119         if (!task->bos)
 120                 return -ENOMEM;
 121
 122         for (i = 0; i < num_bos; i++)
 123                 drm_gem_object_get(&bos[i]->base.base);
 124
 125         err = drm_sched_job_init(&task->base, &context->base, vm);
 126         if (err) {
 127                 kfree(task->bos);
 128                 return err;
 129         }
 130
 131         task->num_bos = num_bos;
 132         task->vm = lima_vm_get(vm);
 133
 134         xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
 135
 136         return 0;
 137 }
 138
 139 void lima_sched_task_fini(struct lima_sched_task *task)
 140 {
 141         struct dma_fence *fence;
 142         unsigned long index;
 143         int i;
 144
 145         drm_sched_job_cleanup(&task->base);
 146
 147         xa_for_each(&task->deps, index, fence) {
 148                 dma_fence_put(fence);
 149         }
 150         xa_destroy(&task->deps);
 151
 152         if (task->bos) {
 153                 for (i = 0; i < task->num_bos; i++)
 154                         drm_gem_object_put(&task->bos[i]->base.base);
 155                 kfree(task->bos);
 156         }
 157
 158         lima_vm_put(task->vm);
 159 }
 160
 161 int lima_sched_context_init(struct lima_sched_pipe *pipe,
 162                             struct lima_sched_context *context,
 163                             atomic_t *guilty)
 164 {
 165         struct drm_gpu_scheduler *sched = &pipe->base;
 166
 167         return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,
 168                                      &sched, 1, guilty);
 169 }
 170
 171 void lima_sched_context_fini(struct lima_sched_pipe *pipe,
 172                              struct lima_sched_context *context)
 173 {
 174         drm_sched_entity_fini(&context->base);
 175 }
 176
 177 struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
 178                                                 struct lima_sched_task *task)
 179 {
 180         struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
 181
 182         trace_lima_task_submit(task);
 183         drm_sched_entity_push_job(&task->base, &context->base);
 184         return fence;
 185 }
 186
 187 static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
 188                                                struct drm_sched_entity *entity)
 189 {
 190         struct lima_sched_task *task = to_lima_task(job);
 191
 192         if (!xa_empty(&task->deps))
 193                 return xa_erase(&task->deps, task->last_dep++);
 194
 195         return NULL;
 196 }
 197
 198 static int lima_pm_busy(struct lima_device *ldev)
 199 {
 200         int ret;
 201
 202         /* resume GPU if it has been suspended by runtime PM */
 203         ret = pm_runtime_get_sync(ldev->dev);
 204         if (ret < 0)
 205                 return ret;
 206
 207         lima_devfreq_record_busy(&ldev->devfreq);
 208         return 0;
 209 }
 210
 211 static void lima_pm_idle(struct lima_device *ldev)
 212 {
 213         lima_devfreq_record_idle(&ldev->devfreq);
 214
 215         /* GPU can do auto runtime suspend */
 216         pm_runtime_mark_last_busy(ldev->dev);
 217         pm_runtime_put_autosuspend(ldev->dev);
 218 }
 219
 220 static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
 221 {
 222         struct lima_sched_task *task = to_lima_task(job);
 223         struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
 224         struct lima_device *ldev = pipe->ldev;
 225         struct lima_fence *fence;
 226         struct dma_fence *ret;
 227         int i, err;
 228
 229         /* after GPU reset */
 230         if (job->s_fence->finished.error < 0)
 231                 return NULL;
 232
 233         fence = lima_fence_create(pipe);
 234         if (!fence)
 235                 return NULL;
 236
 237         err = lima_pm_busy(ldev);
 238         if (err < 0) {
 239                 dma_fence_put(&fence->base);
 240                 return NULL;
 241         }
 242
 243         task->fence = &fence->base;
 244
 245         /* for caller usage of the fence, otherwise irq handler
 246          * may consume the fence before caller use it
 247          */
 248         ret = dma_fence_get(task->fence);
 249
 250         pipe->current_task = task;
 251
 252         /* this is needed for MMU to work correctly, otherwise GP/PP
 253          * will hang or page fault for unknown reason after running for
 254          * a while.
 255          *
 256          * Need to investigate:
 257          * 1. is it related to TLB
 258          * 2. how much performance will be affected by L2 cache flush
 259          * 3. can we reduce the calling of this function because all
 260          *    GP/PP use the same L2 cache on mali400
 261          *
 262          * TODO:
 263          * 1. move this to task fini to save some wait time?
 264          * 2. when GP/PP use different l2 cache, need PP wait GP l2
 265          *    cache flush?
 266          */
 267         for (i = 0; i < pipe->num_l2_cache; i++)
 268                 lima_l2_cache_flush(pipe->l2_cache[i]);
 269
 270         lima_vm_put(pipe->current_vm);
 271         pipe->current_vm = lima_vm_get(task->vm);
 272
 273         if (pipe->bcast_mmu)
 274                 lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);
 275         else {
 276                 for (i = 0; i < pipe->num_mmu; i++)
 277                         lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);
 278         }
 279
 280         trace_lima_task_run(task);
 281
 282         pipe->error = false;
 283         pipe->task_run(pipe, task);
 284
 285         return task->fence;
 286 }
 287
 288 static void lima_sched_build_error_task_list(struct lima_sched_task *task)
 289 {
 290         struct lima_sched_error_task *et;
 291         struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);
 292         struct lima_ip *ip = pipe->processor[0];
 293         int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;
 294         struct lima_device *dev = ip->dev;
 295         struct lima_sched_context *sched_ctx =
 296                 container_of(task->base.entity,
 297                              struct lima_sched_context, base);
 298         struct lima_ctx *ctx =
 299                 container_of(sched_ctx, struct lima_ctx, context[pipe_id]);
 300         struct lima_dump_task *dt;
 301         struct lima_dump_chunk *chunk;
 302         struct lima_dump_chunk_pid *pid_chunk;
 303         struct lima_dump_chunk_buffer *buffer_chunk;
 304         u32 size, task_size, mem_size;
 305         int i;
 306
 307         mutex_lock(&dev->error_task_list_lock);
 308
 309         if (dev->dump.num_tasks >= lima_max_error_tasks) {
 310                 dev_info(dev->dev, "fail to save task state from %s pid %d: "
 311                          "error task list is full\n", ctx->pname, ctx->pid);
 312                 goto out;
 313         }
 314
 315         /* frame chunk */
 316         size = sizeof(struct lima_dump_chunk) + pipe->frame_size;
 317         /* process name chunk */
 318         size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);
 319         /* pid chunk */
 320         size += sizeof(struct lima_dump_chunk);
 321         /* buffer chunks */
 322         for (i = 0; i < task->num_bos; i++) {
 323                 struct lima_bo *bo = task->bos[i];
 324
 325                 size += sizeof(struct lima_dump_chunk);
 326                 size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);
 327         }
 328
 329         task_size = size + sizeof(struct lima_dump_task);
 330         mem_size = task_size + sizeof(*et);
 331         et = kvmalloc(mem_size, GFP_KERNEL);
 332         if (!et) {
 333                 dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",
 334                         mem_size);
 335                 goto out;
 336         }
 337
 338         et->data = et + 1;
 339         et->size = task_size;
 340
 341         dt = et->data;
 342         memset(dt, 0, sizeof(*dt));
 343         dt->id = pipe_id;
 344         dt->size = size;
 345
 346         chunk = (struct lima_dump_chunk *)(dt + 1);
 347         memset(chunk, 0, sizeof(*chunk));
 348         chunk->id = LIMA_DUMP_CHUNK_FRAME;
 349         chunk->size = pipe->frame_size;
 350         memcpy(chunk + 1, task->frame, pipe->frame_size);
 351         dt->num_chunks++;
 352
 353         chunk = (void *)(chunk + 1) + chunk->size;
 354         memset(chunk, 0, sizeof(*chunk));
 355         chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;
 356         chunk->size = sizeof(ctx->pname);
 357         memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));
 358         dt->num_chunks++;
 359
 360         pid_chunk = (void *)(chunk + 1) + chunk->size;
 361         memset(pid_chunk, 0, sizeof(*pid_chunk));
 362         pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;
 363         pid_chunk->pid = ctx->pid;
 364         dt->num_chunks++;
 365
 366         buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;
 367         for (i = 0; i < task->num_bos; i++) {
 368                 struct lima_bo *bo = task->bos[i];
 369                 void *data;
 370
 371                 memset(buffer_chunk, 0, sizeof(*buffer_chunk));
 372                 buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;
 373                 buffer_chunk->va = lima_vm_get_va(task->vm, bo);
 374
 375                 if (bo->heap_size) {
 376                         buffer_chunk->size = bo->heap_size;
 377
 378                         data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,
 379                                     VM_MAP, pgprot_writecombine(PAGE_KERNEL));
 380                         if (!data) {
 381                                 kvfree(et);
 382                                 goto out;
 383                         }
 384
 385                         memcpy(buffer_chunk + 1, data, buffer_chunk->size);
 386
 387                         vunmap(data);
 388                 } else {
 389                         buffer_chunk->size = lima_bo_size(bo);
 390
 391                         data = drm_gem_shmem_vmap(&bo->base.base);
 392                         if (IS_ERR_OR_NULL(data)) {
 393                                 kvfree(et);
 394                                 goto out;
 395                         }
 396
 397                         memcpy(buffer_chunk + 1, data, buffer_chunk->size);
 398
 399                         drm_gem_shmem_vunmap(&bo->base.base, data);
 400                 }
 401
 402                 buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;
 403                 dt->num_chunks++;
 404         }
 405
 406         list_add(&et->list, &dev->error_task_list);
 407         dev->dump.size += et->size;
 408         dev->dump.num_tasks++;
 409
 410         dev_info(dev->dev, "save error task state success\n");
 411
 412 out:
 413         mutex_unlock(&dev->error_task_list_lock);
 414 }
 415
 416 static void lima_sched_timedout_job(struct drm_sched_job *job)
 417 {
 418         struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
 419         struct lima_sched_task *task = to_lima_task(job);
 420         struct lima_device *ldev = pipe->ldev;
 421
 422         if (!pipe->error)
 423                 DRM_ERROR("lima job timeout\n");
 424
 425         drm_sched_stop(&pipe->base, &task->base);
 426
 427         drm_sched_increase_karma(&task->base);
 428
 429         lima_sched_build_error_task_list(task);
 430
 431         pipe->task_error(pipe);
 432
 433         if (pipe->bcast_mmu)
 434                 lima_mmu_page_fault_resume(pipe->bcast_mmu);
 435         else {
 436                 int i;
 437
 438                 for (i = 0; i < pipe->num_mmu; i++)
 439                         lima_mmu_page_fault_resume(pipe->mmu[i]);
 440         }
 441
 442         lima_vm_put(pipe->current_vm);
 443         pipe->current_vm = NULL;
 444         pipe->current_task = NULL;
 445
 446         lima_pm_idle(ldev);
 447
 448         drm_sched_resubmit_jobs(&pipe->base);
 449         drm_sched_start(&pipe->base, true);
 450 }
 451
 452 static void lima_sched_free_job(struct drm_sched_job *job)
 453 {
 454         struct lima_sched_task *task = to_lima_task(job);
 455         struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
 456         struct lima_vm *vm = task->vm;
 457         struct lima_bo **bos = task->bos;
 458         int i;
 459
 460         dma_fence_put(task->fence);
 461
 462         for (i = 0; i < task->num_bos; i++)
 463                 lima_vm_bo_del(vm, bos[i]);
 464
 465         lima_sched_task_fini(task);
 466         kmem_cache_free(pipe->task_slab, task);
 467 }
 468
 469 static const struct drm_sched_backend_ops lima_sched_ops = {
 470         .dependency = lima_sched_dependency,
 471         .run_job = lima_sched_run_job,
 472         .timedout_job = lima_sched_timedout_job,
 473         .free_job = lima_sched_free_job,
 474 };
 475
 476 static void lima_sched_recover_work(struct work_struct *work)
 477 {
 478         struct lima_sched_pipe *pipe =
 479                 container_of(work, struct lima_sched_pipe, recover_work);
 480         int i;
 481
 482         for (i = 0; i < pipe->num_l2_cache; i++)
 483                 lima_l2_cache_flush(pipe->l2_cache[i]);
 484
 485         if (pipe->bcast_mmu) {
 486                 lima_mmu_flush_tlb(pipe->bcast_mmu);
 487         } else {
 488                 for (i = 0; i < pipe->num_mmu; i++)
 489                         lima_mmu_flush_tlb(pipe->mmu[i]);
 490         }
 491
 492         if (pipe->task_recover(pipe))
 493                 drm_sched_fault(&pipe->base);
 494 }
 495
 496 int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
 497 {
 498         unsigned int timeout = lima_sched_timeout_ms > 0 ?
 499                                lima_sched_timeout_ms : 500;
 500
 501         pipe->fence_context = dma_fence_context_alloc(1);
 502         spin_lock_init(&pipe->fence_lock);
 503
 504         INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
 505
 506         return drm_sched_init(&pipe->base, &lima_sched_ops, 1,
 507                               lima_job_hang_limit, msecs_to_jiffies(timeout),
 508                               name);
 509 }
 510
 511 void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
 512 {
 513         drm_sched_fini(&pipe->base);
 514 }
 515
 516 void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
 517 {
 518         struct lima_sched_task *task = pipe->current_task;
 519         struct lima_device *ldev = pipe->ldev;
 520
 521         if (pipe->error) {
 522                 if (task && task->recoverable)
 523                         schedule_work(&pipe->recover_work);
 524                 else
 525                         drm_sched_fault(&pipe->base);
 526         } else {
 527                 pipe->task_fini(pipe);
 528                 dma_fence_signal(task->fence);
 529
 530                 lima_pm_idle(ldev);
 531         }
 532 }