drivers/gpu/drm/lima/lima_sched.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
   3
   4 #include <linux/dma-buf-map.h>
   5 #include <linux/kthread.h>
   6 #include <linux/slab.h>
   7 #include <linux/vmalloc.h>
   8 #include <linux/pm_runtime.h>
   9
  10 #include "lima_devfreq.h"
  11 #include "lima_drv.h"
  12 #include "lima_sched.h"
  13 #include "lima_vm.h"
  14 #include "lima_mmu.h"
  15 #include "lima_l2_cache.h"
  16 #include "lima_gem.h"
  17 #include "lima_trace.h"
  18
  19 struct lima_fence {
  20         struct dma_fence base;
  21         struct lima_sched_pipe *pipe;
  22 };
  23
  24 static struct kmem_cache *lima_fence_slab;
  25 static int lima_fence_slab_refcnt;
  26
  27 int lima_sched_slab_init(void)
  28 {
  29         if (!lima_fence_slab) {
  30                 lima_fence_slab = kmem_cache_create(
  31                         "lima_fence", sizeof(struct lima_fence), 0,
  32                         SLAB_HWCACHE_ALIGN, NULL);
  33                 if (!lima_fence_slab)
  34                         return -ENOMEM;
  35         }
  36
  37         lima_fence_slab_refcnt++;
  38         return 0;
  39 }
  40
  41 void lima_sched_slab_fini(void)
  42 {
  43         if (!--lima_fence_slab_refcnt) {
  44                 kmem_cache_destroy(lima_fence_slab);
  45                 lima_fence_slab = NULL;
  46         }
  47 }
  48
  49 static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
  50 {
  51         return container_of(fence, struct lima_fence, base);
  52 }
  53
  54 static const char *lima_fence_get_driver_name(struct dma_fence *fence)
  55 {
  56         return "lima";
  57 }
  58
  59 static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
  60 {
  61         struct lima_fence *f = to_lima_fence(fence);
  62
  63         return f->pipe->base.name;
  64 }
  65
  66 static void lima_fence_release_rcu(struct rcu_head *rcu)
  67 {
  68         struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
  69         struct lima_fence *fence = to_lima_fence(f);
  70
  71         kmem_cache_free(lima_fence_slab, fence);
  72 }
  73
  74 static void lima_fence_release(struct dma_fence *fence)
  75 {
  76         struct lima_fence *f = to_lima_fence(fence);
  77
  78         call_rcu(&f->base.rcu, lima_fence_release_rcu);
  79 }
  80
  81 static const struct dma_fence_ops lima_fence_ops = {
  82         .get_driver_name = lima_fence_get_driver_name,
  83         .get_timeline_name = lima_fence_get_timeline_name,
  84         .release = lima_fence_release,
  85 };
  86
  87 static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
  88 {
  89         struct lima_fence *fence;
  90
  91         fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
  92         if (!fence)
  93                 return NULL;
  94
  95         fence->pipe = pipe;
  96         dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
  97                        pipe->fence_context, ++pipe->fence_seqno);
  98
  99         return fence;
 100 }
 101
 102 static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
 103 {
 104         return container_of(job, struct lima_sched_task, base);
 105 }
 106
 107 static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
 108 {
 109         return container_of(sched, struct lima_sched_pipe, base);
 110 }
 111
 112 int lima_sched_task_init(struct lima_sched_task *task,
 113                          struct lima_sched_context *context,
 114                          struct lima_bo **bos, int num_bos,
 115                          struct lima_vm *vm)
 116 {
 117         int err, i;
 118
 119         task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
 120         if (!task->bos)
 121                 return -ENOMEM;
 122
 123         for (i = 0; i < num_bos; i++)
 124                 drm_gem_object_get(&bos[i]->base.base);
 125
 126         err = drm_sched_job_init(&task->base, &context->base, vm);
 127         if (err) {
 128                 kfree(task->bos);
 129                 return err;
 130         }
 131
 132         drm_sched_job_arm(&task->base);
 133
 134         task->num_bos = num_bos;
 135         task->vm = lima_vm_get(vm);
 136
 137         return 0;
 138 }
 139
 140 void lima_sched_task_fini(struct lima_sched_task *task)
 141 {
 142         int i;
 143
 144         drm_sched_job_cleanup(&task->base);
 145
 146         if (task->bos) {
 147                 for (i = 0; i < task->num_bos; i++)
 148                         drm_gem_object_put(&task->bos[i]->base.base);
 149                 kfree(task->bos);
 150         }
 151
 152         lima_vm_put(task->vm);
 153 }
 154
 155 int lima_sched_context_init(struct lima_sched_pipe *pipe,
 156                             struct lima_sched_context *context,
 157                             atomic_t *guilty)
 158 {
 159         struct drm_gpu_scheduler *sched = &pipe->base;
 160
 161         return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL,
 162                                      &sched, 1, guilty);
 163 }
 164
 165 void lima_sched_context_fini(struct lima_sched_pipe *pipe,
 166                              struct lima_sched_context *context)
 167 {
 168         drm_sched_entity_fini(&context->base);
 169 }
 170
 171 struct dma_fence *lima_sched_context_queue_task(struct lima_sched_task *task)
 172 {
 173         struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
 174
 175         trace_lima_task_submit(task);
 176         drm_sched_entity_push_job(&task->base);
 177         return fence;
 178 }
 179
 180 static int lima_pm_busy(struct lima_device *ldev)
 181 {
 182         int ret;
 183
 184         /* resume GPU if it has been suspended by runtime PM */
 185         ret = pm_runtime_resume_and_get(ldev->dev);
 186         if (ret < 0)
 187                 return ret;
 188
 189         lima_devfreq_record_busy(&ldev->devfreq);
 190         return 0;
 191 }
 192
 193 static void lima_pm_idle(struct lima_device *ldev)
 194 {
 195         lima_devfreq_record_idle(&ldev->devfreq);
 196
 197         /* GPU can do auto runtime suspend */
 198         pm_runtime_mark_last_busy(ldev->dev);
 199         pm_runtime_put_autosuspend(ldev->dev);
 200 }
 201
 202 static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
 203 {
 204         struct lima_sched_task *task = to_lima_task(job);
 205         struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
 206         struct lima_device *ldev = pipe->ldev;
 207         struct lima_fence *fence;
 208         int i, err;
 209
 210         /* after GPU reset */
 211         if (job->s_fence->finished.error < 0)
 212                 return NULL;
 213
 214         fence = lima_fence_create(pipe);
 215         if (!fence)
 216                 return NULL;
 217
 218         err = lima_pm_busy(ldev);
 219         if (err < 0) {
 220                 dma_fence_put(&fence->base);
 221                 return NULL;
 222         }
 223
 224         task->fence = &fence->base;
 225
 226         /* for caller usage of the fence, otherwise irq handler
 227          * may consume the fence before caller use it
 228          */
 229         dma_fence_get(task->fence);
 230
 231         pipe->current_task = task;
 232
 233         /* this is needed for MMU to work correctly, otherwise GP/PP
 234          * will hang or page fault for unknown reason after running for
 235          * a while.
 236          *
 237          * Need to investigate:
 238          * 1. is it related to TLB
 239          * 2. how much performance will be affected by L2 cache flush
 240          * 3. can we reduce the calling of this function because all
 241          *    GP/PP use the same L2 cache on mali400
 242          *
 243          * TODO:
 244          * 1. move this to task fini to save some wait time?
 245          * 2. when GP/PP use different l2 cache, need PP wait GP l2
 246          *    cache flush?
 247          */
 248         for (i = 0; i < pipe->num_l2_cache; i++)
 249                 lima_l2_cache_flush(pipe->l2_cache[i]);
 250
 251         lima_vm_put(pipe->current_vm);
 252         pipe->current_vm = lima_vm_get(task->vm);
 253
 254         if (pipe->bcast_mmu)
 255                 lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm);
 256         else {
 257                 for (i = 0; i < pipe->num_mmu; i++)
 258                         lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm);
 259         }
 260
 261         trace_lima_task_run(task);
 262
 263         pipe->error = false;
 264         pipe->task_run(pipe, task);
 265
 266         return task->fence;
 267 }
 268
 269 static void lima_sched_build_error_task_list(struct lima_sched_task *task)
 270 {
 271         struct lima_sched_error_task *et;
 272         struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched);
 273         struct lima_ip *ip = pipe->processor[0];
 274         int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp;
 275         struct lima_device *dev = ip->dev;
 276         struct lima_sched_context *sched_ctx =
 277                 container_of(task->base.entity,
 278                              struct lima_sched_context, base);
 279         struct lima_ctx *ctx =
 280                 container_of(sched_ctx, struct lima_ctx, context[pipe_id]);
 281         struct lima_dump_task *dt;
 282         struct lima_dump_chunk *chunk;
 283         struct lima_dump_chunk_pid *pid_chunk;
 284         struct lima_dump_chunk_buffer *buffer_chunk;
 285         u32 size, task_size, mem_size;
 286         int i;
 287         struct dma_buf_map map;
 288         int ret;
 289
 290         mutex_lock(&dev->error_task_list_lock);
 291
 292         if (dev->dump.num_tasks >= lima_max_error_tasks) {
 293                 dev_info(dev->dev, "fail to save task state from %s pid %d: "
 294                          "error task list is full\n", ctx->pname, ctx->pid);
 295                 goto out;
 296         }
 297
 298         /* frame chunk */
 299         size = sizeof(struct lima_dump_chunk) + pipe->frame_size;
 300         /* process name chunk */
 301         size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname);
 302         /* pid chunk */
 303         size += sizeof(struct lima_dump_chunk);
 304         /* buffer chunks */
 305         for (i = 0; i < task->num_bos; i++) {
 306                 struct lima_bo *bo = task->bos[i];
 307
 308                 size += sizeof(struct lima_dump_chunk);
 309                 size += bo->heap_size ? bo->heap_size : lima_bo_size(bo);
 310         }
 311
 312         task_size = size + sizeof(struct lima_dump_task);
 313         mem_size = task_size + sizeof(*et);
 314         et = kvmalloc(mem_size, GFP_KERNEL);
 315         if (!et) {
 316                 dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n",
 317                         mem_size);
 318                 goto out;
 319         }
 320
 321         et->data = et + 1;
 322         et->size = task_size;
 323
 324         dt = et->data;
 325         memset(dt, 0, sizeof(*dt));
 326         dt->id = pipe_id;
 327         dt->size = size;
 328
 329         chunk = (struct lima_dump_chunk *)(dt + 1);
 330         memset(chunk, 0, sizeof(*chunk));
 331         chunk->id = LIMA_DUMP_CHUNK_FRAME;
 332         chunk->size = pipe->frame_size;
 333         memcpy(chunk + 1, task->frame, pipe->frame_size);
 334         dt->num_chunks++;
 335
 336         chunk = (void *)(chunk + 1) + chunk->size;
 337         memset(chunk, 0, sizeof(*chunk));
 338         chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME;
 339         chunk->size = sizeof(ctx->pname);
 340         memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname));
 341         dt->num_chunks++;
 342
 343         pid_chunk = (void *)(chunk + 1) + chunk->size;
 344         memset(pid_chunk, 0, sizeof(*pid_chunk));
 345         pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID;
 346         pid_chunk->pid = ctx->pid;
 347         dt->num_chunks++;
 348
 349         buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size;
 350         for (i = 0; i < task->num_bos; i++) {
 351                 struct lima_bo *bo = task->bos[i];
 352                 void *data;
 353
 354                 memset(buffer_chunk, 0, sizeof(*buffer_chunk));
 355                 buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER;
 356                 buffer_chunk->va = lima_vm_get_va(task->vm, bo);
 357
 358                 if (bo->heap_size) {
 359                         buffer_chunk->size = bo->heap_size;
 360
 361                         data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT,
 362                                     VM_MAP, pgprot_writecombine(PAGE_KERNEL));
 363                         if (!data) {
 364                                 kvfree(et);
 365                                 goto out;
 366                         }
 367
 368                         memcpy(buffer_chunk + 1, data, buffer_chunk->size);
 369
 370                         vunmap(data);
 371                 } else {
 372                         buffer_chunk->size = lima_bo_size(bo);
 373
 374                         ret = drm_gem_shmem_vmap(&bo->base, &map);
 375                         if (ret) {
 376                                 kvfree(et);
 377                                 goto out;
 378                         }
 379
 380                         memcpy(buffer_chunk + 1, map.vaddr, buffer_chunk->size);
 381
 382                         drm_gem_shmem_vunmap(&bo->base, &map);
 383                 }
 384
 385                 buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size;
 386                 dt->num_chunks++;
 387         }
 388
 389         list_add(&et->list, &dev->error_task_list);
 390         dev->dump.size += et->size;
 391         dev->dump.num_tasks++;
 392
 393         dev_info(dev->dev, "save error task state success\n");
 394
 395 out:
 396         mutex_unlock(&dev->error_task_list_lock);
 397 }
 398
 399 static enum drm_gpu_sched_stat lima_sched_timedout_job(struct drm_sched_job *job)
 400 {
 401         struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
 402         struct lima_sched_task *task = to_lima_task(job);
 403         struct lima_device *ldev = pipe->ldev;
 404
 405         if (!pipe->error)
 406                 DRM_ERROR("lima job timeout\n");
 407
 408         drm_sched_stop(&pipe->base, &task->base);
 409
 410         drm_sched_increase_karma(&task->base);
 411
 412         if (lima_max_error_tasks)
 413                 lima_sched_build_error_task_list(task);
 414
 415         pipe->task_error(pipe);
 416
 417         if (pipe->bcast_mmu)
 418                 lima_mmu_page_fault_resume(pipe->bcast_mmu);
 419         else {
 420                 int i;
 421
 422                 for (i = 0; i < pipe->num_mmu; i++)
 423                         lima_mmu_page_fault_resume(pipe->mmu[i]);
 424         }
 425
 426         lima_vm_put(pipe->current_vm);
 427         pipe->current_vm = NULL;
 428         pipe->current_task = NULL;
 429
 430         lima_pm_idle(ldev);
 431
 432         drm_sched_resubmit_jobs(&pipe->base);
 433         drm_sched_start(&pipe->base, true);
 434
 435         return DRM_GPU_SCHED_STAT_NOMINAL;
 436 }
 437
 438 static void lima_sched_free_job(struct drm_sched_job *job)
 439 {
 440         struct lima_sched_task *task = to_lima_task(job);
 441         struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
 442         struct lima_vm *vm = task->vm;
 443         struct lima_bo **bos = task->bos;
 444         int i;
 445
 446         dma_fence_put(task->fence);
 447
 448         for (i = 0; i < task->num_bos; i++)
 449                 lima_vm_bo_del(vm, bos[i]);
 450
 451         lima_sched_task_fini(task);
 452         kmem_cache_free(pipe->task_slab, task);
 453 }
 454
 455 static const struct drm_sched_backend_ops lima_sched_ops = {
 456         .run_job = lima_sched_run_job,
 457         .timedout_job = lima_sched_timedout_job,
 458         .free_job = lima_sched_free_job,
 459 };
 460
 461 static void lima_sched_recover_work(struct work_struct *work)
 462 {
 463         struct lima_sched_pipe *pipe =
 464                 container_of(work, struct lima_sched_pipe, recover_work);
 465         int i;
 466
 467         for (i = 0; i < pipe->num_l2_cache; i++)
 468                 lima_l2_cache_flush(pipe->l2_cache[i]);
 469
 470         if (pipe->bcast_mmu) {
 471                 lima_mmu_flush_tlb(pipe->bcast_mmu);
 472         } else {
 473                 for (i = 0; i < pipe->num_mmu; i++)
 474                         lima_mmu_flush_tlb(pipe->mmu[i]);
 475         }
 476
 477         if (pipe->task_recover(pipe))
 478                 drm_sched_fault(&pipe->base);
 479 }
 480
 481 int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
 482 {
 483         unsigned int timeout = lima_sched_timeout_ms > 0 ?
 484                                lima_sched_timeout_ms : 500;
 485
 486         pipe->fence_context = dma_fence_context_alloc(1);
 487         spin_lock_init(&pipe->fence_lock);
 488
 489         INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
 490
 491         return drm_sched_init(&pipe->base, &lima_sched_ops, 1,
 492                               lima_job_hang_limit,
 493                               msecs_to_jiffies(timeout), NULL,
 494                               NULL, name, pipe->ldev->dev);
 495 }
 496
 497 void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
 498 {
 499         drm_sched_fini(&pipe->base);
 500 }
 501
 502 void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
 503 {
 504         struct lima_sched_task *task = pipe->current_task;
 505         struct lima_device *ldev = pipe->ldev;
 506
 507         if (pipe->error) {
 508                 if (task && task->recoverable)
 509                         schedule_work(&pipe->recover_work);
 510                 else
 511                         drm_sched_fault(&pipe->base);
 512         } else {
 513                 pipe->task_fini(pipe);
 514                 dma_fence_signal(task->fence);
 515
 516                 lima_pm_idle(ldev);
 517         }
 518 }