drivers/gpu/drm/i915/gvt/scheduler.c

   1 /*
   2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Zhi Wang <zhi.a.wang@intel.com>
  25  *
  26  * Contributors:
  27  *    Ping Gao <ping.a.gao@intel.com>
  28  *    Tina Zhang <tina.zhang@intel.com>
  29  *    Chanbin Du <changbin.du@intel.com>
  30  *    Min He <min.he@intel.com>
  31  *    Bing Niu <bing.niu@intel.com>
  32  *    Zhenyu Wang <zhenyuw@linux.intel.com>
  33  *
  34  */
  35
  36 #include <linux/kthread.h>
  37
  38 #include "gem/i915_gem_pm.h"
  39 #include "gt/intel_context.h"
  40 #include "gt/intel_ring.h"
  41
  42 #include "i915_drv.h"
  43 #include "i915_gem_gtt.h"
  44 #include "gvt.h"
  45
  46 #define RING_CTX_OFF(x) \
  47         offsetof(struct execlist_ring_context, x)
  48
  49 static void set_context_pdp_root_pointer(
  50                 struct execlist_ring_context *ring_context,
  51                 u32 pdp[8])
  52 {
  53         int i;
  54
  55         for (i = 0; i < 8; i++)
  56                 ring_context->pdps[i].val = pdp[7 - i];
  57 }
  58
  59 static void update_shadow_pdps(struct intel_vgpu_workload *workload)
  60 {
  61         struct execlist_ring_context *shadow_ring_context;
  62         struct intel_context *ctx = workload->req->context;
  63
  64         if (WARN_ON(!workload->shadow_mm))
  65                 return;
  66
  67         if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount)))
  68                 return;
  69
  70         shadow_ring_context = (struct execlist_ring_context *)ctx->lrc_reg_state;
  71         set_context_pdp_root_pointer(shadow_ring_context,
  72                         (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps);
  73 }
  74
  75 /*
  76  * when populating shadow ctx from guest, we should not overrride oa related
  77  * registers, so that they will not be overlapped by guest oa configs. Thus
  78  * made it possible to capture oa data from host for both host and guests.
  79  */
  80 static void sr_oa_regs(struct intel_vgpu_workload *workload,
  81                 u32 *reg_state, bool save)
  82 {
  83         struct drm_i915_private *dev_priv = workload->vgpu->gvt->gt->i915;
  84         u32 ctx_oactxctrl = dev_priv->perf.ctx_oactxctrl_offset;
  85         u32 ctx_flexeu0 = dev_priv->perf.ctx_flexeu0_offset;
  86         int i = 0;
  87         u32 flex_mmio[] = {
  88                 i915_mmio_reg_offset(EU_PERF_CNTL0),
  89                 i915_mmio_reg_offset(EU_PERF_CNTL1),
  90                 i915_mmio_reg_offset(EU_PERF_CNTL2),
  91                 i915_mmio_reg_offset(EU_PERF_CNTL3),
  92                 i915_mmio_reg_offset(EU_PERF_CNTL4),
  93                 i915_mmio_reg_offset(EU_PERF_CNTL5),
  94                 i915_mmio_reg_offset(EU_PERF_CNTL6),
  95         };
  96
  97         if (workload->engine->id != RCS0)
  98                 return;
  99
 100         if (save) {
 101                 workload->oactxctrl = reg_state[ctx_oactxctrl + 1];
 102
 103                 for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
 104                         u32 state_offset = ctx_flexeu0 + i * 2;
 105
 106                         workload->flex_mmio[i] = reg_state[state_offset + 1];
 107                 }
 108         } else {
 109                 reg_state[ctx_oactxctrl] =
 110                         i915_mmio_reg_offset(GEN8_OACTXCONTROL);
 111                 reg_state[ctx_oactxctrl + 1] = workload->oactxctrl;
 112
 113                 for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
 114                         u32 state_offset = ctx_flexeu0 + i * 2;
 115                         u32 mmio = flex_mmio[i];
 116
 117                         reg_state[state_offset] = mmio;
 118                         reg_state[state_offset + 1] = workload->flex_mmio[i];
 119                 }
 120         }
 121 }
 122
 123 static int populate_shadow_context(struct intel_vgpu_workload *workload)
 124 {
 125         struct intel_vgpu *vgpu = workload->vgpu;
 126         struct intel_gvt *gvt = vgpu->gvt;
 127         struct intel_context *ctx = workload->req->context;
 128         struct execlist_ring_context *shadow_ring_context;
 129         void *dst;
 130         void *context_base;
 131         unsigned long context_gpa, context_page_num;
 132         unsigned long gpa_base; /* first gpa of consecutive GPAs */
 133         unsigned long gpa_size; /* size of consecutive GPAs */
 134         struct intel_vgpu_submission *s = &vgpu->submission;
 135         int i;
 136         bool skip = false;
 137         int ring_id = workload->engine->id;
 138
 139         GEM_BUG_ON(!intel_context_is_pinned(ctx));
 140
 141         context_base = (void *) ctx->lrc_reg_state -
 142                                 (LRC_STATE_PN << I915_GTT_PAGE_SHIFT);
 143
 144         shadow_ring_context = (void *) ctx->lrc_reg_state;
 145
 146         sr_oa_regs(workload, (u32 *)shadow_ring_context, true);
 147 #define COPY_REG(name) \
 148         intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
 149                 + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
 150 #define COPY_REG_MASKED(name) {\
 151                 intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
 152                                               + RING_CTX_OFF(name.val),\
 153                                               &shadow_ring_context->name.val, 4);\
 154                 shadow_ring_context->name.val |= 0xffff << 16;\
 155         }
 156
 157         COPY_REG_MASKED(ctx_ctrl);
 158         COPY_REG(ctx_timestamp);
 159
 160         if (workload->engine->id == RCS0) {
 161                 COPY_REG(bb_per_ctx_ptr);
 162                 COPY_REG(rcs_indirect_ctx);
 163                 COPY_REG(rcs_indirect_ctx_offset);
 164         }
 165 #undef COPY_REG
 166 #undef COPY_REG_MASKED
 167
 168         intel_gvt_hypervisor_read_gpa(vgpu,
 169                         workload->ring_context_gpa +
 170                         sizeof(*shadow_ring_context),
 171                         (void *)shadow_ring_context +
 172                         sizeof(*shadow_ring_context),
 173                         I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 174
 175         sr_oa_regs(workload, (u32 *)shadow_ring_context, false);
 176
 177         gvt_dbg_sched("ring %s workload lrca %x, ctx_id %x, ctx gpa %llx",
 178                         workload->engine->name, workload->ctx_desc.lrca,
 179                         workload->ctx_desc.context_id,
 180                         workload->ring_context_gpa);
 181
 182         /* only need to ensure this context is not pinned/unpinned during the
 183          * period from last submission to this this submission.
 184          * Upon reaching this function, the currently submitted context is not
 185          * supposed to get unpinned. If a misbehaving guest driver ever does
 186          * this, it would corrupt itself.
 187          */
 188         if (s->last_ctx[ring_id].valid &&
 189                         (s->last_ctx[ring_id].lrca ==
 190                                 workload->ctx_desc.lrca) &&
 191                         (s->last_ctx[ring_id].ring_context_gpa ==
 192                                 workload->ring_context_gpa))
 193                 skip = true;
 194
 195         s->last_ctx[ring_id].lrca = workload->ctx_desc.lrca;
 196         s->last_ctx[ring_id].ring_context_gpa = workload->ring_context_gpa;
 197
 198         if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val) || skip)
 199                 return 0;
 200
 201         s->last_ctx[ring_id].valid = false;
 202         context_page_num = workload->engine->context_size;
 203         context_page_num = context_page_num >> PAGE_SHIFT;
 204
 205         if (IS_BROADWELL(gvt->gt->i915) && workload->engine->id == RCS0)
 206                 context_page_num = 19;
 207
 208         /* find consecutive GPAs from gma until the first inconsecutive GPA.
 209          * read from the continuous GPAs into dst virtual address
 210          */
 211         gpa_size = 0;
 212         for (i = 2; i < context_page_num; i++) {
 213                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 214                                 (u32)((workload->ctx_desc.lrca + i) <<
 215                                 I915_GTT_PAGE_SHIFT));
 216                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 217                         gvt_vgpu_err("Invalid guest context descriptor\n");
 218                         return -EFAULT;
 219                 }
 220
 221                 if (gpa_size == 0) {
 222                         gpa_base = context_gpa;
 223                         dst = context_base + (i << I915_GTT_PAGE_SHIFT);
 224                 } else if (context_gpa != gpa_base + gpa_size)
 225                         goto read;
 226
 227                 gpa_size += I915_GTT_PAGE_SIZE;
 228
 229                 if (i == context_page_num - 1)
 230                         goto read;
 231
 232                 continue;
 233
 234 read:
 235                 intel_gvt_hypervisor_read_gpa(vgpu, gpa_base, dst, gpa_size);
 236                 gpa_base = context_gpa;
 237                 gpa_size = I915_GTT_PAGE_SIZE;
 238                 dst = context_base + (i << I915_GTT_PAGE_SHIFT);
 239         }
 240         s->last_ctx[ring_id].valid = true;
 241         return 0;
 242 }
 243
 244 static inline bool is_gvt_request(struct i915_request *rq)
 245 {
 246         return intel_context_force_single_submission(rq->context);
 247 }
 248
 249 static void save_ring_hw_state(struct intel_vgpu *vgpu,
 250                                const struct intel_engine_cs *engine)
 251 {
 252         struct intel_uncore *uncore = engine->uncore;
 253         i915_reg_t reg;
 254
 255         reg = RING_INSTDONE(engine->mmio_base);
 256         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) =
 257                 intel_uncore_read(uncore, reg);
 258
 259         reg = RING_ACTHD(engine->mmio_base);
 260         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) =
 261                 intel_uncore_read(uncore, reg);
 262
 263         reg = RING_ACTHD_UDW(engine->mmio_base);
 264         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) =
 265                 intel_uncore_read(uncore, reg);
 266 }
 267
 268 static int shadow_context_status_change(struct notifier_block *nb,
 269                 unsigned long action, void *data)
 270 {
 271         struct i915_request *rq = data;
 272         struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
 273                                 shadow_ctx_notifier_block[rq->engine->id]);
 274         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 275         enum intel_engine_id ring_id = rq->engine->id;
 276         struct intel_vgpu_workload *workload;
 277         unsigned long flags;
 278
 279         if (!is_gvt_request(rq)) {
 280                 spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
 281                 if (action == INTEL_CONTEXT_SCHEDULE_IN &&
 282                     scheduler->engine_owner[ring_id]) {
 283                         /* Switch ring from vGPU to host. */
 284                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
 285                                               NULL, rq->engine);
 286                         scheduler->engine_owner[ring_id] = NULL;
 287                 }
 288                 spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
 289
 290                 return NOTIFY_OK;
 291         }
 292
 293         workload = scheduler->current_workload[ring_id];
 294         if (unlikely(!workload))
 295                 return NOTIFY_OK;
 296
 297         switch (action) {
 298         case INTEL_CONTEXT_SCHEDULE_IN:
 299                 spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
 300                 if (workload->vgpu != scheduler->engine_owner[ring_id]) {
 301                         /* Switch ring from host to vGPU or vGPU to vGPU. */
 302                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
 303                                               workload->vgpu, rq->engine);
 304                         scheduler->engine_owner[ring_id] = workload->vgpu;
 305                 } else
 306                         gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n",
 307                                       ring_id, workload->vgpu->id);
 308                 spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
 309                 atomic_set(&workload->shadow_ctx_active, 1);
 310                 break;
 311         case INTEL_CONTEXT_SCHEDULE_OUT:
 312                 save_ring_hw_state(workload->vgpu, rq->engine);
 313                 atomic_set(&workload->shadow_ctx_active, 0);
 314                 break;
 315         case INTEL_CONTEXT_SCHEDULE_PREEMPTED:
 316                 save_ring_hw_state(workload->vgpu, rq->engine);
 317                 break;
 318         default:
 319                 WARN_ON(1);
 320                 return NOTIFY_OK;
 321         }
 322         wake_up(&workload->shadow_ctx_status_wq);
 323         return NOTIFY_OK;
 324 }
 325
 326 static void
 327 shadow_context_descriptor_update(struct intel_context *ce,
 328                                  struct intel_vgpu_workload *workload)
 329 {
 330         u64 desc = ce->lrc.desc;
 331
 332         /*
 333          * Update bits 0-11 of the context descriptor which includes flags
 334          * like GEN8_CTX_* cached in desc_template
 335          */
 336         desc &= ~(0x3ull << GEN8_CTX_ADDRESSING_MODE_SHIFT);
 337         desc |= (u64)workload->ctx_desc.addressing_mode <<
 338                 GEN8_CTX_ADDRESSING_MODE_SHIFT;
 339
 340         ce->lrc.desc = desc;
 341 }
 342
 343 static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
 344 {
 345         struct intel_vgpu *vgpu = workload->vgpu;
 346         struct i915_request *req = workload->req;
 347         void *shadow_ring_buffer_va;
 348         u32 *cs;
 349         int err;
 350
 351         if (IS_GEN(req->engine->i915, 9) && is_inhibit_context(req->context))
 352                 intel_vgpu_restore_inhibit_context(vgpu, req);
 353
 354         /*
 355          * To track whether a request has started on HW, we can emit a
 356          * breadcrumb at the beginning of the request and check its
 357          * timeline's HWSP to see if the breadcrumb has advanced past the
 358          * start of this request. Actually, the request must have the
 359          * init_breadcrumb if its timeline set has_init_bread_crumb, or the
 360          * scheduler might get a wrong state of it during reset. Since the
 361          * requests from gvt always set the has_init_breadcrumb flag, here
 362          * need to do the emit_init_breadcrumb for all the requests.
 363          */
 364         if (req->engine->emit_init_breadcrumb) {
 365                 err = req->engine->emit_init_breadcrumb(req);
 366                 if (err) {
 367                         gvt_vgpu_err("fail to emit init breadcrumb\n");
 368                         return err;
 369                 }
 370         }
 371
 372         /* allocate shadow ring buffer */
 373         cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
 374         if (IS_ERR(cs)) {
 375                 gvt_vgpu_err("fail to alloc size =%ld shadow  ring buffer\n",
 376                         workload->rb_len);
 377                 return PTR_ERR(cs);
 378         }
 379
 380         shadow_ring_buffer_va = workload->shadow_ring_buffer_va;
 381
 382         /* get shadow ring buffer va */
 383         workload->shadow_ring_buffer_va = cs;
 384
 385         memcpy(cs, shadow_ring_buffer_va,
 386                         workload->rb_len);
 387
 388         cs += workload->rb_len / sizeof(u32);
 389         intel_ring_advance(workload->req, cs);
 390
 391         return 0;
 392 }
 393
 394 static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 395 {
 396         if (!wa_ctx->indirect_ctx.obj)
 397                 return;
 398
 399         i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
 400         i915_gem_object_put(wa_ctx->indirect_ctx.obj);
 401
 402         wa_ctx->indirect_ctx.obj = NULL;
 403         wa_ctx->indirect_ctx.shadow_va = NULL;
 404 }
 405
 406 static void set_dma_address(struct i915_page_directory *pd, dma_addr_t addr)
 407 {
 408         struct scatterlist *sg = pd->pt.base->mm.pages->sgl;
 409
 410         /* This is not a good idea */
 411         sg->dma_address = addr;
 412 }
 413
 414 static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload,
 415                                           struct intel_context *ce)
 416 {
 417         struct intel_vgpu_mm *mm = workload->shadow_mm;
 418         struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(ce->vm);
 419         int i = 0;
 420
 421         if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
 422                 set_dma_address(ppgtt->pd, mm->ppgtt_mm.shadow_pdps[0]);
 423         } else {
 424                 for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) {
 425                         struct i915_page_directory * const pd =
 426                                 i915_pd_entry(ppgtt->pd, i);
 427                         /* skip now as current i915 ppgtt alloc won't allocate
 428                            top level pdp for non 4-level table, won't impact
 429                            shadow ppgtt. */
 430                         if (!pd)
 431                                 break;
 432
 433                         set_dma_address(pd, mm->ppgtt_mm.shadow_pdps[i]);
 434                 }
 435         }
 436 }
 437
 438 static int
 439 intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)
 440 {
 441         struct intel_vgpu *vgpu = workload->vgpu;
 442         struct intel_vgpu_submission *s = &vgpu->submission;
 443         struct i915_request *rq;
 444
 445         if (workload->req)
 446                 return 0;
 447
 448         rq = i915_request_create(s->shadow[workload->engine->id]);
 449         if (IS_ERR(rq)) {
 450                 gvt_vgpu_err("fail to allocate gem request\n");
 451                 return PTR_ERR(rq);
 452         }
 453
 454         workload->req = i915_request_get(rq);
 455         return 0;
 456 }
 457
 458 /**
 459  * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
 460  * shadow it as well, include ringbuffer,wa_ctx and ctx.
 461  * @workload: an abstract entity for each execlist submission.
 462  *
 463  * This function is called before the workload submitting to i915, to make
 464  * sure the content of the workload is valid.
 465  */
 466 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 467 {
 468         struct intel_vgpu *vgpu = workload->vgpu;
 469         struct intel_vgpu_submission *s = &vgpu->submission;
 470         int ret;
 471
 472         lockdep_assert_held(&vgpu->vgpu_lock);
 473
 474         if (workload->shadow)
 475                 return 0;
 476
 477         if (!test_and_set_bit(workload->engine->id, s->shadow_ctx_desc_updated))
 478                 shadow_context_descriptor_update(s->shadow[workload->engine->id],
 479                                                  workload);
 480
 481         ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
 482         if (ret)
 483                 return ret;
 484
 485         if (workload->engine->id == RCS0 &&
 486             workload->wa_ctx.indirect_ctx.size) {
 487                 ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
 488                 if (ret)
 489                         goto err_shadow;
 490         }
 491
 492         workload->shadow = true;
 493         return 0;
 494
 495 err_shadow:
 496         release_shadow_wa_ctx(&workload->wa_ctx);
 497         return ret;
 498 }
 499
 500 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);
 501
 502 static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 503 {
 504         struct intel_gvt *gvt = workload->vgpu->gvt;
 505         const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
 506         struct intel_vgpu_shadow_bb *bb;
 507         int ret;
 508
 509         list_for_each_entry(bb, &workload->shadow_bb, list) {
 510                 /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
 511                  * is only updated into ring_scan_buffer, not real ring address
 512                  * allocated in later copy_workload_to_ring_buffer. pls be noted
 513                  * shadow_ring_buffer_va is now pointed to real ring buffer va
 514                  * in copy_workload_to_ring_buffer.
 515                  */
 516
 517                 if (bb->bb_offset)
 518                         bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
 519                                 + bb->bb_offset;
 520
 521                 /*
 522                  * For non-priv bb, scan&shadow is only for
 523                  * debugging purpose, so the content of shadow bb
 524                  * is the same as original bb. Therefore,
 525                  * here, rather than switch to shadow bb's gma
 526                  * address, we directly use original batch buffer's
 527                  * gma address, and send original bb to hardware
 528                  * directly
 529                  */
 530                 if (!bb->ppgtt) {
 531                         bb->vma = i915_gem_object_ggtt_pin(bb->obj,
 532                                                            NULL, 0, 0, 0);
 533                         if (IS_ERR(bb->vma)) {
 534                                 ret = PTR_ERR(bb->vma);
 535                                 goto err;
 536                         }
 537
 538                         /* relocate shadow batch buffer */
 539                         bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
 540                         if (gmadr_bytes == 8)
 541                                 bb->bb_start_cmd_va[2] = 0;
 542
 543                         ret = i915_vma_move_to_active(bb->vma,
 544                                                       workload->req,
 545                                                       0);
 546                         if (ret)
 547                                 goto err;
 548                 }
 549
 550                 /* No one is going to touch shadow bb from now on. */
 551                 i915_gem_object_flush_map(bb->obj);
 552         }
 553         return 0;
 554 err:
 555         release_shadow_batch_buffer(workload);
 556         return ret;
 557 }
 558
 559 static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 560 {
 561         struct intel_vgpu_workload *workload =
 562                 container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx);
 563         struct i915_request *rq = workload->req;
 564         struct execlist_ring_context *shadow_ring_context =
 565                 (struct execlist_ring_context *)rq->context->lrc_reg_state;
 566
 567         shadow_ring_context->bb_per_ctx_ptr.val =
 568                 (shadow_ring_context->bb_per_ctx_ptr.val &
 569                 (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
 570         shadow_ring_context->rcs_indirect_ctx.val =
 571                 (shadow_ring_context->rcs_indirect_ctx.val &
 572                 (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
 573 }
 574
 575 static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 576 {
 577         struct i915_vma *vma;
 578         unsigned char *per_ctx_va =
 579                 (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
 580                 wa_ctx->indirect_ctx.size;
 581
 582         if (wa_ctx->indirect_ctx.size == 0)
 583                 return 0;
 584
 585         vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
 586                                        0, CACHELINE_BYTES, 0);
 587         if (IS_ERR(vma))
 588                 return PTR_ERR(vma);
 589
 590         /* FIXME: we are not tracking our pinned VMA leaving it
 591          * up to the core to fix up the stray pin_count upon
 592          * free.
 593          */
 594
 595         wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
 596
 597         wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
 598         memset(per_ctx_va, 0, CACHELINE_BYTES);
 599
 600         update_wa_ctx_2_shadow_ctx(wa_ctx);
 601         return 0;
 602 }
 603
 604 static void update_vreg_in_ctx(struct intel_vgpu_workload *workload)
 605 {
 606         vgpu_vreg_t(workload->vgpu, RING_START(workload->engine->mmio_base)) =
 607                 workload->rb_start;
 608 }
 609
 610 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 611 {
 612         struct intel_vgpu_shadow_bb *bb, *pos;
 613
 614         if (list_empty(&workload->shadow_bb))
 615                 return;
 616
 617         bb = list_first_entry(&workload->shadow_bb,
 618                         struct intel_vgpu_shadow_bb, list);
 619
 620         list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {
 621                 if (bb->obj) {
 622                         if (bb->va && !IS_ERR(bb->va))
 623                                 i915_gem_object_unpin_map(bb->obj);
 624
 625                         if (bb->vma && !IS_ERR(bb->vma))
 626                                 i915_vma_unpin(bb->vma);
 627
 628                         i915_gem_object_put(bb->obj);
 629                 }
 630                 list_del(&bb->list);
 631                 kfree(bb);
 632         }
 633 }
 634
 635 static int
 636 intel_vgpu_shadow_mm_pin(struct intel_vgpu_workload *workload)
 637 {
 638         struct intel_vgpu *vgpu = workload->vgpu;
 639         struct intel_vgpu_mm *m;
 640         int ret = 0;
 641
 642         ret = intel_vgpu_pin_mm(workload->shadow_mm);
 643         if (ret) {
 644                 gvt_vgpu_err("fail to vgpu pin mm\n");
 645                 return ret;
 646         }
 647
 648         if (workload->shadow_mm->type != INTEL_GVT_MM_PPGTT ||
 649             !workload->shadow_mm->ppgtt_mm.shadowed) {
 650                 gvt_vgpu_err("workload shadow ppgtt isn't ready\n");
 651                 return -EINVAL;
 652         }
 653
 654         if (!list_empty(&workload->lri_shadow_mm)) {
 655                 list_for_each_entry(m, &workload->lri_shadow_mm,
 656                                     ppgtt_mm.link) {
 657                         ret = intel_vgpu_pin_mm(m);
 658                         if (ret) {
 659                                 list_for_each_entry_from_reverse(m,
 660                                                                  &workload->lri_shadow_mm,
 661                                                                  ppgtt_mm.link)
 662                                         intel_vgpu_unpin_mm(m);
 663                                 gvt_vgpu_err("LRI shadow ppgtt fail to pin\n");
 664                                 break;
 665                         }
 666                 }
 667         }
 668
 669         if (ret)
 670                 intel_vgpu_unpin_mm(workload->shadow_mm);
 671
 672         return ret;
 673 }
 674
 675 static void
 676 intel_vgpu_shadow_mm_unpin(struct intel_vgpu_workload *workload)
 677 {
 678         struct intel_vgpu_mm *m;
 679
 680         if (!list_empty(&workload->lri_shadow_mm)) {
 681                 list_for_each_entry(m, &workload->lri_shadow_mm,
 682                                     ppgtt_mm.link)
 683                         intel_vgpu_unpin_mm(m);
 684         }
 685         intel_vgpu_unpin_mm(workload->shadow_mm);
 686 }
 687
 688 static int prepare_workload(struct intel_vgpu_workload *workload)
 689 {
 690         struct intel_vgpu *vgpu = workload->vgpu;
 691         struct intel_vgpu_submission *s = &vgpu->submission;
 692         int ret = 0;
 693
 694         ret = intel_vgpu_shadow_mm_pin(workload);
 695         if (ret) {
 696                 gvt_vgpu_err("fail to pin shadow mm\n");
 697                 return ret;
 698         }
 699
 700         update_shadow_pdps(workload);
 701
 702         set_context_ppgtt_from_shadow(workload, s->shadow[workload->engine->id]);
 703
 704         ret = intel_vgpu_sync_oos_pages(workload->vgpu);
 705         if (ret) {
 706                 gvt_vgpu_err("fail to vgpu sync oos pages\n");
 707                 goto err_unpin_mm;
 708         }
 709
 710         ret = intel_vgpu_flush_post_shadow(workload->vgpu);
 711         if (ret) {
 712                 gvt_vgpu_err("fail to flush post shadow\n");
 713                 goto err_unpin_mm;
 714         }
 715
 716         ret = copy_workload_to_ring_buffer(workload);
 717         if (ret) {
 718                 gvt_vgpu_err("fail to generate request\n");
 719                 goto err_unpin_mm;
 720         }
 721
 722         ret = prepare_shadow_batch_buffer(workload);
 723         if (ret) {
 724                 gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
 725                 goto err_unpin_mm;
 726         }
 727
 728         ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
 729         if (ret) {
 730                 gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
 731                 goto err_shadow_batch;
 732         }
 733
 734         if (workload->prepare) {
 735                 ret = workload->prepare(workload);
 736                 if (ret)
 737                         goto err_shadow_wa_ctx;
 738         }
 739
 740         return 0;
 741 err_shadow_wa_ctx:
 742         release_shadow_wa_ctx(&workload->wa_ctx);
 743 err_shadow_batch:
 744         release_shadow_batch_buffer(workload);
 745 err_unpin_mm:
 746         intel_vgpu_shadow_mm_unpin(workload);
 747         return ret;
 748 }
 749
 750 static int dispatch_workload(struct intel_vgpu_workload *workload)
 751 {
 752         struct intel_vgpu *vgpu = workload->vgpu;
 753         struct i915_request *rq;
 754         int ret;
 755
 756         gvt_dbg_sched("ring id %s prepare to dispatch workload %p\n",
 757                       workload->engine->name, workload);
 758
 759         mutex_lock(&vgpu->vgpu_lock);
 760
 761         ret = intel_gvt_workload_req_alloc(workload);
 762         if (ret)
 763                 goto err_req;
 764
 765         ret = intel_gvt_scan_and_shadow_workload(workload);
 766         if (ret)
 767                 goto out;
 768
 769         ret = populate_shadow_context(workload);
 770         if (ret) {
 771                 release_shadow_wa_ctx(&workload->wa_ctx);
 772                 goto out;
 773         }
 774
 775         ret = prepare_workload(workload);
 776 out:
 777         if (ret) {
 778                 /* We might still need to add request with
 779                  * clean ctx to retire it properly..
 780                  */
 781                 rq = fetch_and_zero(&workload->req);
 782                 i915_request_put(rq);
 783         }
 784
 785         if (!IS_ERR_OR_NULL(workload->req)) {
 786                 gvt_dbg_sched("ring id %s submit workload to i915 %p\n",
 787                               workload->engine->name, workload->req);
 788                 i915_request_add(workload->req);
 789                 workload->dispatched = true;
 790         }
 791 err_req:
 792         if (ret)
 793                 workload->status = ret;
 794         mutex_unlock(&vgpu->vgpu_lock);
 795         return ret;
 796 }
 797
 798 static struct intel_vgpu_workload *
 799 pick_next_workload(struct intel_gvt *gvt, struct intel_engine_cs *engine)
 800 {
 801         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 802         struct intel_vgpu_workload *workload = NULL;
 803
 804         mutex_lock(&gvt->sched_lock);
 805
 806         /*
 807          * no current vgpu / will be scheduled out / no workload
 808          * bail out
 809          */
 810         if (!scheduler->current_vgpu) {
 811                 gvt_dbg_sched("ring %s stop - no current vgpu\n", engine->name);
 812                 goto out;
 813         }
 814
 815         if (scheduler->need_reschedule) {
 816                 gvt_dbg_sched("ring %s stop - will reschedule\n", engine->name);
 817                 goto out;
 818         }
 819
 820         if (!scheduler->current_vgpu->active ||
 821             list_empty(workload_q_head(scheduler->current_vgpu, engine)))
 822                 goto out;
 823
 824         /*
 825          * still have current workload, maybe the workload disptacher
 826          * fail to submit it for some reason, resubmit it.
 827          */
 828         if (scheduler->current_workload[engine->id]) {
 829                 workload = scheduler->current_workload[engine->id];
 830                 gvt_dbg_sched("ring %s still have current workload %p\n",
 831                               engine->name, workload);
 832                 goto out;
 833         }
 834
 835         /*
 836          * pick a workload as current workload
 837          * once current workload is set, schedule policy routines
 838          * will wait the current workload is finished when trying to
 839          * schedule out a vgpu.
 840          */
 841         scheduler->current_workload[engine->id] =
 842                 list_first_entry(workload_q_head(scheduler->current_vgpu,
 843                                                  engine),
 844                                  struct intel_vgpu_workload, list);
 845
 846         workload = scheduler->current_workload[engine->id];
 847
 848         gvt_dbg_sched("ring %s pick new workload %p\n", engine->name, workload);
 849
 850         atomic_inc(&workload->vgpu->submission.running_workload_num);
 851 out:
 852         mutex_unlock(&gvt->sched_lock);
 853         return workload;
 854 }
 855
 856 static void update_guest_pdps(struct intel_vgpu *vgpu,
 857                               u64 ring_context_gpa, u32 pdp[8])
 858 {
 859         u64 gpa;
 860         int i;
 861
 862         gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val);
 863
 864         for (i = 0; i < 8; i++)
 865                 intel_gvt_hypervisor_write_gpa(vgpu,
 866                                 gpa + i * 8, &pdp[7 - i], 4);
 867 }
 868
 869 static __maybe_unused bool
 870 check_shadow_context_ppgtt(struct execlist_ring_context *c, struct intel_vgpu_mm *m)
 871 {
 872         if (m->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
 873                 u64 shadow_pdp = c->pdps[7].val | (u64) c->pdps[6].val << 32;
 874
 875                 if (shadow_pdp != m->ppgtt_mm.shadow_pdps[0]) {
 876                         gvt_dbg_mm("4-level context ppgtt not match LRI command\n");
 877                         return false;
 878                 }
 879                 return true;
 880         } else {
 881                 /* see comment in LRI handler in cmd_parser.c */
 882                 gvt_dbg_mm("invalid shadow mm type\n");
 883                 return false;
 884         }
 885 }
 886
 887 static void update_guest_context(struct intel_vgpu_workload *workload)
 888 {
 889         struct i915_request *rq = workload->req;
 890         struct intel_vgpu *vgpu = workload->vgpu;
 891         struct execlist_ring_context *shadow_ring_context;
 892         struct intel_context *ctx = workload->req->context;
 893         void *context_base;
 894         void *src;
 895         unsigned long context_gpa, context_page_num;
 896         unsigned long gpa_base; /* first gpa of consecutive GPAs */
 897         unsigned long gpa_size; /* size of consecutive GPAs*/
 898         int i;
 899         u32 ring_base;
 900         u32 head, tail;
 901         u16 wrap_count;
 902
 903         gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id,
 904                       workload->ctx_desc.lrca);
 905
 906         GEM_BUG_ON(!intel_context_is_pinned(ctx));
 907
 908         head = workload->rb_head;
 909         tail = workload->rb_tail;
 910         wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF;
 911
 912         if (tail < head) {
 913                 if (wrap_count == RB_HEAD_WRAP_CNT_MAX)
 914                         wrap_count = 0;
 915                 else
 916                         wrap_count += 1;
 917         }
 918
 919         head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail;
 920
 921         ring_base = rq->engine->mmio_base;
 922         vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail;
 923         vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head;
 924
 925         context_page_num = rq->engine->context_size;
 926         context_page_num = context_page_num >> PAGE_SHIFT;
 927
 928         if (IS_BROADWELL(rq->engine->i915) && rq->engine->id == RCS0)
 929                 context_page_num = 19;
 930
 931         context_base = (void *) ctx->lrc_reg_state -
 932                         (LRC_STATE_PN << I915_GTT_PAGE_SHIFT);
 933
 934         /* find consecutive GPAs from gma until the first inconsecutive GPA.
 935          * write to the consecutive GPAs from src virtual address
 936          */
 937         gpa_size = 0;
 938         for (i = 2; i < context_page_num; i++) {
 939                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 940                                 (u32)((workload->ctx_desc.lrca + i) <<
 941                                         I915_GTT_PAGE_SHIFT));
 942                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 943                         gvt_vgpu_err("invalid guest context descriptor\n");
 944                         return;
 945                 }
 946
 947                 if (gpa_size == 0) {
 948                         gpa_base = context_gpa;
 949                         src = context_base + (i << I915_GTT_PAGE_SHIFT);
 950                 } else if (context_gpa != gpa_base + gpa_size)
 951                         goto write;
 952
 953                 gpa_size += I915_GTT_PAGE_SIZE;
 954
 955                 if (i == context_page_num - 1)
 956                         goto write;
 957
 958                 continue;
 959
 960 write:
 961                 intel_gvt_hypervisor_write_gpa(vgpu, gpa_base, src, gpa_size);
 962                 gpa_base = context_gpa;
 963                 gpa_size = I915_GTT_PAGE_SIZE;
 964                 src = context_base + (i << I915_GTT_PAGE_SHIFT);
 965         }
 966
 967         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
 968                 RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
 969
 970         shadow_ring_context = (void *) ctx->lrc_reg_state;
 971
 972         if (!list_empty(&workload->lri_shadow_mm)) {
 973                 struct intel_vgpu_mm *m = list_last_entry(&workload->lri_shadow_mm,
 974                                                           struct intel_vgpu_mm,
 975                                                           ppgtt_mm.link);
 976                 GEM_BUG_ON(!check_shadow_context_ppgtt(shadow_ring_context, m));
 977                 update_guest_pdps(vgpu, workload->ring_context_gpa,
 978                                   (void *)m->ppgtt_mm.guest_pdps);
 979         }
 980
 981 #define COPY_REG(name) \
 982         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
 983                 RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
 984
 985         COPY_REG(ctx_ctrl);
 986         COPY_REG(ctx_timestamp);
 987
 988 #undef COPY_REG
 989
 990         intel_gvt_hypervisor_write_gpa(vgpu,
 991                         workload->ring_context_gpa +
 992                         sizeof(*shadow_ring_context),
 993                         (void *)shadow_ring_context +
 994                         sizeof(*shadow_ring_context),
 995                         I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 996 }
 997
 998 void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
 999                                 intel_engine_mask_t engine_mask)
1000 {
1001         struct intel_vgpu_submission *s = &vgpu->submission;
1002         struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;
1003         struct intel_engine_cs *engine;
1004         struct intel_vgpu_workload *pos, *n;
1005         intel_engine_mask_t tmp;
1006
1007         /* free the unsubmited workloads in the queues. */
1008         for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) {
1009                 list_for_each_entry_safe(pos, n,
1010                         &s->workload_q_head[engine->id], list) {
1011                         list_del_init(&pos->list);
1012                         intel_vgpu_destroy_workload(pos);
1013                 }
1014                 clear_bit(engine->id, s->shadow_ctx_desc_updated);
1015         }
1016 }
1017
1018 static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
1019 {
1020         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1021         struct intel_vgpu_workload *workload =
1022                 scheduler->current_workload[ring_id];
1023         struct intel_vgpu *vgpu = workload->vgpu;
1024         struct intel_vgpu_submission *s = &vgpu->submission;
1025         struct i915_request *rq = workload->req;
1026         int event;
1027
1028         mutex_lock(&vgpu->vgpu_lock);
1029         mutex_lock(&gvt->sched_lock);
1030
1031         /* For the workload w/ request, needs to wait for the context
1032          * switch to make sure request is completed.
1033          * For the workload w/o request, directly complete the workload.
1034          */
1035         if (rq) {
1036                 wait_event(workload->shadow_ctx_status_wq,
1037                            !atomic_read(&workload->shadow_ctx_active));
1038
1039                 /* If this request caused GPU hang, req->fence.error will
1040                  * be set to -EIO. Use -EIO to set workload status so
1041                  * that when this request caused GPU hang, didn't trigger
1042                  * context switch interrupt to guest.
1043                  */
1044                 if (likely(workload->status == -EINPROGRESS)) {
1045                         if (workload->req->fence.error == -EIO)
1046                                 workload->status = -EIO;
1047                         else
1048                                 workload->status = 0;
1049                 }
1050
1051                 if (!workload->status &&
1052                     !(vgpu->resetting_eng & BIT(ring_id))) {
1053                         update_guest_context(workload);
1054
1055                         for_each_set_bit(event, workload->pending_events,
1056                                          INTEL_GVT_EVENT_MAX)
1057                                 intel_vgpu_trigger_virtual_event(vgpu, event);
1058                 }
1059
1060                 i915_request_put(fetch_and_zero(&workload->req));
1061         }
1062
1063         gvt_dbg_sched("ring id %d complete workload %p status %d\n",
1064                         ring_id, workload, workload->status);
1065
1066         scheduler->current_workload[ring_id] = NULL;
1067
1068         list_del_init(&workload->list);
1069
1070         if (workload->status || vgpu->resetting_eng & BIT(ring_id)) {
1071                 /* if workload->status is not successful means HW GPU
1072                  * has occurred GPU hang or something wrong with i915/GVT,
1073                  * and GVT won't inject context switch interrupt to guest.
1074                  * So this error is a vGPU hang actually to the guest.
1075                  * According to this we should emunlate a vGPU hang. If
1076                  * there are pending workloads which are already submitted
1077                  * from guest, we should clean them up like HW GPU does.
1078                  *
1079                  * if it is in middle of engine resetting, the pending
1080                  * workloads won't be submitted to HW GPU and will be
1081                  * cleaned up during the resetting process later, so doing
1082                  * the workload clean up here doesn't have any impact.
1083                  **/
1084                 intel_vgpu_clean_workloads(vgpu, BIT(ring_id));
1085         }
1086
1087         workload->complete(workload);
1088
1089         intel_vgpu_shadow_mm_unpin(workload);
1090         intel_vgpu_destroy_workload(workload);
1091
1092         atomic_dec(&s->running_workload_num);
1093         wake_up(&scheduler->workload_complete_wq);
1094
1095         if (gvt->scheduler.need_reschedule)
1096                 intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);
1097
1098         mutex_unlock(&gvt->sched_lock);
1099         mutex_unlock(&vgpu->vgpu_lock);
1100 }
1101
1102 static int workload_thread(void *arg)
1103 {
1104         struct intel_engine_cs *engine = arg;
1105         const bool need_force_wake = INTEL_GEN(engine->i915) >= 9;
1106         struct intel_gvt *gvt = engine->i915->gvt;
1107         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1108         struct intel_vgpu_workload *workload = NULL;
1109         struct intel_vgpu *vgpu = NULL;
1110         int ret;
1111         DEFINE_WAIT_FUNC(wait, woken_wake_function);
1112
1113         gvt_dbg_core("workload thread for ring %s started\n", engine->name);
1114
1115         while (!kthread_should_stop()) {
1116                 intel_wakeref_t wakeref;
1117
1118                 add_wait_queue(&scheduler->waitq[engine->id], &wait);
1119                 do {
1120                         workload = pick_next_workload(gvt, engine);
1121                         if (workload)
1122                                 break;
1123                         wait_woken(&wait, TASK_INTERRUPTIBLE,
1124                                    MAX_SCHEDULE_TIMEOUT);
1125                 } while (!kthread_should_stop());
1126                 remove_wait_queue(&scheduler->waitq[engine->id], &wait);
1127
1128                 if (!workload)
1129                         break;
1130
1131                 gvt_dbg_sched("ring %s next workload %p vgpu %d\n",
1132                               engine->name, workload,
1133                               workload->vgpu->id);
1134
1135                 wakeref = intel_runtime_pm_get(engine->uncore->rpm);
1136
1137                 gvt_dbg_sched("ring %s will dispatch workload %p\n",
1138                               engine->name, workload);
1139
1140                 if (need_force_wake)
1141                         intel_uncore_forcewake_get(engine->uncore,
1142                                                    FORCEWAKE_ALL);
1143                 /*
1144                  * Update the vReg of the vGPU which submitted this
1145                  * workload. The vGPU may use these registers for checking
1146                  * the context state. The value comes from GPU commands
1147                  * in this workload.
1148                  */
1149                 update_vreg_in_ctx(workload);
1150
1151                 ret = dispatch_workload(workload);
1152
1153                 if (ret) {
1154                         vgpu = workload->vgpu;
1155                         gvt_vgpu_err("fail to dispatch workload, skip\n");
1156                         goto complete;
1157                 }
1158
1159                 gvt_dbg_sched("ring %s wait workload %p\n",
1160                               engine->name, workload);
1161                 i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT);
1162
1163 complete:
1164                 gvt_dbg_sched("will complete workload %p, status: %d\n",
1165                               workload, workload->status);
1166
1167                 complete_current_workload(gvt, engine->id);
1168
1169                 if (need_force_wake)
1170                         intel_uncore_forcewake_put(engine->uncore,
1171                                                    FORCEWAKE_ALL);
1172
1173                 intel_runtime_pm_put(engine->uncore->rpm, wakeref);
1174                 if (ret && (vgpu_is_vm_unhealthy(ret)))
1175                         enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
1176         }
1177         return 0;
1178 }
1179
1180 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
1181 {
1182         struct intel_vgpu_submission *s = &vgpu->submission;
1183         struct intel_gvt *gvt = vgpu->gvt;
1184         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1185
1186         if (atomic_read(&s->running_workload_num)) {
1187                 gvt_dbg_sched("wait vgpu idle\n");
1188
1189                 wait_event(scheduler->workload_complete_wq,
1190                                 !atomic_read(&s->running_workload_num));
1191         }
1192 }
1193
1194 void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
1195 {
1196         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1197         struct intel_engine_cs *engine;
1198         enum intel_engine_id i;
1199
1200         gvt_dbg_core("clean workload scheduler\n");
1201
1202         for_each_engine(engine, gvt->gt, i) {
1203                 atomic_notifier_chain_unregister(
1204                                         &engine->context_status_notifier,
1205                                         &gvt->shadow_ctx_notifier_block[i]);
1206                 kthread_stop(scheduler->thread[i]);
1207         }
1208 }
1209
1210 int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
1211 {
1212         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1213         struct intel_engine_cs *engine;
1214         enum intel_engine_id i;
1215         int ret;
1216
1217         gvt_dbg_core("init workload scheduler\n");
1218
1219         init_waitqueue_head(&scheduler->workload_complete_wq);
1220
1221         for_each_engine(engine, gvt->gt, i) {
1222                 init_waitqueue_head(&scheduler->waitq[i]);
1223
1224                 scheduler->thread[i] = kthread_run(workload_thread, engine,
1225                                                    "gvt:%s", engine->name);
1226                 if (IS_ERR(scheduler->thread[i])) {
1227                         gvt_err("fail to create workload thread\n");
1228                         ret = PTR_ERR(scheduler->thread[i]);
1229                         goto err;
1230                 }
1231
1232                 gvt->shadow_ctx_notifier_block[i].notifier_call =
1233                                         shadow_context_status_change;
1234                 atomic_notifier_chain_register(&engine->context_status_notifier,
1235                                         &gvt->shadow_ctx_notifier_block[i]);
1236         }
1237
1238         return 0;
1239
1240 err:
1241         intel_gvt_clean_workload_scheduler(gvt);
1242         return ret;
1243 }
1244
1245 static void
1246 i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s,
1247                                 struct i915_ppgtt *ppgtt)
1248 {
1249         int i;
1250
1251         if (i915_vm_is_4lvl(&ppgtt->vm)) {
1252                 set_dma_address(ppgtt->pd, s->i915_context_pml4);
1253         } else {
1254                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1255                         struct i915_page_directory * const pd =
1256                                 i915_pd_entry(ppgtt->pd, i);
1257
1258                         set_dma_address(pd, s->i915_context_pdps[i]);
1259                 }
1260         }
1261 }
1262
1263 /**
1264  * intel_vgpu_clean_submission - free submission-related resource for vGPU
1265  * @vgpu: a vGPU
1266  *
1267  * This function is called when a vGPU is being destroyed.
1268  *
1269  */
1270 void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
1271 {
1272         struct intel_vgpu_submission *s = &vgpu->submission;
1273         struct intel_engine_cs *engine;
1274         enum intel_engine_id id;
1275
1276         intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0);
1277
1278         i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(s->shadow[0]->vm));
1279         for_each_engine(engine, vgpu->gvt->gt, id)
1280                 intel_context_put(s->shadow[id]);
1281
1282         kmem_cache_destroy(s->workloads);
1283 }
1284
1285
1286 /**
1287  * intel_vgpu_reset_submission - reset submission-related resource for vGPU
1288  * @vgpu: a vGPU
1289  * @engine_mask: engines expected to be reset
1290  *
1291  * This function is called when a vGPU is being destroyed.
1292  *
1293  */
1294 void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
1295                                  intel_engine_mask_t engine_mask)
1296 {
1297         struct intel_vgpu_submission *s = &vgpu->submission;
1298
1299         if (!s->active)
1300                 return;
1301
1302         intel_vgpu_clean_workloads(vgpu, engine_mask);
1303         s->ops->reset(vgpu, engine_mask);
1304 }
1305
1306 static void
1307 i915_context_ppgtt_root_save(struct intel_vgpu_submission *s,
1308                              struct i915_ppgtt *ppgtt)
1309 {
1310         int i;
1311
1312         if (i915_vm_is_4lvl(&ppgtt->vm)) {
1313                 s->i915_context_pml4 = px_dma(ppgtt->pd);
1314         } else {
1315                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1316                         struct i915_page_directory * const pd =
1317                                 i915_pd_entry(ppgtt->pd, i);
1318
1319                         s->i915_context_pdps[i] = px_dma(pd);
1320                 }
1321         }
1322 }
1323
1324 /**
1325  * intel_vgpu_setup_submission - setup submission-related resource for vGPU
1326  * @vgpu: a vGPU
1327  *
1328  * This function is called when a vGPU is being created.
1329  *
1330  * Returns:
1331  * Zero on success, negative error code if failed.
1332  *
1333  */
1334 int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
1335 {
1336         struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
1337         struct intel_vgpu_submission *s = &vgpu->submission;
1338         struct intel_engine_cs *engine;
1339         struct i915_ppgtt *ppgtt;
1340         enum intel_engine_id i;
1341         int ret;
1342
1343         ppgtt = i915_ppgtt_create(&i915->gt);
1344         if (IS_ERR(ppgtt))
1345                 return PTR_ERR(ppgtt);
1346
1347         i915_context_ppgtt_root_save(s, ppgtt);
1348
1349         for_each_engine(engine, vgpu->gvt->gt, i) {
1350                 struct intel_context *ce;
1351
1352                 INIT_LIST_HEAD(&s->workload_q_head[i]);
1353                 s->shadow[i] = ERR_PTR(-EINVAL);
1354
1355                 ce = intel_context_create(engine);
1356                 if (IS_ERR(ce)) {
1357                         ret = PTR_ERR(ce);
1358                         goto out_shadow_ctx;
1359                 }
1360
1361                 i915_vm_put(ce->vm);
1362                 ce->vm = i915_vm_get(&ppgtt->vm);
1363                 intel_context_set_single_submission(ce);
1364
1365                 /* Max ring buffer size */
1366                 if (!intel_uc_wants_guc_submission(&engine->gt->uc)) {
1367                         const unsigned int ring_size = 512 * SZ_4K;
1368
1369                         ce->ring = __intel_context_ring_size(ring_size);
1370                 }
1371
1372                 s->shadow[i] = ce;
1373         }
1374
1375         bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
1376
1377         s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload",
1378                                                   sizeof(struct intel_vgpu_workload), 0,
1379                                                   SLAB_HWCACHE_ALIGN,
1380                                                   offsetof(struct intel_vgpu_workload, rb_tail),
1381                                                   sizeof_field(struct intel_vgpu_workload, rb_tail),
1382                                                   NULL);
1383
1384         if (!s->workloads) {
1385                 ret = -ENOMEM;
1386                 goto out_shadow_ctx;
1387         }
1388
1389         atomic_set(&s->running_workload_num, 0);
1390         bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES);
1391
1392         memset(s->last_ctx, 0, sizeof(s->last_ctx));
1393
1394         i915_vm_put(&ppgtt->vm);
1395         return 0;
1396
1397 out_shadow_ctx:
1398         i915_context_ppgtt_root_restore(s, ppgtt);
1399         for_each_engine(engine, vgpu->gvt->gt, i) {
1400                 if (IS_ERR(s->shadow[i]))
1401                         break;
1402
1403                 intel_context_put(s->shadow[i]);
1404         }
1405         i915_vm_put(&ppgtt->vm);
1406         return ret;
1407 }
1408
1409 /**
1410  * intel_vgpu_select_submission_ops - select virtual submission interface
1411  * @vgpu: a vGPU
1412  * @engine_mask: either ALL_ENGINES or target engine mask
1413  * @interface: expected vGPU virtual submission interface
1414  *
1415  * This function is called when guest configures submission interface.
1416  *
1417  * Returns:
1418  * Zero on success, negative error code if failed.
1419  *
1420  */
1421 int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
1422                                      intel_engine_mask_t engine_mask,
1423                                      unsigned int interface)
1424 {
1425         struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
1426         struct intel_vgpu_submission *s = &vgpu->submission;
1427         const struct intel_vgpu_submission_ops *ops[] = {
1428                 [INTEL_VGPU_EXECLIST_SUBMISSION] =
1429                         &intel_vgpu_execlist_submission_ops,
1430         };
1431         int ret;
1432
1433         if (drm_WARN_ON(&i915->drm, interface >= ARRAY_SIZE(ops)))
1434                 return -EINVAL;
1435
1436         if (drm_WARN_ON(&i915->drm,
1437                         interface == 0 && engine_mask != ALL_ENGINES))
1438                 return -EINVAL;
1439
1440         if (s->active)
1441                 s->ops->clean(vgpu, engine_mask);
1442
1443         if (interface == 0) {
1444                 s->ops = NULL;
1445                 s->virtual_submission_interface = 0;
1446                 s->active = false;
1447                 gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id);
1448                 return 0;
1449         }
1450
1451         ret = ops[interface]->init(vgpu, engine_mask);
1452         if (ret)
1453                 return ret;
1454
1455         s->ops = ops[interface];
1456         s->virtual_submission_interface = interface;
1457         s->active = true;
1458
1459         gvt_dbg_core("vgpu%d: activate ops [ %s ]\n",
1460                         vgpu->id, s->ops->name);
1461
1462         return 0;
1463 }
1464
1465 /**
1466  * intel_vgpu_destroy_workload - destroy a vGPU workload
1467  * @workload: workload to destroy
1468  *
1469  * This function is called when destroy a vGPU workload.
1470  *
1471  */
1472 void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
1473 {
1474         struct intel_vgpu_submission *s = &workload->vgpu->submission;
1475
1476         intel_context_unpin(s->shadow[workload->engine->id]);
1477         release_shadow_batch_buffer(workload);
1478         release_shadow_wa_ctx(&workload->wa_ctx);
1479
1480         if (!list_empty(&workload->lri_shadow_mm)) {
1481                 struct intel_vgpu_mm *m, *mm;
1482                 list_for_each_entry_safe(m, mm, &workload->lri_shadow_mm,
1483                                          ppgtt_mm.link) {
1484                         list_del(&m->ppgtt_mm.link);
1485                         intel_vgpu_mm_put(m);
1486                 }
1487         }
1488
1489         GEM_BUG_ON(!list_empty(&workload->lri_shadow_mm));
1490         if (workload->shadow_mm)
1491                 intel_vgpu_mm_put(workload->shadow_mm);
1492
1493         kmem_cache_free(s->workloads, workload);
1494 }
1495
1496 static struct intel_vgpu_workload *
1497 alloc_workload(struct intel_vgpu *vgpu)
1498 {
1499         struct intel_vgpu_submission *s = &vgpu->submission;
1500         struct intel_vgpu_workload *workload;
1501
1502         workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);
1503         if (!workload)
1504                 return ERR_PTR(-ENOMEM);
1505
1506         INIT_LIST_HEAD(&workload->list);
1507         INIT_LIST_HEAD(&workload->shadow_bb);
1508         INIT_LIST_HEAD(&workload->lri_shadow_mm);
1509
1510         init_waitqueue_head(&workload->shadow_ctx_status_wq);
1511         atomic_set(&workload->shadow_ctx_active, 0);
1512
1513         workload->status = -EINPROGRESS;
1514         workload->vgpu = vgpu;
1515
1516         return workload;
1517 }
1518
1519 #define RING_CTX_OFF(x) \
1520         offsetof(struct execlist_ring_context, x)
1521
1522 static void read_guest_pdps(struct intel_vgpu *vgpu,
1523                 u64 ring_context_gpa, u32 pdp[8])
1524 {
1525         u64 gpa;
1526         int i;
1527
1528         gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val);
1529
1530         for (i = 0; i < 8; i++)
1531                 intel_gvt_hypervisor_read_gpa(vgpu,
1532                                 gpa + i * 8, &pdp[7 - i], 4);
1533 }
1534
1535 static int prepare_mm(struct intel_vgpu_workload *workload)
1536 {
1537         struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
1538         struct intel_vgpu_mm *mm;
1539         struct intel_vgpu *vgpu = workload->vgpu;
1540         enum intel_gvt_gtt_type root_entry_type;
1541         u64 pdps[GVT_RING_CTX_NR_PDPS];
1542
1543         switch (desc->addressing_mode) {
1544         case 1: /* legacy 32-bit */
1545                 root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1546                 break;
1547         case 3: /* legacy 64-bit */
1548                 root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1549                 break;
1550         default:
1551                 gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
1552                 return -EINVAL;
1553         }
1554
1555         read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps);
1556
1557         mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps);
1558         if (IS_ERR(mm))
1559                 return PTR_ERR(mm);
1560
1561         workload->shadow_mm = mm;
1562         return 0;
1563 }
1564
1565 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
1566                 ((a)->lrca == (b)->lrca))
1567
1568 /**
1569  * intel_vgpu_create_workload - create a vGPU workload
1570  * @vgpu: a vGPU
1571  * @engine: the engine
1572  * @desc: a guest context descriptor
1573  *
1574  * This function is called when creating a vGPU workload.
1575  *
1576  * Returns:
1577  * struct intel_vgpu_workload * on success, negative error code in
1578  * pointer if failed.
1579  *
1580  */
1581 struct intel_vgpu_workload *
1582 intel_vgpu_create_workload(struct intel_vgpu *vgpu,
1583                            const struct intel_engine_cs *engine,
1584                            struct execlist_ctx_descriptor_format *desc)
1585 {
1586         struct intel_vgpu_submission *s = &vgpu->submission;
1587         struct list_head *q = workload_q_head(vgpu, engine);
1588         struct intel_vgpu_workload *last_workload = NULL;
1589         struct intel_vgpu_workload *workload = NULL;
1590         u64 ring_context_gpa;
1591         u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
1592         u32 guest_head;
1593         int ret;
1594
1595         ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
1596                         (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT));
1597         if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
1598                 gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
1599                 return ERR_PTR(-EINVAL);
1600         }
1601
1602         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1603                         RING_CTX_OFF(ring_header.val), &head, 4);
1604
1605         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1606                         RING_CTX_OFF(ring_tail.val), &tail, 4);
1607
1608         guest_head = head;
1609
1610         head &= RB_HEAD_OFF_MASK;
1611         tail &= RB_TAIL_OFF_MASK;
1612
1613         list_for_each_entry_reverse(last_workload, q, list) {
1614
1615                 if (same_context(&last_workload->ctx_desc, desc)) {
1616                         gvt_dbg_el("ring %s cur workload == last\n",
1617                                    engine->name);
1618                         gvt_dbg_el("ctx head %x real head %lx\n", head,
1619                                    last_workload->rb_tail);
1620                         /*
1621                          * cannot use guest context head pointer here,
1622                          * as it might not be updated at this time
1623                          */
1624                         head = last_workload->rb_tail;
1625                         break;
1626                 }
1627         }
1628
1629         gvt_dbg_el("ring %s begin a new workload\n", engine->name);
1630
1631         /* record some ring buffer register values for scan and shadow */
1632         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1633                         RING_CTX_OFF(rb_start.val), &start, 4);
1634         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1635                         RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
1636         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1637                         RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
1638
1639         if (!intel_gvt_ggtt_validate_range(vgpu, start,
1640                                 _RING_CTL_BUF_SIZE(ctl))) {
1641                 gvt_vgpu_err("context contain invalid rb at: 0x%x\n", start);
1642                 return ERR_PTR(-EINVAL);
1643         }
1644
1645         workload = alloc_workload(vgpu);
1646         if (IS_ERR(workload))
1647                 return workload;
1648
1649         workload->engine = engine;
1650         workload->ctx_desc = *desc;
1651         workload->ring_context_gpa = ring_context_gpa;
1652         workload->rb_head = head;
1653         workload->guest_rb_head = guest_head;
1654         workload->rb_tail = tail;
1655         workload->rb_start = start;
1656         workload->rb_ctl = ctl;
1657
1658         if (engine->id == RCS0) {
1659                 intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1660                         RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
1661                 intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1662                         RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
1663
1664                 workload->wa_ctx.indirect_ctx.guest_gma =
1665                         indirect_ctx & INDIRECT_CTX_ADDR_MASK;
1666                 workload->wa_ctx.indirect_ctx.size =
1667                         (indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
1668                         CACHELINE_BYTES;
1669
1670                 if (workload->wa_ctx.indirect_ctx.size != 0) {
1671                         if (!intel_gvt_ggtt_validate_range(vgpu,
1672                                 workload->wa_ctx.indirect_ctx.guest_gma,
1673                                 workload->wa_ctx.indirect_ctx.size)) {
1674                                 gvt_vgpu_err("invalid wa_ctx at: 0x%lx\n",
1675                                     workload->wa_ctx.indirect_ctx.guest_gma);
1676                                 kmem_cache_free(s->workloads, workload);
1677                                 return ERR_PTR(-EINVAL);
1678                         }
1679                 }
1680
1681                 workload->wa_ctx.per_ctx.guest_gma =
1682                         per_ctx & PER_CTX_ADDR_MASK;
1683                 workload->wa_ctx.per_ctx.valid = per_ctx & 1;
1684                 if (workload->wa_ctx.per_ctx.valid) {
1685                         if (!intel_gvt_ggtt_validate_range(vgpu,
1686                                 workload->wa_ctx.per_ctx.guest_gma,
1687                                 CACHELINE_BYTES)) {
1688                                 gvt_vgpu_err("invalid per_ctx at: 0x%lx\n",
1689                                         workload->wa_ctx.per_ctx.guest_gma);
1690                                 kmem_cache_free(s->workloads, workload);
1691                                 return ERR_PTR(-EINVAL);
1692                         }
1693                 }
1694         }
1695
1696         gvt_dbg_el("workload %p ring %s head %x tail %x start %x ctl %x\n",
1697                    workload, engine->name, head, tail, start, ctl);
1698
1699         ret = prepare_mm(workload);
1700         if (ret) {
1701                 kmem_cache_free(s->workloads, workload);
1702                 return ERR_PTR(ret);
1703         }
1704
1705         /* Only scan and shadow the first workload in the queue
1706          * as there is only one pre-allocated buf-obj for shadow.
1707          */
1708         if (list_empty(q)) {
1709                 intel_wakeref_t wakeref;
1710
1711                 with_intel_runtime_pm(engine->gt->uncore->rpm, wakeref)
1712                         ret = intel_gvt_scan_and_shadow_workload(workload);
1713         }
1714
1715         if (ret) {
1716                 if (vgpu_is_vm_unhealthy(ret))
1717                         enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
1718                 intel_vgpu_destroy_workload(workload);
1719                 return ERR_PTR(ret);
1720         }
1721
1722         ret = intel_context_pin(s->shadow[engine->id]);
1723         if (ret) {
1724                 intel_vgpu_destroy_workload(workload);
1725                 return ERR_PTR(ret);
1726         }
1727
1728         return workload;
1729 }
1730
1731 /**
1732  * intel_vgpu_queue_workload - Qeue a vGPU workload
1733  * @workload: the workload to queue in
1734  */
1735 void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload)
1736 {
1737         list_add_tail(&workload->list,
1738                       workload_q_head(workload->vgpu, workload->engine));
1739         intel_gvt_kick_schedule(workload->vgpu->gvt);
1740         wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->engine->id]);
1741 }