drivers/gpu/drm/i915/gt/gen7_renderclear.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "gen7_renderclear.h"
   7 #include "i915_drv.h"
   8 #include "intel_gpu_commands.h"
   9
  10 #define GT3_INLINE_DATA_DELAYS 0x1E00
  11 #define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS))
  12
  13 struct cb_kernel {
  14         const void *data;
  15         u32 size;
  16 };
  17
  18 #define CB_KERNEL(name) { .data = (name), .size = sizeof(name) }
  19
  20 #include "ivb_clear_kernel.c"
  21 static const struct cb_kernel cb_kernel_ivb = CB_KERNEL(ivb_clear_kernel);
  22
  23 #include "hsw_clear_kernel.c"
  24 static const struct cb_kernel cb_kernel_hsw = CB_KERNEL(hsw_clear_kernel);
  25
  26 struct batch_chunk {
  27         struct i915_vma *vma;
  28         u32 offset;
  29         u32 *start;
  30         u32 *end;
  31         u32 max_items;
  32 };
  33
  34 struct batch_vals {
  35         u32 max_threads;
  36         u32 state_start;
  37         u32 surface_start;
  38         u32 surface_height;
  39         u32 surface_width;
  40         u32 size;
  41 };
  42
  43 static int num_primitives(const struct batch_vals *bv)
  44 {
  45         /*
  46          * We need to saturate the GPU with work in order to dispatch
  47          * a shader on every HW thread, and clear the thread-local registers.
  48          * In short, we have to dispatch work faster than the shaders can
  49          * run in order to fill the EU and occupy each HW thread.
  50          */
  51         return bv->max_threads;
  52 }
  53
  54 static void
  55 batch_get_defaults(struct drm_i915_private *i915, struct batch_vals *bv)
  56 {
  57         if (IS_HASWELL(i915)) {
  58                 switch (INTEL_INFO(i915)->gt) {
  59                 default:
  60                 case 1:
  61                         bv->max_threads = 70;
  62                         break;
  63                 case 2:
  64                         bv->max_threads = 140;
  65                         break;
  66                 case 3:
  67                         bv->max_threads = 280;
  68                         break;
  69                 }
  70                 bv->surface_height = 16 * 16;
  71                 bv->surface_width = 32 * 2 * 16;
  72         } else {
  73                 switch (INTEL_INFO(i915)->gt) {
  74                 default:
  75                 case 1: /* including vlv */
  76                         bv->max_threads = 36;
  77                         break;
  78                 case 2:
  79                         bv->max_threads = 128;
  80                         break;
  81                 }
  82                 bv->surface_height = 16 * 8;
  83                 bv->surface_width = 32 * 16;
  84         }
  85         bv->state_start = round_up(SZ_1K + num_primitives(bv) * 64, SZ_4K);
  86         bv->surface_start = bv->state_start + SZ_4K;
  87         bv->size = bv->surface_start + bv->surface_height * bv->surface_width;
  88 }
  89
  90 static void batch_init(struct batch_chunk *bc,
  91                        struct i915_vma *vma,
  92                        u32 *start, u32 offset, u32 max_bytes)
  93 {
  94         bc->vma = vma;
  95         bc->offset = offset;
  96         bc->start = start + bc->offset / sizeof(*bc->start);
  97         bc->end = bc->start;
  98         bc->max_items = max_bytes / sizeof(*bc->start);
  99 }
 100
 101 static u32 batch_offset(const struct batch_chunk *bc, u32 *cs)
 102 {
 103         return (cs - bc->start) * sizeof(*bc->start) + bc->offset;
 104 }
 105
 106 static u32 batch_addr(const struct batch_chunk *bc)
 107 {
 108         return bc->vma->node.start;
 109 }
 110
 111 static void batch_add(struct batch_chunk *bc, const u32 d)
 112 {
 113         GEM_BUG_ON((bc->end - bc->start) >= bc->max_items);
 114         *bc->end++ = d;
 115 }
 116
 117 static u32 *batch_alloc_items(struct batch_chunk *bc, u32 align, u32 items)
 118 {
 119         u32 *map;
 120
 121         if (align) {
 122                 u32 *end = PTR_ALIGN(bc->end, align);
 123
 124                 memset32(bc->end, 0, end - bc->end);
 125                 bc->end = end;
 126         }
 127
 128         map = bc->end;
 129         bc->end += items;
 130
 131         return map;
 132 }
 133
 134 static u32 *batch_alloc_bytes(struct batch_chunk *bc, u32 align, u32 bytes)
 135 {
 136         GEM_BUG_ON(!IS_ALIGNED(bytes, sizeof(*bc->start)));
 137         return batch_alloc_items(bc, align, bytes / sizeof(*bc->start));
 138 }
 139
 140 static u32
 141 gen7_fill_surface_state(struct batch_chunk *state,
 142                         const u32 dst_offset,
 143                         const struct batch_vals *bv)
 144 {
 145         u32 surface_h = bv->surface_height;
 146         u32 surface_w = bv->surface_width;
 147         u32 *cs = batch_alloc_items(state, 32, 8);
 148         u32 offset = batch_offset(state, cs);
 149
 150 #define SURFACE_2D 1
 151 #define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0
 152 #define RENDER_CACHE_READ_WRITE 1
 153
 154         *cs++ = SURFACE_2D << 29 |
 155                 (SURFACEFORMAT_B8G8R8A8_UNORM << 18) |
 156                 (RENDER_CACHE_READ_WRITE << 8);
 157
 158         *cs++ = batch_addr(state) + dst_offset;
 159
 160         *cs++ = ((surface_h / 4 - 1) << 16) | (surface_w / 4 - 1);
 161         *cs++ = surface_w;
 162         *cs++ = 0;
 163         *cs++ = 0;
 164         *cs++ = 0;
 165 #define SHADER_CHANNELS(r, g, b, a) \
 166         (((r) << 25) | ((g) << 22) | ((b) << 19) | ((a) << 16))
 167         *cs++ = SHADER_CHANNELS(4, 5, 6, 7);
 168         batch_advance(state, cs);
 169
 170         return offset;
 171 }
 172
 173 static u32
 174 gen7_fill_binding_table(struct batch_chunk *state,
 175                         const struct batch_vals *bv)
 176 {
 177         u32 surface_start =
 178                 gen7_fill_surface_state(state, bv->surface_start, bv);
 179         u32 *cs = batch_alloc_items(state, 32, 8);
 180         u32 offset = batch_offset(state, cs);
 181
 182         *cs++ = surface_start - state->offset;
 183         *cs++ = 0;
 184         *cs++ = 0;
 185         *cs++ = 0;
 186         *cs++ = 0;
 187         *cs++ = 0;
 188         *cs++ = 0;
 189         *cs++ = 0;
 190         batch_advance(state, cs);
 191
 192         return offset;
 193 }
 194
 195 static u32
 196 gen7_fill_kernel_data(struct batch_chunk *state,
 197                       const u32 *data,
 198                       const u32 size)
 199 {
 200         return batch_offset(state,
 201                             memcpy(batch_alloc_bytes(state, 64, size),
 202                                    data, size));
 203 }
 204
 205 static u32
 206 gen7_fill_interface_descriptor(struct batch_chunk *state,
 207                                const struct batch_vals *bv,
 208                                const struct cb_kernel *kernel,
 209                                unsigned int count)
 210 {
 211         u32 kernel_offset =
 212                 gen7_fill_kernel_data(state, kernel->data, kernel->size);
 213         u32 binding_table = gen7_fill_binding_table(state, bv);
 214         u32 *cs = batch_alloc_items(state, 32, 8 * count);
 215         u32 offset = batch_offset(state, cs);
 216
 217         *cs++ = kernel_offset;
 218         *cs++ = (1 << 7) | (1 << 13);
 219         *cs++ = 0;
 220         *cs++ = (binding_table - state->offset) | 1;
 221         *cs++ = 0;
 222         *cs++ = 0;
 223         *cs++ = 0;
 224         *cs++ = 0;
 225
 226         /* 1 - 63dummy idds */
 227         memset32(cs, 0x00, (count - 1) * 8);
 228         batch_advance(state, cs + (count - 1) * 8);
 229
 230         return offset;
 231 }
 232
 233 static void
 234 gen7_emit_state_base_address(struct batch_chunk *batch,
 235                              u32 surface_state_base)
 236 {
 237         u32 *cs = batch_alloc_items(batch, 0, 10);
 238
 239         *cs++ = STATE_BASE_ADDRESS | (10 - 2);
 240         /* general */
 241         *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
 242         /* surface */
 243         *cs++ = (batch_addr(batch) + surface_state_base) | BASE_ADDRESS_MODIFY;
 244         /* dynamic */
 245         *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
 246         /* indirect */
 247         *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
 248         /* instruction */
 249         *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
 250
 251         /* general/dynamic/indirect/instruction access Bound */
 252         *cs++ = 0;
 253         *cs++ = BASE_ADDRESS_MODIFY;
 254         *cs++ = 0;
 255         *cs++ = BASE_ADDRESS_MODIFY;
 256         batch_advance(batch, cs);
 257 }
 258
 259 static void
 260 gen7_emit_vfe_state(struct batch_chunk *batch,
 261                     const struct batch_vals *bv,
 262                     u32 urb_size, u32 curbe_size,
 263                     u32 mode)
 264 {
 265         u32 threads = bv->max_threads - 1;
 266         u32 *cs = batch_alloc_items(batch, 32, 8);
 267
 268         *cs++ = MEDIA_VFE_STATE | (8 - 2);
 269
 270         /* scratch buffer */
 271         *cs++ = 0;
 272
 273         /* number of threads & urb entries for GPGPU vs Media Mode */
 274         *cs++ = threads << 16 | 1 << 8 | mode << 2;
 275
 276         *cs++ = 0;
 277
 278         /* urb entry size & curbe size in 256 bits unit */
 279         *cs++ = urb_size << 16 | curbe_size;
 280
 281         /* scoreboard */
 282         *cs++ = 0;
 283         *cs++ = 0;
 284         *cs++ = 0;
 285         batch_advance(batch, cs);
 286 }
 287
 288 static void
 289 gen7_emit_interface_descriptor_load(struct batch_chunk *batch,
 290                                     const u32 interface_descriptor,
 291                                     unsigned int count)
 292 {
 293         u32 *cs = batch_alloc_items(batch, 8, 4);
 294
 295         *cs++ = MEDIA_INTERFACE_DESCRIPTOR_LOAD | (4 - 2);
 296         *cs++ = 0;
 297         *cs++ = count * 8 * sizeof(*cs);
 298
 299         /*
 300          * interface descriptor address - it is relative to the dynamics base
 301          * address
 302          */
 303         *cs++ = interface_descriptor;
 304         batch_advance(batch, cs);
 305 }
 306
 307 static void
 308 gen7_emit_media_object(struct batch_chunk *batch,
 309                        unsigned int media_object_index)
 310 {
 311         unsigned int x_offset = (media_object_index % 16) * 64;
 312         unsigned int y_offset = (media_object_index / 16) * 16;
 313         unsigned int pkt = 6 + 3;
 314         u32 *cs;
 315
 316         cs = batch_alloc_items(batch, 8, pkt);
 317
 318         *cs++ = MEDIA_OBJECT | (pkt - 2);
 319
 320         /* interface descriptor offset */
 321         *cs++ = 0;
 322
 323         /* without indirect data */
 324         *cs++ = 0;
 325         *cs++ = 0;
 326
 327         /* scoreboard */
 328         *cs++ = 0;
 329         *cs++ = 0;
 330
 331         /* inline */
 332         *cs++ = y_offset << 16 | x_offset;
 333         *cs++ = 0;
 334         *cs++ = GT3_INLINE_DATA_DELAYS;
 335
 336         batch_advance(batch, cs);
 337 }
 338
 339 static void gen7_emit_pipeline_flush(struct batch_chunk *batch)
 340 {
 341         u32 *cs = batch_alloc_items(batch, 0, 4);
 342
 343         *cs++ = GFX_OP_PIPE_CONTROL(4);
 344         *cs++ = PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
 345                 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 346                 PIPE_CONTROL_DC_FLUSH_ENABLE |
 347                 PIPE_CONTROL_CS_STALL;
 348         *cs++ = 0;
 349         *cs++ = 0;
 350
 351         batch_advance(batch, cs);
 352 }
 353
 354 static void gen7_emit_pipeline_invalidate(struct batch_chunk *batch)
 355 {
 356         u32 *cs = batch_alloc_items(batch, 0, 10);
 357
 358         /* ivb: Stall before STATE_CACHE_INVALIDATE */
 359         *cs++ = GFX_OP_PIPE_CONTROL(5);
 360         *cs++ = PIPE_CONTROL_STALL_AT_SCOREBOARD |
 361                 PIPE_CONTROL_CS_STALL;
 362         *cs++ = 0;
 363         *cs++ = 0;
 364         *cs++ = 0;
 365
 366         *cs++ = GFX_OP_PIPE_CONTROL(5);
 367         *cs++ = PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 368         *cs++ = 0;
 369         *cs++ = 0;
 370         *cs++ = 0;
 371
 372         batch_advance(batch, cs);
 373 }
 374
 375 static void emit_batch(struct i915_vma * const vma,
 376                        u32 *start,
 377                        const struct batch_vals *bv)
 378 {
 379         struct drm_i915_private *i915 = vma->vm->i915;
 380         const unsigned int desc_count = 1;
 381         const unsigned int urb_size = 1;
 382         struct batch_chunk cmds, state;
 383         u32 descriptors;
 384         unsigned int i;
 385
 386         batch_init(&cmds, vma, start, 0, bv->state_start);
 387         batch_init(&state, vma, start, bv->state_start, SZ_4K);
 388
 389         descriptors = gen7_fill_interface_descriptor(&state, bv,
 390                                                      IS_HASWELL(i915) ?
 391                                                      &cb_kernel_hsw :
 392                                                      &cb_kernel_ivb,
 393                                                      desc_count);
 394
 395         /* Reset inherited context registers */
 396         gen7_emit_pipeline_flush(&cmds);
 397         gen7_emit_pipeline_invalidate(&cmds);
 398         batch_add(&cmds, MI_LOAD_REGISTER_IMM(2));
 399         batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7));
 400         batch_add(&cmds, 0xffff0000 |
 401                         ((IS_IVB_GT1(i915) || IS_VALLEYVIEW(i915)) ?
 402                          HIZ_RAW_STALL_OPT_DISABLE :
 403                          0));
 404         batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1));
 405         batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
 406         gen7_emit_pipeline_invalidate(&cmds);
 407         gen7_emit_pipeline_flush(&cmds);
 408
 409         /* Switch to the media pipeline and our base address */
 410         gen7_emit_pipeline_invalidate(&cmds);
 411         batch_add(&cmds, PIPELINE_SELECT | PIPELINE_SELECT_MEDIA);
 412         batch_add(&cmds, MI_NOOP);
 413         gen7_emit_pipeline_invalidate(&cmds);
 414
 415         gen7_emit_pipeline_flush(&cmds);
 416         gen7_emit_state_base_address(&cmds, descriptors);
 417         gen7_emit_pipeline_invalidate(&cmds);
 418
 419         /* Set the clear-residual kernel state */
 420         gen7_emit_vfe_state(&cmds, bv, urb_size - 1, 0, 0);
 421         gen7_emit_interface_descriptor_load(&cmds, descriptors, desc_count);
 422
 423         /* Execute the kernel on all HW threads */
 424         for (i = 0; i < num_primitives(bv); i++)
 425                 gen7_emit_media_object(&cmds, i);
 426
 427         batch_add(&cmds, MI_BATCH_BUFFER_END);
 428 }
 429
 430 int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine,
 431                             struct i915_vma * const vma)
 432 {
 433         struct batch_vals bv;
 434         u32 *batch;
 435
 436         batch_get_defaults(engine->i915, &bv);
 437         if (!vma)
 438                 return bv.size;
 439
 440         GEM_BUG_ON(vma->obj->base.size < bv.size);
 441
 442         batch = i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
 443         if (IS_ERR(batch))
 444                 return PTR_ERR(batch);
 445
 446         emit_batch(vma, memset(batch, 0, bv.size), &bv);
 447
 448         i915_gem_object_flush_map(vma->obj);
 449         __i915_gem_object_release_map(vma->obj);
 450
 451         return 0;
 452 }