1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
4 * Copyright 2014-2023 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Treat context OTables as resources to make use of the resource
29 * backing MOB eviction mechanism, that is used to read back the COTable
30 * whenever the backing MOB is evicted.
33 #include "vmwgfx_bo.h"
34 #include "vmwgfx_drv.h"
35 #include "vmwgfx_mksstat.h"
36 #include "vmwgfx_resource_priv.h"
37 #include "vmwgfx_so.h"
39 #include <drm/ttm/ttm_placement.h>
42 * struct vmw_cotable - Context Object Table resource
44 * @res: struct vmw_resource we are deriving from.
45 * @ctx: non-refcounted pointer to the owning context.
46 * @size_read_back: Size of data read back during eviction.
47 * @seen_entries: Seen entries in command stream for this cotable.
48 * @type: The cotable type.
49 * @scrubbed: Whether the cotable has been scrubbed.
50 * @resource_list: List of resources in the cotable.
53 struct vmw_resource res;
54 struct vmw_resource *ctx;
55 size_t size_read_back;
59 struct list_head resource_list;
63 * struct vmw_cotable_info - Static info about cotable types
65 * @min_initial_entries: Min number of initial intries at cotable allocation
66 * for this cotable type.
67 * @size: Size of each entry.
68 * @unbind_func: Unbind call-back function.
70 struct vmw_cotable_info {
71 u32 min_initial_entries;
73 void (*unbind_func)(struct vmw_private *, struct list_head *,
79 * Getting the initial size right is difficult because it all depends
80 * on what the userspace is doing. The sizes will be aligned up to
81 * a PAGE_SIZE so we just want to make sure that for majority of apps
82 * the initial number of entries doesn't require an immediate resize.
83 * For all cotables except SVGACOTableDXElementLayoutEntry and
84 * SVGACOTableDXBlendStateEntry the initial number of entries fits
85 * within the PAGE_SIZE. For SVGACOTableDXElementLayoutEntry and
86 * SVGACOTableDXBlendStateEntry we want to reserve two pages,
87 * because that's what all apps will require initially.
89 static const struct vmw_cotable_info co_info[] = {
90 {1, sizeof(SVGACOTableDXRTViewEntry), &vmw_view_cotable_list_destroy},
91 {1, sizeof(SVGACOTableDXDSViewEntry), &vmw_view_cotable_list_destroy},
92 {1, sizeof(SVGACOTableDXSRViewEntry), &vmw_view_cotable_list_destroy},
93 {PAGE_SIZE/sizeof(SVGACOTableDXElementLayoutEntry) + 1, sizeof(SVGACOTableDXElementLayoutEntry), NULL},
94 {PAGE_SIZE/sizeof(SVGACOTableDXBlendStateEntry) + 1, sizeof(SVGACOTableDXBlendStateEntry), NULL},
95 {1, sizeof(SVGACOTableDXDepthStencilEntry), NULL},
96 {1, sizeof(SVGACOTableDXRasterizerStateEntry), NULL},
97 {1, sizeof(SVGACOTableDXSamplerEntry), NULL},
98 {1, sizeof(SVGACOTableDXStreamOutputEntry), &vmw_dx_streamoutput_cotable_list_scrub},
99 {1, sizeof(SVGACOTableDXQueryEntry), NULL},
100 {1, sizeof(SVGACOTableDXShaderEntry), &vmw_dx_shader_cotable_list_scrub},
101 {1, sizeof(SVGACOTableDXUAViewEntry), &vmw_view_cotable_list_destroy}
105 * Cotables with bindings that we remove must be scrubbed first,
106 * otherwise, the device will swap in an invalid context when we remove
107 * bindings before scrubbing a cotable...
109 const SVGACOTableType vmw_cotable_scrub_order[] = {
113 SVGA_COTABLE_DXSHADER,
114 SVGA_COTABLE_ELEMENTLAYOUT,
115 SVGA_COTABLE_BLENDSTATE,
116 SVGA_COTABLE_DEPTHSTENCIL,
117 SVGA_COTABLE_RASTERIZERSTATE,
118 SVGA_COTABLE_SAMPLER,
119 SVGA_COTABLE_STREAMOUTPUT,
120 SVGA_COTABLE_DXQUERY,
124 static int vmw_cotable_bind(struct vmw_resource *res,
125 struct ttm_validate_buffer *val_buf);
126 static int vmw_cotable_unbind(struct vmw_resource *res,
128 struct ttm_validate_buffer *val_buf);
129 static int vmw_cotable_create(struct vmw_resource *res);
130 static int vmw_cotable_destroy(struct vmw_resource *res);
132 static const struct vmw_res_func vmw_cotable_func = {
133 .res_type = vmw_res_cotable,
134 .needs_backup = true,
138 .type_name = "context guest backed object tables",
139 .backup_placement = &vmw_mob_placement,
140 .create = vmw_cotable_create,
141 .destroy = vmw_cotable_destroy,
142 .bind = vmw_cotable_bind,
143 .unbind = vmw_cotable_unbind,
147 * vmw_cotable - Convert a struct vmw_resource pointer to a struct
148 * vmw_cotable pointer
150 * @res: Pointer to the resource.
152 static struct vmw_cotable *vmw_cotable(struct vmw_resource *res)
154 return container_of(res, struct vmw_cotable, res);
158 * vmw_cotable_destroy - Cotable resource destroy callback
160 * @res: Pointer to the cotable resource.
162 * There is no device cotable destroy command, so this function only
163 * makes sure that the resource id is set to invalid.
165 static int vmw_cotable_destroy(struct vmw_resource *res)
172 * vmw_cotable_unscrub - Undo a cotable unscrub operation
174 * @res: Pointer to the cotable resource
176 * This function issues commands to (re)bind the cotable to
177 * its backing mob, which needs to be validated and reserved at this point.
178 * This is identical to bind() except the function interface looks different.
180 static int vmw_cotable_unscrub(struct vmw_resource *res)
182 struct vmw_cotable *vcotbl = vmw_cotable(res);
183 struct vmw_private *dev_priv = res->dev_priv;
184 struct ttm_buffer_object *bo = &res->backup->base;
186 SVGA3dCmdHeader header;
187 SVGA3dCmdDXSetCOTable body;
190 WARN_ON_ONCE(bo->resource->mem_type != VMW_PL_MOB);
191 dma_resv_assert_held(bo->base.resv);
193 cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
197 WARN_ON(vcotbl->ctx->id == SVGA3D_INVALID_ID);
198 WARN_ON(bo->resource->mem_type != VMW_PL_MOB);
199 cmd->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
200 cmd->header.size = sizeof(cmd->body);
201 cmd->body.cid = vcotbl->ctx->id;
202 cmd->body.type = vcotbl->type;
203 cmd->body.mobid = bo->resource->start;
204 cmd->body.validSizeInBytes = vcotbl->size_read_back;
206 vmw_cmd_commit_flush(dev_priv, sizeof(*cmd));
207 vcotbl->scrubbed = false;
213 * vmw_cotable_bind - Undo a cotable unscrub operation
215 * @res: Pointer to the cotable resource
216 * @val_buf: Pointer to a struct ttm_validate_buffer prepared by the caller
217 * for convenience / fencing.
219 * This function issues commands to (re)bind the cotable to
220 * its backing mob, which needs to be validated and reserved at this point.
222 static int vmw_cotable_bind(struct vmw_resource *res,
223 struct ttm_validate_buffer *val_buf)
226 * The create() callback may have changed @res->backup without
227 * the caller noticing, and with val_buf->bo still pointing to
228 * the old backup buffer. Although hackish, and not used currently,
229 * take the opportunity to correct the value here so that it's not
230 * misused in the future.
232 val_buf->bo = &res->backup->base;
234 return vmw_cotable_unscrub(res);
238 * vmw_cotable_scrub - Scrub the cotable from the device.
240 * @res: Pointer to the cotable resource.
241 * @readback: Whether initiate a readback of the cotable data to the backup
244 * In some situations (context swapouts) it might be desirable to make the
245 * device forget about the cotable without performing a full unbind. A full
246 * unbind requires reserved backup buffers and it might not be possible to
247 * reserve them due to locking order violation issues. The vmw_cotable_scrub
248 * function implements a partial unbind() without that requirement but with the
249 * following restrictions.
250 * 1) Before the cotable is again used by the GPU, vmw_cotable_unscrub() must
252 * 2) Before the cotable backing buffer is used by the CPU, or during the
253 * resource destruction, vmw_cotable_unbind() must be called.
255 int vmw_cotable_scrub(struct vmw_resource *res, bool readback)
257 struct vmw_cotable *vcotbl = vmw_cotable(res);
258 struct vmw_private *dev_priv = res->dev_priv;
262 SVGA3dCmdHeader header;
263 SVGA3dCmdDXReadbackCOTable body;
266 SVGA3dCmdHeader header;
267 SVGA3dCmdDXSetCOTable body;
270 if (vcotbl->scrubbed)
273 if (co_info[vcotbl->type].unbind_func)
274 co_info[vcotbl->type].unbind_func(dev_priv,
275 &vcotbl->resource_list,
277 submit_size = sizeof(*cmd1);
279 submit_size += sizeof(*cmd0);
281 cmd1 = VMW_CMD_RESERVE(dev_priv, submit_size);
285 vcotbl->size_read_back = 0;
287 cmd0 = (void *) cmd1;
288 cmd0->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
289 cmd0->header.size = sizeof(cmd0->body);
290 cmd0->body.cid = vcotbl->ctx->id;
291 cmd0->body.type = vcotbl->type;
292 cmd1 = (void *) &cmd0[1];
293 vcotbl->size_read_back = res->backup_size;
295 cmd1->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
296 cmd1->header.size = sizeof(cmd1->body);
297 cmd1->body.cid = vcotbl->ctx->id;
298 cmd1->body.type = vcotbl->type;
299 cmd1->body.mobid = SVGA3D_INVALID_ID;
300 cmd1->body.validSizeInBytes = 0;
301 vmw_cmd_commit_flush(dev_priv, submit_size);
302 vcotbl->scrubbed = true;
304 /* Trigger a create() on next validate. */
311 * vmw_cotable_unbind - Cotable resource unbind callback
313 * @res: Pointer to the cotable resource.
314 * @readback: Whether to read back cotable data to the backup buffer.
315 * @val_buf: Pointer to a struct ttm_validate_buffer prepared by the caller
316 * for convenience / fencing.
318 * Unbinds the cotable from the device and fences the backup buffer.
320 static int vmw_cotable_unbind(struct vmw_resource *res,
322 struct ttm_validate_buffer *val_buf)
324 struct vmw_cotable *vcotbl = vmw_cotable(res);
325 struct vmw_private *dev_priv = res->dev_priv;
326 struct ttm_buffer_object *bo = val_buf->bo;
327 struct vmw_fence_obj *fence;
329 if (!vmw_resource_mob_attached(res))
332 WARN_ON_ONCE(bo->resource->mem_type != VMW_PL_MOB);
333 dma_resv_assert_held(bo->base.resv);
335 mutex_lock(&dev_priv->binding_mutex);
336 if (!vcotbl->scrubbed)
337 vmw_dx_context_scrub_cotables(vcotbl->ctx, readback);
338 mutex_unlock(&dev_priv->binding_mutex);
339 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
340 vmw_bo_fence_single(bo, fence);
341 if (likely(fence != NULL))
342 vmw_fence_obj_unreference(&fence);
348 * vmw_cotable_readback - Read back a cotable without unbinding.
350 * @res: The cotable resource.
352 * Reads back a cotable to its backing mob without scrubbing the MOB from
353 * the cotable. The MOB is fenced for subsequent CPU access.
355 static int vmw_cotable_readback(struct vmw_resource *res)
357 struct vmw_cotable *vcotbl = vmw_cotable(res);
358 struct vmw_private *dev_priv = res->dev_priv;
361 SVGA3dCmdHeader header;
362 SVGA3dCmdDXReadbackCOTable body;
364 struct vmw_fence_obj *fence;
366 if (!vcotbl->scrubbed) {
367 cmd = VMW_CMD_RESERVE(dev_priv, sizeof(*cmd));
371 cmd->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
372 cmd->header.size = sizeof(cmd->body);
373 cmd->body.cid = vcotbl->ctx->id;
374 cmd->body.type = vcotbl->type;
375 vcotbl->size_read_back = res->backup_size;
376 vmw_cmd_commit(dev_priv, sizeof(*cmd));
379 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
380 vmw_bo_fence_single(&res->backup->base, fence);
381 vmw_fence_obj_unreference(&fence);
387 * vmw_cotable_resize - Resize a cotable.
389 * @res: The cotable resource.
390 * @new_size: The new size.
392 * Resizes a cotable and binds the new backup buffer.
393 * On failure the cotable is left intact.
394 * Important! This function may not fail once the MOB switch has been
395 * committed to hardware. That would put the device context in an
396 * invalid state which we can't currently recover from.
398 static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
400 struct ttm_operation_ctx ctx = { false, false };
401 struct vmw_private *dev_priv = res->dev_priv;
402 struct vmw_cotable *vcotbl = vmw_cotable(res);
403 struct vmw_bo *buf, *old_buf = res->backup;
404 struct ttm_buffer_object *bo, *old_bo = &res->backup->base;
405 size_t old_size = res->backup_size;
406 size_t old_size_read_back = vcotbl->size_read_back;
407 size_t cur_size_read_back;
408 struct ttm_bo_kmap_obj old_map, new_map;
412 MKS_STAT_TIME_DECL(MKSSTAT_KERN_COTABLE_RESIZE);
413 MKS_STAT_TIME_PUSH(MKSSTAT_KERN_COTABLE_RESIZE);
415 ret = vmw_cotable_readback(res);
419 cur_size_read_back = vcotbl->size_read_back;
420 vcotbl->size_read_back = old_size_read_back;
423 * While device is processing, Allocate and reserve a buffer object
424 * for the new COTable. Initially pin the buffer object to make sure
425 * we can use tryreserve without failure.
427 ret = vmw_bo_create(dev_priv, new_size, &vmw_mob_placement,
430 DRM_ERROR("Failed initializing new cotable MOB.\n");
435 WARN_ON_ONCE(ttm_bo_reserve(bo, false, true, NULL));
437 ret = ttm_bo_wait(old_bo, false, false);
438 if (unlikely(ret != 0)) {
439 DRM_ERROR("Failed waiting for cotable unbind.\n");
444 * Do a page by page copy of COTables. This eliminates slow vmap()s.
445 * This should really be a TTM utility.
447 for (i = 0; i < PFN_UP(old_bo->resource->size); ++i) {
450 ret = ttm_bo_kmap(old_bo, i, 1, &old_map);
451 if (unlikely(ret != 0)) {
452 DRM_ERROR("Failed mapping old COTable on resize.\n");
455 ret = ttm_bo_kmap(bo, i, 1, &new_map);
456 if (unlikely(ret != 0)) {
457 DRM_ERROR("Failed mapping new COTable on resize.\n");
460 memcpy(ttm_kmap_obj_virtual(&new_map, &dummy),
461 ttm_kmap_obj_virtual(&old_map, &dummy),
463 ttm_bo_kunmap(&new_map);
464 ttm_bo_kunmap(&old_map);
467 /* Unpin new buffer, and switch backup buffers. */
468 ret = ttm_bo_validate(bo, &vmw_mob_placement, &ctx);
469 if (unlikely(ret != 0)) {
470 DRM_ERROR("Failed validating new COTable backup buffer.\n");
474 vmw_resource_mob_detach(res);
476 res->backup_size = new_size;
477 vcotbl->size_read_back = cur_size_read_back;
480 * Now tell the device to switch. If this fails, then we need to
481 * revert the full resize.
483 ret = vmw_cotable_unscrub(res);
485 DRM_ERROR("Failed switching COTable backup buffer.\n");
486 res->backup = old_buf;
487 res->backup_size = old_size;
488 vcotbl->size_read_back = old_size_read_back;
489 vmw_resource_mob_attach(res);
493 vmw_resource_mob_attach(res);
494 /* Let go of the old mob. */
495 vmw_bo_unreference(&old_buf);
496 res->id = vcotbl->type;
498 ret = dma_resv_reserve_fences(bo->base.resv, 1);
502 /* Release the pin acquired in vmw_bo_init */
505 MKS_STAT_TIME_POP(MKSSTAT_KERN_COTABLE_RESIZE);
510 ttm_bo_kunmap(&old_map);
513 ttm_bo_unreserve(bo);
514 vmw_bo_unreference(&buf);
517 MKS_STAT_TIME_POP(MKSSTAT_KERN_COTABLE_RESIZE);
523 * vmw_cotable_create - Cotable resource create callback
525 * @res: Pointer to a cotable resource.
527 * There is no separate create command for cotables, so this callback, which
528 * is called before bind() in the validation sequence is instead used for two
530 * 1) Unscrub the cotable if it is scrubbed and still attached to a backup
532 * 2) Resize the cotable if needed.
534 static int vmw_cotable_create(struct vmw_resource *res)
536 struct vmw_cotable *vcotbl = vmw_cotable(res);
537 size_t new_size = res->backup_size;
541 /* Check whether we need to resize the cotable */
542 needed_size = (vcotbl->seen_entries + 1) * co_info[vcotbl->type].size;
543 while (needed_size > new_size)
546 if (likely(new_size <= res->backup_size)) {
547 if (vcotbl->scrubbed && vmw_resource_mob_attached(res)) {
548 ret = vmw_cotable_unscrub(res);
552 res->id = vcotbl->type;
556 return vmw_cotable_resize(res, new_size);
560 * vmw_hw_cotable_destroy - Cotable hw_destroy callback
562 * @res: Pointer to a cotable resource.
564 * The final (part of resource destruction) destroy callback.
566 static void vmw_hw_cotable_destroy(struct vmw_resource *res)
568 (void) vmw_cotable_destroy(res);
572 * vmw_cotable_free - Cotable resource destructor
574 * @res: Pointer to a cotable resource.
576 static void vmw_cotable_free(struct vmw_resource *res)
582 * vmw_cotable_alloc - Create a cotable resource
584 * @dev_priv: Pointer to a device private struct.
585 * @ctx: Pointer to the context resource.
586 * The cotable resource will not add a refcount.
587 * @type: The cotable type.
589 struct vmw_resource *vmw_cotable_alloc(struct vmw_private *dev_priv,
590 struct vmw_resource *ctx,
593 struct vmw_cotable *vcotbl;
597 vcotbl = kzalloc(sizeof(*vcotbl), GFP_KERNEL);
598 if (unlikely(!vcotbl)) {
603 ret = vmw_resource_init(dev_priv, &vcotbl->res, true,
604 vmw_cotable_free, &vmw_cotable_func);
605 if (unlikely(ret != 0))
608 INIT_LIST_HEAD(&vcotbl->resource_list);
609 vcotbl->res.id = type;
610 vcotbl->res.backup_size = PAGE_SIZE;
611 num_entries = PAGE_SIZE / co_info[type].size;
612 if (num_entries < co_info[type].min_initial_entries) {
613 vcotbl->res.backup_size = co_info[type].min_initial_entries *
615 vcotbl->res.backup_size = PFN_ALIGN(vcotbl->res.backup_size);
618 vcotbl->scrubbed = true;
619 vcotbl->seen_entries = -1;
623 vcotbl->res.hw_destroy = vmw_hw_cotable_destroy;
634 * vmw_cotable_notify - Notify the cotable about an item creation
636 * @res: Pointer to a cotable resource.
639 int vmw_cotable_notify(struct vmw_resource *res, int id)
641 struct vmw_cotable *vcotbl = vmw_cotable(res);
643 if (id < 0 || id >= SVGA_COTABLE_MAX_IDS) {
644 DRM_ERROR("Illegal COTable id. Type is %u. Id is %d\n",
645 (unsigned) vcotbl->type, id);
649 if (vcotbl->seen_entries < id) {
650 /* Trigger a call to create() on next validate */
652 vcotbl->seen_entries = id;
659 * vmw_cotable_add_resource - add a view to the cotable's list of active views.
661 * @res: pointer struct vmw_resource representing the cotable.
662 * @head: pointer to the struct list_head member of the resource, dedicated
663 * to the cotable active resource list.
665 void vmw_cotable_add_resource(struct vmw_resource *res, struct list_head *head)
667 struct vmw_cotable *vcotbl =
668 container_of(res, struct vmw_cotable, res);
670 list_add_tail(head, &vcotbl->resource_list);