drivers/gpu/drm/i915/gem/i915_gem_client_blt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "gt/intel_context.h"
   8 #include "gt/intel_engine_pm.h"
   9 #include "i915_gem_client_blt.h"
  10 #include "i915_gem_object_blt.h"
  11
  12 struct i915_sleeve {
  13         struct i915_vma *vma;
  14         struct drm_i915_gem_object *obj;
  15         struct sg_table *pages;
  16         struct i915_page_sizes page_sizes;
  17 };
  18
  19 static int vma_set_pages(struct i915_vma *vma)
  20 {
  21         struct i915_sleeve *sleeve = vma->private;
  22
  23         vma->pages = sleeve->pages;
  24         vma->page_sizes = sleeve->page_sizes;
  25
  26         return 0;
  27 }
  28
  29 static void vma_clear_pages(struct i915_vma *vma)
  30 {
  31         GEM_BUG_ON(!vma->pages);
  32         vma->pages = NULL;
  33 }
  34
  35 static void vma_bind(struct i915_address_space *vm,
  36                      struct i915_vm_pt_stash *stash,
  37                      struct i915_vma *vma,
  38                      enum i915_cache_level cache_level,
  39                      u32 flags)
  40 {
  41         vm->vma_ops.bind_vma(vm, stash, vma, cache_level, flags);
  42 }
  43
  44 static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma)
  45 {
  46         vm->vma_ops.unbind_vma(vm, vma);
  47 }
  48
  49 static const struct i915_vma_ops proxy_vma_ops = {
  50         .set_pages = vma_set_pages,
  51         .clear_pages = vma_clear_pages,
  52         .bind_vma = vma_bind,
  53         .unbind_vma = vma_unbind,
  54 };
  55
  56 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
  57                                          struct drm_i915_gem_object *obj,
  58                                          struct sg_table *pages,
  59                                          struct i915_page_sizes *page_sizes)
  60 {
  61         struct i915_sleeve *sleeve;
  62         struct i915_vma *vma;
  63         int err;
  64
  65         sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
  66         if (!sleeve)
  67                 return ERR_PTR(-ENOMEM);
  68
  69         vma = i915_vma_instance(obj, vm, NULL);
  70         if (IS_ERR(vma)) {
  71                 err = PTR_ERR(vma);
  72                 goto err_free;
  73         }
  74
  75         vma->private = sleeve;
  76         vma->ops = &proxy_vma_ops;
  77
  78         sleeve->vma = vma;
  79         sleeve->pages = pages;
  80         sleeve->page_sizes = *page_sizes;
  81
  82         return sleeve;
  83
  84 err_free:
  85         kfree(sleeve);
  86         return ERR_PTR(err);
  87 }
  88
  89 static void destroy_sleeve(struct i915_sleeve *sleeve)
  90 {
  91         kfree(sleeve);
  92 }
  93
  94 struct clear_pages_work {
  95         struct dma_fence dma;
  96         struct dma_fence_cb cb;
  97         struct i915_sw_fence wait;
  98         struct work_struct work;
  99         struct irq_work irq_work;
 100         struct i915_sleeve *sleeve;
 101         struct intel_context *ce;
 102         u32 value;
 103 };
 104
 105 static const char *clear_pages_work_driver_name(struct dma_fence *fence)
 106 {
 107         return DRIVER_NAME;
 108 }
 109
 110 static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
 111 {
 112         return "clear";
 113 }
 114
 115 static void clear_pages_work_release(struct dma_fence *fence)
 116 {
 117         struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
 118
 119         destroy_sleeve(w->sleeve);
 120
 121         i915_sw_fence_fini(&w->wait);
 122
 123         BUILD_BUG_ON(offsetof(typeof(*w), dma));
 124         dma_fence_free(&w->dma);
 125 }
 126
 127 static const struct dma_fence_ops clear_pages_work_ops = {
 128         .get_driver_name = clear_pages_work_driver_name,
 129         .get_timeline_name = clear_pages_work_timeline_name,
 130         .release = clear_pages_work_release,
 131 };
 132
 133 static void clear_pages_signal_irq_worker(struct irq_work *work)
 134 {
 135         struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
 136
 137         dma_fence_signal(&w->dma);
 138         dma_fence_put(&w->dma);
 139 }
 140
 141 static void clear_pages_dma_fence_cb(struct dma_fence *fence,
 142                                      struct dma_fence_cb *cb)
 143 {
 144         struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
 145
 146         if (fence->error)
 147                 dma_fence_set_error(&w->dma, fence->error);
 148
 149         /*
 150          * Push the signalling of the fence into yet another worker to avoid
 151          * the nightmare locking around the fence spinlock.
 152          */
 153         irq_work_queue(&w->irq_work);
 154 }
 155
 156 static void clear_pages_worker(struct work_struct *work)
 157 {
 158         struct clear_pages_work *w = container_of(work, typeof(*w), work);
 159         struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
 160         struct i915_vma *vma = w->sleeve->vma;
 161         struct i915_request *rq;
 162         struct i915_vma *batch;
 163         int err = w->dma.error;
 164
 165         if (unlikely(err))
 166                 goto out_signal;
 167
 168         if (obj->cache_dirty) {
 169                 if (i915_gem_object_has_struct_page(obj))
 170                         drm_clflush_sg(w->sleeve->pages);
 171                 obj->cache_dirty = false;
 172         }
 173         obj->read_domains = I915_GEM_GPU_DOMAINS;
 174         obj->write_domain = 0;
 175
 176         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 177         if (unlikely(err))
 178                 goto out_signal;
 179
 180         batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
 181         if (IS_ERR(batch)) {
 182                 err = PTR_ERR(batch);
 183                 goto out_unpin;
 184         }
 185
 186         rq = intel_context_create_request(w->ce);
 187         if (IS_ERR(rq)) {
 188                 err = PTR_ERR(rq);
 189                 goto out_batch;
 190         }
 191
 192         /* There's no way the fence has signalled */
 193         if (dma_fence_add_callback(&rq->fence, &w->cb,
 194                                    clear_pages_dma_fence_cb))
 195                 GEM_BUG_ON(1);
 196
 197         err = intel_emit_vma_mark_active(batch, rq);
 198         if (unlikely(err))
 199                 goto out_request;
 200
 201         if (w->ce->engine->emit_init_breadcrumb) {
 202                 err = w->ce->engine->emit_init_breadcrumb(rq);
 203                 if (unlikely(err))
 204                         goto out_request;
 205         }
 206
 207         /*
 208          * w->dma is already exported via (vma|obj)->resv we need only
 209          * keep track of the GPU activity within this vma/request, and
 210          * propagate the signal from the request to w->dma.
 211          */
 212         err = __i915_vma_move_to_active(vma, rq);
 213         if (err)
 214                 goto out_request;
 215
 216         err = w->ce->engine->emit_bb_start(rq,
 217                                            batch->node.start, batch->node.size,
 218                                            0);
 219 out_request:
 220         if (unlikely(err)) {
 221                 i915_request_set_error_once(rq, err);
 222                 err = 0;
 223         }
 224
 225         i915_request_add(rq);
 226 out_batch:
 227         intel_emit_vma_release(w->ce, batch);
 228 out_unpin:
 229         i915_vma_unpin(vma);
 230 out_signal:
 231         if (unlikely(err)) {
 232                 dma_fence_set_error(&w->dma, err);
 233                 dma_fence_signal(&w->dma);
 234                 dma_fence_put(&w->dma);
 235         }
 236 }
 237
 238 static int __i915_sw_fence_call
 239 clear_pages_work_notify(struct i915_sw_fence *fence,
 240                         enum i915_sw_fence_notify state)
 241 {
 242         struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
 243
 244         switch (state) {
 245         case FENCE_COMPLETE:
 246                 schedule_work(&w->work);
 247                 break;
 248
 249         case FENCE_FREE:
 250                 dma_fence_put(&w->dma);
 251                 break;
 252         }
 253
 254         return NOTIFY_DONE;
 255 }
 256
 257 static DEFINE_SPINLOCK(fence_lock);
 258
 259 /* XXX: better name please */
 260 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
 261                                      struct intel_context *ce,
 262                                      struct sg_table *pages,
 263                                      struct i915_page_sizes *page_sizes,
 264                                      u32 value)
 265 {
 266         struct clear_pages_work *work;
 267         struct i915_sleeve *sleeve;
 268         int err;
 269
 270         sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
 271         if (IS_ERR(sleeve))
 272                 return PTR_ERR(sleeve);
 273
 274         work = kmalloc(sizeof(*work), GFP_KERNEL);
 275         if (!work) {
 276                 destroy_sleeve(sleeve);
 277                 return -ENOMEM;
 278         }
 279
 280         work->value = value;
 281         work->sleeve = sleeve;
 282         work->ce = ce;
 283
 284         INIT_WORK(&work->work, clear_pages_worker);
 285
 286         init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
 287
 288         dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
 289         i915_sw_fence_init(&work->wait, clear_pages_work_notify);
 290
 291         i915_gem_object_lock(obj);
 292         err = i915_sw_fence_await_reservation(&work->wait,
 293                                               obj->base.resv, NULL, true, 0,
 294                                               I915_FENCE_GFP);
 295         if (err < 0) {
 296                 dma_fence_set_error(&work->dma, err);
 297         } else {
 298                 dma_resv_add_excl_fence(obj->base.resv, &work->dma);
 299                 err = 0;
 300         }
 301         i915_gem_object_unlock(obj);
 302
 303         dma_fence_get(&work->dma);
 304         i915_sw_fence_commit(&work->wait);
 305
 306         return err;
 307 }
 308
 309 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 310 #include "selftests/i915_gem_client_blt.c"
 311 #endif