drivers/gpu/drm/xe/xe_dma_buf.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  */
   5
   6 #include "xe_dma_buf.h"
   7
   8 #include <kunit/test.h>
   9 #include <linux/dma-buf.h>
  10 #include <linux/pci-p2pdma.h>
  11
  12 #include <drm/drm_device.h>
  13 #include <drm/drm_prime.h>
  14 #include <drm/ttm/ttm_tt.h>
  15
  16 #include "tests/xe_test.h"
  17 #include "xe_bo.h"
  18 #include "xe_device.h"
  19 #include "xe_ttm_vram_mgr.h"
  20 #include "xe_vm.h"
  21
  22 MODULE_IMPORT_NS(DMA_BUF);
  23
  24 static int xe_dma_buf_attach(struct dma_buf *dmabuf,
  25                              struct dma_buf_attachment *attach)
  26 {
  27         struct drm_gem_object *obj = attach->dmabuf->priv;
  28
  29         if (attach->peer2peer &&
  30             pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
  31                 attach->peer2peer = false;
  32
  33         if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
  34                 return -EOPNOTSUPP;
  35
  36         xe_device_mem_access_get(to_xe_device(obj->dev));
  37         return 0;
  38 }
  39
  40 static void xe_dma_buf_detach(struct dma_buf *dmabuf,
  41                               struct dma_buf_attachment *attach)
  42 {
  43         struct drm_gem_object *obj = attach->dmabuf->priv;
  44
  45         xe_device_mem_access_put(to_xe_device(obj->dev));
  46 }
  47
  48 static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
  49 {
  50         struct drm_gem_object *obj = attach->dmabuf->priv;
  51         struct xe_bo *bo = gem_to_xe_bo(obj);
  52         struct xe_device *xe = xe_bo_device(bo);
  53         int ret;
  54
  55         /*
  56          * For now only support pinning in TT memory, for two reasons:
  57          * 1) Avoid pinning in a placement not accessible to some importers.
  58          * 2) Pinning in VRAM requires PIN accounting which is a to-do.
  59          */
  60         if (xe_bo_is_pinned(bo) && bo->ttm.resource->placement != XE_PL_TT) {
  61                 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n");
  62                 return -EINVAL;
  63         }
  64
  65         ret = xe_bo_migrate(bo, XE_PL_TT);
  66         if (ret) {
  67                 if (ret != -EINTR && ret != -ERESTARTSYS)
  68                         drm_dbg(&xe->drm,
  69                                 "Failed migrating dma-buf to TT memory: %pe\n",
  70                                 ERR_PTR(ret));
  71                 return ret;
  72         }
  73
  74         ret = xe_bo_pin_external(bo);
  75         xe_assert(xe, !ret);
  76
  77         return 0;
  78 }
  79
  80 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
  81 {
  82         struct drm_gem_object *obj = attach->dmabuf->priv;
  83         struct xe_bo *bo = gem_to_xe_bo(obj);
  84
  85         xe_bo_unpin_external(bo);
  86 }
  87
  88 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
  89                                        enum dma_data_direction dir)
  90 {
  91         struct dma_buf *dma_buf = attach->dmabuf;
  92         struct drm_gem_object *obj = dma_buf->priv;
  93         struct xe_bo *bo = gem_to_xe_bo(obj);
  94         struct sg_table *sgt;
  95         int r = 0;
  96
  97         if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
  98                 return ERR_PTR(-EOPNOTSUPP);
  99
 100         if (!xe_bo_is_pinned(bo)) {
 101                 if (!attach->peer2peer)
 102                         r = xe_bo_migrate(bo, XE_PL_TT);
 103                 else
 104                         r = xe_bo_validate(bo, NULL, false);
 105                 if (r)
 106                         return ERR_PTR(r);
 107         }
 108
 109         switch (bo->ttm.resource->mem_type) {
 110         case XE_PL_TT:
 111                 sgt = drm_prime_pages_to_sg(obj->dev,
 112                                             bo->ttm.ttm->pages,
 113                                             bo->ttm.ttm->num_pages);
 114                 if (IS_ERR(sgt))
 115                         return sgt;
 116
 117                 if (dma_map_sgtable(attach->dev, sgt, dir,
 118                                     DMA_ATTR_SKIP_CPU_SYNC))
 119                         goto error_free;
 120                 break;
 121
 122         case XE_PL_VRAM0:
 123         case XE_PL_VRAM1:
 124                 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo),
 125                                               bo->ttm.resource, 0,
 126                                               bo->ttm.base.size, attach->dev,
 127                                               dir, &sgt);
 128                 if (r)
 129                         return ERR_PTR(r);
 130                 break;
 131         default:
 132                 return ERR_PTR(-EINVAL);
 133         }
 134
 135         return sgt;
 136
 137 error_free:
 138         sg_free_table(sgt);
 139         kfree(sgt);
 140         return ERR_PTR(-EBUSY);
 141 }
 142
 143 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
 144                              struct sg_table *sgt,
 145                              enum dma_data_direction dir)
 146 {
 147         struct dma_buf *dma_buf = attach->dmabuf;
 148         struct xe_bo *bo = gem_to_xe_bo(dma_buf->priv);
 149
 150         if (!xe_bo_is_vram(bo)) {
 151                 dma_unmap_sgtable(attach->dev, sgt, dir, 0);
 152                 sg_free_table(sgt);
 153                 kfree(sgt);
 154         } else {
 155                 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt);
 156         }
 157 }
 158
 159 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
 160                                        enum dma_data_direction direction)
 161 {
 162         struct drm_gem_object *obj = dma_buf->priv;
 163         struct xe_bo *bo = gem_to_xe_bo(obj);
 164         bool reads =  (direction == DMA_BIDIRECTIONAL ||
 165                        direction == DMA_FROM_DEVICE);
 166
 167         if (!reads)
 168                 return 0;
 169
 170         /* Can we do interruptible lock here? */
 171         xe_bo_lock(bo, false);
 172         (void)xe_bo_migrate(bo, XE_PL_TT);
 173         xe_bo_unlock(bo);
 174
 175         return 0;
 176 }
 177
 178 const struct dma_buf_ops xe_dmabuf_ops = {
 179         .attach = xe_dma_buf_attach,
 180         .detach = xe_dma_buf_detach,
 181         .pin = xe_dma_buf_pin,
 182         .unpin = xe_dma_buf_unpin,
 183         .map_dma_buf = xe_dma_buf_map,
 184         .unmap_dma_buf = xe_dma_buf_unmap,
 185         .release = drm_gem_dmabuf_release,
 186         .begin_cpu_access = xe_dma_buf_begin_cpu_access,
 187         .mmap = drm_gem_dmabuf_mmap,
 188         .vmap = drm_gem_dmabuf_vmap,
 189         .vunmap = drm_gem_dmabuf_vunmap,
 190 };
 191
 192 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
 193 {
 194         struct xe_bo *bo = gem_to_xe_bo(obj);
 195         struct dma_buf *buf;
 196
 197         if (bo->vm)
 198                 return ERR_PTR(-EPERM);
 199
 200         buf = drm_gem_prime_export(obj, flags);
 201         if (!IS_ERR(buf))
 202                 buf->ops = &xe_dmabuf_ops;
 203
 204         return buf;
 205 }
 206
 207 static struct drm_gem_object *
 208 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
 209                     struct dma_buf *dma_buf)
 210 {
 211         struct dma_resv *resv = dma_buf->resv;
 212         struct xe_device *xe = to_xe_device(dev);
 213         struct xe_bo *bo;
 214         int ret;
 215
 216         dma_resv_lock(resv, NULL);
 217         bo = ___xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
 218                                     0, /* Will require 1way or 2way for vm_bind */
 219                                     ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT);
 220         if (IS_ERR(bo)) {
 221                 ret = PTR_ERR(bo);
 222                 goto error;
 223         }
 224         dma_resv_unlock(resv);
 225
 226         return &bo->ttm.base;
 227
 228 error:
 229         dma_resv_unlock(resv);
 230         return ERR_PTR(ret);
 231 }
 232
 233 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
 234 {
 235         struct drm_gem_object *obj = attach->importer_priv;
 236         struct xe_bo *bo = gem_to_xe_bo(obj);
 237
 238         XE_WARN_ON(xe_bo_evict(bo, false));
 239 }
 240
 241 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
 242         .allow_peer2peer = true,
 243         .move_notify = xe_dma_buf_move_notify
 244 };
 245
 246 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 247
 248 struct dma_buf_test_params {
 249         struct xe_test_priv base;
 250         const struct dma_buf_attach_ops *attach_ops;
 251         bool force_different_devices;
 252         u32 mem_mask;
 253 };
 254
 255 #define to_dma_buf_test_params(_priv) \
 256         container_of(_priv, struct dma_buf_test_params, base)
 257 #endif
 258
 259 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
 260                                            struct dma_buf *dma_buf)
 261 {
 262         XE_TEST_DECLARE(struct dma_buf_test_params *test =
 263                         to_dma_buf_test_params
 264                         (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
 265         const struct dma_buf_attach_ops *attach_ops;
 266         struct dma_buf_attachment *attach;
 267         struct drm_gem_object *obj;
 268         struct xe_bo *bo;
 269
 270         if (dma_buf->ops == &xe_dmabuf_ops) {
 271                 obj = dma_buf->priv;
 272                 if (obj->dev == dev &&
 273                     !XE_TEST_ONLY(test && test->force_different_devices)) {
 274                         /*
 275                          * Importing dmabuf exported from out own gem increases
 276                          * refcount on gem itself instead of f_count of dmabuf.
 277                          */
 278                         drm_gem_object_get(obj);
 279                         return obj;
 280                 }
 281         }
 282
 283         /*
 284          * Don't publish the bo until we have a valid attachment, and a
 285          * valid attachment needs the bo address. So pre-create a bo before
 286          * creating the attachment and publish.
 287          */
 288         bo = xe_bo_alloc();
 289         if (IS_ERR(bo))
 290                 return ERR_CAST(bo);
 291
 292         attach_ops = &xe_dma_buf_attach_ops;
 293 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 294         if (test)
 295                 attach_ops = test->attach_ops;
 296 #endif
 297
 298         attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
 299         if (IS_ERR(attach)) {
 300                 obj = ERR_CAST(attach);
 301                 goto out_err;
 302         }
 303
 304         /* Errors here will take care of freeing the bo. */
 305         obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
 306         if (IS_ERR(obj))
 307                 return obj;
 308
 309
 310         get_dma_buf(dma_buf);
 311         obj->import_attach = attach;
 312         return obj;
 313
 314 out_err:
 315         xe_bo_free(bo);
 316
 317         return obj;
 318 }
 319
 320 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 321 #include "tests/xe_dma_buf.c"
 322 #endif