drivers/infiniband/core/umem_dmabuf.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
   2 /*
   3  * Copyright (c) 2020 Intel Corporation. All rights reserved.
   4  */
   5
   6 #include <linux/dma-buf.h>
   7 #include <linux/dma-resv.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/module.h>
  10
  11 #include "uverbs.h"
  12
  13 MODULE_IMPORT_NS(DMA_BUF);
  14
  15 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
  16 {
  17         struct sg_table *sgt;
  18         struct scatterlist *sg;
  19         struct dma_fence *fence;
  20         unsigned long start, end, cur = 0;
  21         unsigned int nmap = 0;
  22         int i;
  23
  24         dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
  25
  26         if (umem_dmabuf->sgt)
  27                 goto wait_fence;
  28
  29         sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL);
  30         if (IS_ERR(sgt))
  31                 return PTR_ERR(sgt);
  32
  33         /* modify the sg list in-place to match umem address and length */
  34
  35         start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE);
  36         end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length,
  37                     PAGE_SIZE);
  38         for_each_sgtable_dma_sg(sgt, sg, i) {
  39                 if (start < cur + sg_dma_len(sg) && cur < end)
  40                         nmap++;
  41                 if (cur <= start && start < cur + sg_dma_len(sg)) {
  42                         unsigned long offset = start - cur;
  43
  44                         umem_dmabuf->first_sg = sg;
  45                         umem_dmabuf->first_sg_offset = offset;
  46                         sg_dma_address(sg) += offset;
  47                         sg_dma_len(sg) -= offset;
  48                         cur += offset;
  49                 }
  50                 if (cur < end && end <= cur + sg_dma_len(sg)) {
  51                         unsigned long trim = cur + sg_dma_len(sg) - end;
  52
  53                         umem_dmabuf->last_sg = sg;
  54                         umem_dmabuf->last_sg_trim = trim;
  55                         sg_dma_len(sg) -= trim;
  56                         break;
  57                 }
  58                 cur += sg_dma_len(sg);
  59         }
  60
  61         umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg;
  62         umem_dmabuf->umem.sgt_append.sgt.nents = nmap;
  63         umem_dmabuf->sgt = sgt;
  64
  65 wait_fence:
  66         /*
  67          * Although the sg list is valid now, the content of the pages
  68          * may be not up-to-date. Wait for the exporter to finish
  69          * the migration.
  70          */
  71         fence = dma_resv_excl_fence(umem_dmabuf->attach->dmabuf->resv);
  72         if (fence)
  73                 return dma_fence_wait(fence, false);
  74
  75         return 0;
  76 }
  77 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);
  78
  79 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)
  80 {
  81         dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
  82
  83         if (!umem_dmabuf->sgt)
  84                 return;
  85
  86         /* retore the original sg list */
  87         if (umem_dmabuf->first_sg) {
  88                 sg_dma_address(umem_dmabuf->first_sg) -=
  89                         umem_dmabuf->first_sg_offset;
  90                 sg_dma_len(umem_dmabuf->first_sg) +=
  91                         umem_dmabuf->first_sg_offset;
  92                 umem_dmabuf->first_sg = NULL;
  93                 umem_dmabuf->first_sg_offset = 0;
  94         }
  95         if (umem_dmabuf->last_sg) {
  96                 sg_dma_len(umem_dmabuf->last_sg) +=
  97                         umem_dmabuf->last_sg_trim;
  98                 umem_dmabuf->last_sg = NULL;
  99                 umem_dmabuf->last_sg_trim = 0;
 100         }
 101
 102         dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt,
 103                                  DMA_BIDIRECTIONAL);
 104
 105         umem_dmabuf->sgt = NULL;
 106 }
 107 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);
 108
 109 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
 110                                           unsigned long offset, size_t size,
 111                                           int fd, int access,
 112                                           const struct dma_buf_attach_ops *ops)
 113 {
 114         struct dma_buf *dmabuf;
 115         struct ib_umem_dmabuf *umem_dmabuf;
 116         struct ib_umem *umem;
 117         unsigned long end;
 118         struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);
 119
 120         if (check_add_overflow(offset, (unsigned long)size, &end))
 121                 return ret;
 122
 123         if (unlikely(!ops || !ops->move_notify))
 124                 return ret;
 125
 126         dmabuf = dma_buf_get(fd);
 127         if (IS_ERR(dmabuf))
 128                 return ERR_CAST(dmabuf);
 129
 130         if (dmabuf->size < end)
 131                 goto out_release_dmabuf;
 132
 133         umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
 134         if (!umem_dmabuf) {
 135                 ret = ERR_PTR(-ENOMEM);
 136                 goto out_release_dmabuf;
 137         }
 138
 139         umem = &umem_dmabuf->umem;
 140         umem->ibdev = device;
 141         umem->length = size;
 142         umem->address = offset;
 143         umem->writable = ib_access_writable(access);
 144         umem->is_dmabuf = 1;
 145
 146         if (!ib_umem_num_pages(umem))
 147                 goto out_free_umem;
 148
 149         umem_dmabuf->attach = dma_buf_dynamic_attach(
 150                                         dmabuf,
 151                                         device->dma_device,
 152                                         ops,
 153                                         umem_dmabuf);
 154         if (IS_ERR(umem_dmabuf->attach)) {
 155                 ret = ERR_CAST(umem_dmabuf->attach);
 156                 goto out_free_umem;
 157         }
 158         return umem_dmabuf;
 159
 160 out_free_umem:
 161         kfree(umem_dmabuf);
 162
 163 out_release_dmabuf:
 164         dma_buf_put(dmabuf);
 165         return ret;
 166 }
 167 EXPORT_SYMBOL(ib_umem_dmabuf_get);
 168
 169 static void
 170 ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach)
 171 {
 172         struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv;
 173
 174         ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev,
 175                                "Invalidate callback should not be called when memory is pinned\n");
 176 }
 177
 178 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
 179         .allow_peer2peer = true,
 180         .move_notify = ib_umem_dmabuf_unsupported_move_notify,
 181 };
 182
 183 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
 184                                                  unsigned long offset,
 185                                                  size_t size, int fd,
 186                                                  int access)
 187 {
 188         struct ib_umem_dmabuf *umem_dmabuf;
 189         int err;
 190
 191         umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access,
 192                                          &ib_umem_dmabuf_attach_pinned_ops);
 193         if (IS_ERR(umem_dmabuf))
 194                 return umem_dmabuf;
 195
 196         dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL);
 197         err = dma_buf_pin(umem_dmabuf->attach);
 198         if (err)
 199                 goto err_release;
 200         umem_dmabuf->pinned = 1;
 201
 202         err = ib_umem_dmabuf_map_pages(umem_dmabuf);
 203         if (err)
 204                 goto err_unpin;
 205         dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
 206
 207         return umem_dmabuf;
 208
 209 err_unpin:
 210         dma_buf_unpin(umem_dmabuf->attach);
 211 err_release:
 212         dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
 213         ib_umem_release(&umem_dmabuf->umem);
 214         return ERR_PTR(err);
 215 }
 216 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned);
 217
 218 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
 219 {
 220         struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
 221
 222         dma_resv_lock(dmabuf->resv, NULL);
 223         ib_umem_dmabuf_unmap_pages(umem_dmabuf);
 224         if (umem_dmabuf->pinned)
 225                 dma_buf_unpin(umem_dmabuf->attach);
 226         dma_resv_unlock(dmabuf->resv);
 227
 228         dma_buf_detach(dmabuf, umem_dmabuf->attach);
 229         dma_buf_put(dmabuf);
 230         kfree(umem_dmabuf);
 231 }