drivers/infiniband/core/umem_dmabuf.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
   2 /*
   3  * Copyright (c) 2020 Intel Corporation. All rights reserved.
   4  */
   5
   6 #include <linux/dma-buf.h>
   7 #include <linux/dma-resv.h>
   8 #include <linux/dma-mapping.h>
   9
  10 #include "uverbs.h"
  11
  12 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
  13 {
  14         struct sg_table *sgt;
  15         struct scatterlist *sg;
  16         struct dma_fence *fence;
  17         unsigned long start, end, cur = 0;
  18         unsigned int nmap = 0;
  19         int i;
  20
  21         dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
  22
  23         if (umem_dmabuf->sgt)
  24                 goto wait_fence;
  25
  26         sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL);
  27         if (IS_ERR(sgt))
  28                 return PTR_ERR(sgt);
  29
  30         /* modify the sg list in-place to match umem address and length */
  31
  32         start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE);
  33         end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length,
  34                     PAGE_SIZE);
  35         for_each_sgtable_dma_sg(sgt, sg, i) {
  36                 if (start < cur + sg_dma_len(sg) && cur < end)
  37                         nmap++;
  38                 if (cur <= start && start < cur + sg_dma_len(sg)) {
  39                         unsigned long offset = start - cur;
  40
  41                         umem_dmabuf->first_sg = sg;
  42                         umem_dmabuf->first_sg_offset = offset;
  43                         sg_dma_address(sg) += offset;
  44                         sg_dma_len(sg) -= offset;
  45                         cur += offset;
  46                 }
  47                 if (cur < end && end <= cur + sg_dma_len(sg)) {
  48                         unsigned long trim = cur + sg_dma_len(sg) - end;
  49
  50                         umem_dmabuf->last_sg = sg;
  51                         umem_dmabuf->last_sg_trim = trim;
  52                         sg_dma_len(sg) -= trim;
  53                         break;
  54                 }
  55                 cur += sg_dma_len(sg);
  56         }
  57
  58         umem_dmabuf->umem.sg_head.sgl = umem_dmabuf->first_sg;
  59         umem_dmabuf->umem.sg_head.nents = nmap;
  60         umem_dmabuf->umem.nmap = nmap;
  61         umem_dmabuf->sgt = sgt;
  62
  63 wait_fence:
  64         /*
  65          * Although the sg list is valid now, the content of the pages
  66          * may be not up-to-date. Wait for the exporter to finish
  67          * the migration.
  68          */
  69         fence = dma_resv_get_excl(umem_dmabuf->attach->dmabuf->resv);
  70         if (fence)
  71                 return dma_fence_wait(fence, false);
  72
  73         return 0;
  74 }
  75 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);
  76
  77 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)
  78 {
  79         dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
  80
  81         if (!umem_dmabuf->sgt)
  82                 return;
  83
  84         /* retore the original sg list */
  85         if (umem_dmabuf->first_sg) {
  86                 sg_dma_address(umem_dmabuf->first_sg) -=
  87                         umem_dmabuf->first_sg_offset;
  88                 sg_dma_len(umem_dmabuf->first_sg) +=
  89                         umem_dmabuf->first_sg_offset;
  90                 umem_dmabuf->first_sg = NULL;
  91                 umem_dmabuf->first_sg_offset = 0;
  92         }
  93         if (umem_dmabuf->last_sg) {
  94                 sg_dma_len(umem_dmabuf->last_sg) +=
  95                         umem_dmabuf->last_sg_trim;
  96                 umem_dmabuf->last_sg = NULL;
  97                 umem_dmabuf->last_sg_trim = 0;
  98         }
  99
 100         dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt,
 101                                  DMA_BIDIRECTIONAL);
 102
 103         umem_dmabuf->sgt = NULL;
 104 }
 105 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);
 106
 107 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
 108                                           unsigned long offset, size_t size,
 109                                           int fd, int access,
 110                                           const struct dma_buf_attach_ops *ops)
 111 {
 112         struct dma_buf *dmabuf;
 113         struct ib_umem_dmabuf *umem_dmabuf;
 114         struct ib_umem *umem;
 115         unsigned long end;
 116         struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);
 117
 118         if (check_add_overflow(offset, (unsigned long)size, &end))
 119                 return ret;
 120
 121         if (unlikely(!ops || !ops->move_notify))
 122                 return ret;
 123
 124         dmabuf = dma_buf_get(fd);
 125         if (IS_ERR(dmabuf))
 126                 return ERR_CAST(dmabuf);
 127
 128         if (dmabuf->size < end)
 129                 goto out_release_dmabuf;
 130
 131         umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
 132         if (!umem_dmabuf) {
 133                 ret = ERR_PTR(-ENOMEM);
 134                 goto out_release_dmabuf;
 135         }
 136
 137         umem = &umem_dmabuf->umem;
 138         umem->ibdev = device;
 139         umem->length = size;
 140         umem->address = offset;
 141         umem->writable = ib_access_writable(access);
 142         umem->is_dmabuf = 1;
 143
 144         if (!ib_umem_num_pages(umem))
 145                 goto out_free_umem;
 146
 147         umem_dmabuf->attach = dma_buf_dynamic_attach(
 148                                         dmabuf,
 149                                         device->dma_device,
 150                                         ops,
 151                                         umem_dmabuf);
 152         if (IS_ERR(umem_dmabuf->attach)) {
 153                 ret = ERR_CAST(umem_dmabuf->attach);
 154                 goto out_free_umem;
 155         }
 156         return umem_dmabuf;
 157
 158 out_free_umem:
 159         kfree(umem_dmabuf);
 160
 161 out_release_dmabuf:
 162         dma_buf_put(dmabuf);
 163         return ret;
 164 }
 165 EXPORT_SYMBOL(ib_umem_dmabuf_get);
 166
 167 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
 168 {
 169         struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
 170
 171         dma_resv_lock(dmabuf->resv, NULL);
 172         ib_umem_dmabuf_unmap_pages(umem_dmabuf);
 173         dma_resv_unlock(dmabuf->resv);
 174
 175         dma_buf_detach(dmabuf, umem_dmabuf->attach);
 176         dma_buf_put(dmabuf);
 177         kfree(umem_dmabuf);
 178 }