1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
3 * Copyright (c) 2020 Intel Corporation. All rights reserved.
6 #include <linux/dma-buf.h>
7 #include <linux/dma-resv.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/module.h>
13 MODULE_IMPORT_NS(DMA_BUF);
15 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
18 struct scatterlist *sg;
19 struct dma_fence *fence;
20 unsigned long start, end, cur = 0;
21 unsigned int nmap = 0;
24 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
29 sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL);
33 /* modify the sg list in-place to match umem address and length */
35 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE);
36 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length,
38 for_each_sgtable_dma_sg(sgt, sg, i) {
39 if (start < cur + sg_dma_len(sg) && cur < end)
41 if (cur <= start && start < cur + sg_dma_len(sg)) {
42 unsigned long offset = start - cur;
44 umem_dmabuf->first_sg = sg;
45 umem_dmabuf->first_sg_offset = offset;
46 sg_dma_address(sg) += offset;
47 sg_dma_len(sg) -= offset;
50 if (cur < end && end <= cur + sg_dma_len(sg)) {
51 unsigned long trim = cur + sg_dma_len(sg) - end;
53 umem_dmabuf->last_sg = sg;
54 umem_dmabuf->last_sg_trim = trim;
55 sg_dma_len(sg) -= trim;
58 cur += sg_dma_len(sg);
61 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg;
62 umem_dmabuf->umem.sgt_append.sgt.nents = nmap;
63 umem_dmabuf->sgt = sgt;
67 * Although the sg list is valid now, the content of the pages
68 * may be not up-to-date. Wait for the exporter to finish
71 fence = dma_resv_excl_fence(umem_dmabuf->attach->dmabuf->resv);
73 return dma_fence_wait(fence, false);
77 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);
79 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)
81 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
83 if (!umem_dmabuf->sgt)
86 /* retore the original sg list */
87 if (umem_dmabuf->first_sg) {
88 sg_dma_address(umem_dmabuf->first_sg) -=
89 umem_dmabuf->first_sg_offset;
90 sg_dma_len(umem_dmabuf->first_sg) +=
91 umem_dmabuf->first_sg_offset;
92 umem_dmabuf->first_sg = NULL;
93 umem_dmabuf->first_sg_offset = 0;
95 if (umem_dmabuf->last_sg) {
96 sg_dma_len(umem_dmabuf->last_sg) +=
97 umem_dmabuf->last_sg_trim;
98 umem_dmabuf->last_sg = NULL;
99 umem_dmabuf->last_sg_trim = 0;
102 dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt,
105 umem_dmabuf->sgt = NULL;
107 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);
109 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
110 unsigned long offset, size_t size,
112 const struct dma_buf_attach_ops *ops)
114 struct dma_buf *dmabuf;
115 struct ib_umem_dmabuf *umem_dmabuf;
116 struct ib_umem *umem;
118 struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);
120 if (check_add_overflow(offset, (unsigned long)size, &end))
123 if (unlikely(!ops || !ops->move_notify))
126 dmabuf = dma_buf_get(fd);
128 return ERR_CAST(dmabuf);
130 if (dmabuf->size < end)
131 goto out_release_dmabuf;
133 umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
135 ret = ERR_PTR(-ENOMEM);
136 goto out_release_dmabuf;
139 umem = &umem_dmabuf->umem;
140 umem->ibdev = device;
142 umem->address = offset;
143 umem->writable = ib_access_writable(access);
146 if (!ib_umem_num_pages(umem))
149 umem_dmabuf->attach = dma_buf_dynamic_attach(
154 if (IS_ERR(umem_dmabuf->attach)) {
155 ret = ERR_CAST(umem_dmabuf->attach);
167 EXPORT_SYMBOL(ib_umem_dmabuf_get);
170 ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach)
172 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv;
174 ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev,
175 "Invalidate callback should not be called when memory is pinned\n");
178 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
179 .allow_peer2peer = true,
180 .move_notify = ib_umem_dmabuf_unsupported_move_notify,
183 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
184 unsigned long offset,
188 struct ib_umem_dmabuf *umem_dmabuf;
191 umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access,
192 &ib_umem_dmabuf_attach_pinned_ops);
193 if (IS_ERR(umem_dmabuf))
196 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL);
197 err = dma_buf_pin(umem_dmabuf->attach);
200 umem_dmabuf->pinned = 1;
202 err = ib_umem_dmabuf_map_pages(umem_dmabuf);
205 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
210 dma_buf_unpin(umem_dmabuf->attach);
212 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
213 ib_umem_release(&umem_dmabuf->umem);
216 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned);
218 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
220 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
222 dma_resv_lock(dmabuf->resv, NULL);
223 ib_umem_dmabuf_unmap_pages(umem_dmabuf);
224 if (umem_dmabuf->pinned)
225 dma_buf_unpin(umem_dmabuf->attach);
226 dma_resv_unlock(dmabuf->resv);
228 dma_buf_detach(dmabuf, umem_dmabuf->attach);