1 | // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
2 | /* |
3 | * Copyright (c) 2020 Intel Corporation. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/dma-buf.h> |
7 | #include <linux/dma-resv.h> |
8 | #include <linux/dma-mapping.h> |
9 | #include <linux/module.h> |
10 | |
11 | #include "uverbs.h" |
12 | |
13 | MODULE_IMPORT_NS(DMA_BUF); |
14 | |
15 | int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) |
16 | { |
17 | struct sg_table *sgt; |
18 | struct scatterlist *sg; |
19 | unsigned long start, end, cur = 0; |
20 | unsigned int nmap = 0; |
21 | long ret; |
22 | int i; |
23 | |
24 | dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); |
25 | |
26 | if (umem_dmabuf->sgt) |
27 | goto wait_fence; |
28 | |
29 | sgt = dma_buf_map_attachment(umem_dmabuf->attach, |
30 | DMA_BIDIRECTIONAL); |
31 | if (IS_ERR(ptr: sgt)) |
32 | return PTR_ERR(ptr: sgt); |
33 | |
34 | /* modify the sg list in-place to match umem address and length */ |
35 | |
36 | start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE); |
37 | end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, |
38 | PAGE_SIZE); |
39 | for_each_sgtable_dma_sg(sgt, sg, i) { |
40 | if (start < cur + sg_dma_len(sg) && cur < end) |
41 | nmap++; |
42 | if (cur <= start && start < cur + sg_dma_len(sg)) { |
43 | unsigned long offset = start - cur; |
44 | |
45 | umem_dmabuf->first_sg = sg; |
46 | umem_dmabuf->first_sg_offset = offset; |
47 | sg_dma_address(sg) += offset; |
48 | sg_dma_len(sg) -= offset; |
49 | cur += offset; |
50 | } |
51 | if (cur < end && end <= cur + sg_dma_len(sg)) { |
52 | unsigned long trim = cur + sg_dma_len(sg) - end; |
53 | |
54 | umem_dmabuf->last_sg = sg; |
55 | umem_dmabuf->last_sg_trim = trim; |
56 | sg_dma_len(sg) -= trim; |
57 | break; |
58 | } |
59 | cur += sg_dma_len(sg); |
60 | } |
61 | |
62 | umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; |
63 | umem_dmabuf->umem.sgt_append.sgt.nents = nmap; |
64 | umem_dmabuf->sgt = sgt; |
65 | |
66 | wait_fence: |
67 | /* |
68 | * Although the sg list is valid now, the content of the pages |
69 | * may be not up-to-date. Wait for the exporter to finish |
70 | * the migration. |
71 | */ |
72 | ret = dma_resv_wait_timeout(obj: umem_dmabuf->attach->dmabuf->resv, |
73 | usage: DMA_RESV_USAGE_KERNEL, |
74 | intr: false, MAX_SCHEDULE_TIMEOUT); |
75 | if (ret < 0) |
76 | return ret; |
77 | if (ret == 0) |
78 | return -ETIMEDOUT; |
79 | return 0; |
80 | } |
81 | EXPORT_SYMBOL(ib_umem_dmabuf_map_pages); |
82 | |
83 | void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) |
84 | { |
85 | dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); |
86 | |
87 | if (!umem_dmabuf->sgt) |
88 | return; |
89 | |
90 | /* retore the original sg list */ |
91 | if (umem_dmabuf->first_sg) { |
92 | sg_dma_address(umem_dmabuf->first_sg) -= |
93 | umem_dmabuf->first_sg_offset; |
94 | sg_dma_len(umem_dmabuf->first_sg) += |
95 | umem_dmabuf->first_sg_offset; |
96 | umem_dmabuf->first_sg = NULL; |
97 | umem_dmabuf->first_sg_offset = 0; |
98 | } |
99 | if (umem_dmabuf->last_sg) { |
100 | sg_dma_len(umem_dmabuf->last_sg) += |
101 | umem_dmabuf->last_sg_trim; |
102 | umem_dmabuf->last_sg = NULL; |
103 | umem_dmabuf->last_sg_trim = 0; |
104 | } |
105 | |
106 | dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt, |
107 | DMA_BIDIRECTIONAL); |
108 | |
109 | umem_dmabuf->sgt = NULL; |
110 | } |
111 | EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages); |
112 | |
113 | struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, |
114 | unsigned long offset, size_t size, |
115 | int fd, int access, |
116 | const struct dma_buf_attach_ops *ops) |
117 | { |
118 | struct dma_buf *dmabuf; |
119 | struct ib_umem_dmabuf *umem_dmabuf; |
120 | struct ib_umem *umem; |
121 | unsigned long end; |
122 | struct ib_umem_dmabuf *ret = ERR_PTR(error: -EINVAL); |
123 | |
124 | if (check_add_overflow(offset, (unsigned long)size, &end)) |
125 | return ret; |
126 | |
127 | if (unlikely(!ops || !ops->move_notify)) |
128 | return ret; |
129 | |
130 | dmabuf = dma_buf_get(fd); |
131 | if (IS_ERR(ptr: dmabuf)) |
132 | return ERR_CAST(ptr: dmabuf); |
133 | |
134 | if (dmabuf->size < end) |
135 | goto out_release_dmabuf; |
136 | |
137 | umem_dmabuf = kzalloc(size: sizeof(*umem_dmabuf), GFP_KERNEL); |
138 | if (!umem_dmabuf) { |
139 | ret = ERR_PTR(error: -ENOMEM); |
140 | goto out_release_dmabuf; |
141 | } |
142 | |
143 | umem = &umem_dmabuf->umem; |
144 | umem->ibdev = device; |
145 | umem->length = size; |
146 | umem->address = offset; |
147 | umem->writable = ib_access_writable(access_flags: access); |
148 | umem->is_dmabuf = 1; |
149 | |
150 | if (!ib_umem_num_pages(umem)) |
151 | goto out_free_umem; |
152 | |
153 | umem_dmabuf->attach = dma_buf_dynamic_attach( |
154 | dmabuf, |
155 | dev: device->dma_device, |
156 | importer_ops: ops, |
157 | importer_priv: umem_dmabuf); |
158 | if (IS_ERR(ptr: umem_dmabuf->attach)) { |
159 | ret = ERR_CAST(ptr: umem_dmabuf->attach); |
160 | goto out_free_umem; |
161 | } |
162 | return umem_dmabuf; |
163 | |
164 | out_free_umem: |
165 | kfree(objp: umem_dmabuf); |
166 | |
167 | out_release_dmabuf: |
168 | dma_buf_put(dmabuf); |
169 | return ret; |
170 | } |
171 | EXPORT_SYMBOL(ib_umem_dmabuf_get); |
172 | |
173 | static void |
174 | ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach) |
175 | { |
176 | struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; |
177 | |
178 | ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev, |
179 | "Invalidate callback should not be called when memory is pinned\n" ); |
180 | } |
181 | |
182 | static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { |
183 | .allow_peer2peer = true, |
184 | .move_notify = ib_umem_dmabuf_unsupported_move_notify, |
185 | }; |
186 | |
187 | struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, |
188 | unsigned long offset, |
189 | size_t size, int fd, |
190 | int access) |
191 | { |
192 | struct ib_umem_dmabuf *umem_dmabuf; |
193 | int err; |
194 | |
195 | umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access, |
196 | &ib_umem_dmabuf_attach_pinned_ops); |
197 | if (IS_ERR(ptr: umem_dmabuf)) |
198 | return umem_dmabuf; |
199 | |
200 | dma_resv_lock(obj: umem_dmabuf->attach->dmabuf->resv, NULL); |
201 | err = dma_buf_pin(attach: umem_dmabuf->attach); |
202 | if (err) |
203 | goto err_release; |
204 | umem_dmabuf->pinned = 1; |
205 | |
206 | err = ib_umem_dmabuf_map_pages(umem_dmabuf); |
207 | if (err) |
208 | goto err_unpin; |
209 | dma_resv_unlock(obj: umem_dmabuf->attach->dmabuf->resv); |
210 | |
211 | return umem_dmabuf; |
212 | |
213 | err_unpin: |
214 | dma_buf_unpin(attach: umem_dmabuf->attach); |
215 | err_release: |
216 | dma_resv_unlock(obj: umem_dmabuf->attach->dmabuf->resv); |
217 | ib_umem_release(umem: &umem_dmabuf->umem); |
218 | return ERR_PTR(error: err); |
219 | } |
220 | EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned); |
221 | |
222 | void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) |
223 | { |
224 | struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; |
225 | |
226 | dma_resv_lock(obj: dmabuf->resv, NULL); |
227 | ib_umem_dmabuf_unmap_pages(umem_dmabuf); |
228 | if (umem_dmabuf->pinned) |
229 | dma_buf_unpin(attach: umem_dmabuf->attach); |
230 | dma_resv_unlock(obj: dmabuf->resv); |
231 | |
232 | dma_buf_detach(dmabuf, attach: umem_dmabuf->attach); |
233 | dma_buf_put(dmabuf); |
234 | kfree(objp: umem_dmabuf); |
235 | } |
236 | |