1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright 2018-2020 Broadcom. |
4 | */ |
5 | #include <linux/dma-mapping.h> |
6 | #include <linux/mm.h> |
7 | #include <linux/pagemap.h> |
8 | #include <linux/pgtable.h> |
9 | #include <linux/vmalloc.h> |
10 | |
11 | #include <asm/page.h> |
12 | #include <asm/unaligned.h> |
13 | |
14 | #include <uapi/linux/misc/bcm_vk.h> |
15 | |
16 | #include "bcm_vk.h" |
17 | #include "bcm_vk_msg.h" |
18 | #include "bcm_vk_sg.h" |
19 | |
20 | /* |
21 | * Valkyrie has a hardware limitation of 16M transfer size. |
22 | * So limit the SGL chunks to 16M. |
23 | */ |
24 | #define BCM_VK_MAX_SGL_CHUNK SZ_16M |
25 | |
26 | static int bcm_vk_dma_alloc(struct device *dev, |
27 | struct bcm_vk_dma *dma, |
28 | int dir, |
29 | struct _vk_data *vkdata); |
30 | static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma); |
31 | |
32 | /* Uncomment to dump SGLIST */ |
33 | /* #define BCM_VK_DUMP_SGLIST */ |
34 | |
35 | static int bcm_vk_dma_alloc(struct device *dev, |
36 | struct bcm_vk_dma *dma, |
37 | int direction, |
38 | struct _vk_data *vkdata) |
39 | { |
40 | dma_addr_t addr, sg_addr; |
41 | int err; |
42 | int i; |
43 | int offset; |
44 | u32 size; |
45 | u32 remaining_size; |
46 | u32 transfer_size; |
47 | u64 data; |
48 | unsigned long first, last; |
49 | struct _vk_data *sgdata; |
50 | |
51 | /* Get 64-bit user address */ |
52 | data = get_unaligned(&vkdata->address); |
53 | |
54 | /* offset into first page */ |
55 | offset = offset_in_page(data); |
56 | |
57 | /* Calculate number of pages */ |
58 | first = (data & PAGE_MASK) >> PAGE_SHIFT; |
59 | last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT; |
60 | dma->nr_pages = last - first + 1; |
61 | |
62 | /* Allocate DMA pages */ |
63 | dma->pages = kmalloc_array(n: dma->nr_pages, |
64 | size: sizeof(struct page *), |
65 | GFP_KERNEL); |
66 | if (!dma->pages) |
67 | return -ENOMEM; |
68 | |
69 | dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n" , |
70 | data, vkdata->size, dma->nr_pages); |
71 | |
72 | dma->direction = direction; |
73 | |
74 | /* Get user pages into memory */ |
75 | err = get_user_pages_fast(start: data & PAGE_MASK, |
76 | nr_pages: dma->nr_pages, |
77 | gup_flags: direction == DMA_FROM_DEVICE, |
78 | pages: dma->pages); |
79 | if (err != dma->nr_pages) { |
80 | dma->nr_pages = (err >= 0) ? err : 0; |
81 | dev_err(dev, "get_user_pages_fast, err=%d [%d]\n" , |
82 | err, dma->nr_pages); |
83 | return err < 0 ? err : -EINVAL; |
84 | } |
85 | |
86 | /* Max size of sg list is 1 per mapped page + fields at start */ |
87 | dma->sglen = (dma->nr_pages * sizeof(*sgdata)) + |
88 | (sizeof(u32) * SGLIST_VKDATA_START); |
89 | |
90 | /* Allocate sglist */ |
91 | dma->sglist = dma_alloc_coherent(dev, |
92 | size: dma->sglen, |
93 | dma_handle: &dma->handle, |
94 | GFP_KERNEL); |
95 | if (!dma->sglist) |
96 | return -ENOMEM; |
97 | |
98 | dma->sglist[SGLIST_NUM_SG] = 0; |
99 | dma->sglist[SGLIST_TOTALSIZE] = vkdata->size; |
100 | remaining_size = vkdata->size; |
101 | sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; |
102 | |
103 | /* Map all pages into DMA */ |
104 | size = min_t(size_t, PAGE_SIZE - offset, remaining_size); |
105 | remaining_size -= size; |
106 | sg_addr = dma_map_page(dev, |
107 | dma->pages[0], |
108 | offset, |
109 | size, |
110 | dma->direction); |
111 | transfer_size = size; |
112 | if (unlikely(dma_mapping_error(dev, sg_addr))) { |
113 | __free_page(dma->pages[0]); |
114 | return -EIO; |
115 | } |
116 | |
117 | for (i = 1; i < dma->nr_pages; i++) { |
118 | size = min_t(size_t, PAGE_SIZE, remaining_size); |
119 | remaining_size -= size; |
120 | addr = dma_map_page(dev, |
121 | dma->pages[i], |
122 | 0, |
123 | size, |
124 | dma->direction); |
125 | if (unlikely(dma_mapping_error(dev, addr))) { |
126 | __free_page(dma->pages[i]); |
127 | return -EIO; |
128 | } |
129 | |
130 | /* |
131 | * Compress SG list entry when pages are contiguous |
132 | * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK |
133 | */ |
134 | if ((addr == (sg_addr + transfer_size)) && |
135 | ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) { |
136 | /* pages are contiguous, add to same sg entry */ |
137 | transfer_size += size; |
138 | } else { |
139 | /* pages are not contiguous, write sg entry */ |
140 | sgdata->size = transfer_size; |
141 | put_unaligned(sg_addr, (u64 *)&sgdata->address); |
142 | dma->sglist[SGLIST_NUM_SG]++; |
143 | |
144 | /* start new sg entry */ |
145 | sgdata++; |
146 | sg_addr = addr; |
147 | transfer_size = size; |
148 | } |
149 | } |
150 | /* Write last sg list entry */ |
151 | sgdata->size = transfer_size; |
152 | put_unaligned(sg_addr, (u64 *)&sgdata->address); |
153 | dma->sglist[SGLIST_NUM_SG]++; |
154 | |
155 | /* Update pointers and size field to point to sglist */ |
156 | put_unaligned((u64)dma->handle, &vkdata->address); |
157 | vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) + |
158 | (sizeof(u32) * SGLIST_VKDATA_START); |
159 | |
160 | #ifdef BCM_VK_DUMP_SGLIST |
161 | dev_dbg(dev, |
162 | "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n" , |
163 | (u64)dma->sglist, |
164 | dma->handle, |
165 | dma->sglen, |
166 | vkdata->size); |
167 | for (i = 0; i < vkdata->size / sizeof(u32); i++) |
168 | dev_dbg(dev, "i:0x%x 0x%x\n" , i, dma->sglist[i]); |
169 | #endif |
170 | |
171 | return 0; |
172 | } |
173 | |
174 | int bcm_vk_sg_alloc(struct device *dev, |
175 | struct bcm_vk_dma *dma, |
176 | int dir, |
177 | struct _vk_data *vkdata, |
178 | int num) |
179 | { |
180 | int i; |
181 | int rc = -EINVAL; |
182 | |
183 | /* Convert user addresses to DMA SG List */ |
184 | for (i = 0; i < num; i++) { |
185 | if (vkdata[i].size && vkdata[i].address) { |
186 | /* |
187 | * If both size and address are non-zero |
188 | * then DMA alloc. |
189 | */ |
190 | rc = bcm_vk_dma_alloc(dev, |
191 | dma: &dma[i], |
192 | direction: dir, |
193 | vkdata: &vkdata[i]); |
194 | } else if (vkdata[i].size || |
195 | vkdata[i].address) { |
196 | /* |
197 | * If one of size and address are zero |
198 | * there is a problem. |
199 | */ |
200 | dev_err(dev, |
201 | "Invalid vkdata %x 0x%x 0x%llx\n" , |
202 | i, vkdata[i].size, vkdata[i].address); |
203 | rc = -EINVAL; |
204 | } else { |
205 | /* |
206 | * If size and address are both zero |
207 | * don't convert, but return success. |
208 | */ |
209 | rc = 0; |
210 | } |
211 | |
212 | if (rc) |
213 | goto fail_alloc; |
214 | } |
215 | return rc; |
216 | |
217 | fail_alloc: |
218 | while (i > 0) { |
219 | i--; |
220 | if (dma[i].sglist) |
221 | bcm_vk_dma_free(dev, dma: &dma[i]); |
222 | } |
223 | return rc; |
224 | } |
225 | |
226 | static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma) |
227 | { |
228 | dma_addr_t addr; |
229 | int i; |
230 | int num_sg; |
231 | u32 size; |
232 | struct _vk_data *vkdata; |
233 | |
234 | dev_dbg(dev, "free sglist=%p sglen=0x%x\n" , dma->sglist, dma->sglen); |
235 | |
236 | /* Unmap all pages in the sglist */ |
237 | num_sg = dma->sglist[SGLIST_NUM_SG]; |
238 | vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; |
239 | for (i = 0; i < num_sg; i++) { |
240 | size = vkdata[i].size; |
241 | addr = get_unaligned(&vkdata[i].address); |
242 | |
243 | dma_unmap_page(dev, addr, size, dma->direction); |
244 | } |
245 | |
246 | /* Free allocated sglist */ |
247 | dma_free_coherent(dev, size: dma->sglen, cpu_addr: dma->sglist, dma_handle: dma->handle); |
248 | |
249 | /* Release lock on all pages */ |
250 | for (i = 0; i < dma->nr_pages; i++) |
251 | put_page(page: dma->pages[i]); |
252 | |
253 | /* Free allocated dma pages */ |
254 | kfree(objp: dma->pages); |
255 | dma->sglist = NULL; |
256 | |
257 | return 0; |
258 | } |
259 | |
260 | int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, |
261 | int *proc_cnt) |
262 | { |
263 | int i; |
264 | |
265 | *proc_cnt = 0; |
266 | /* Unmap and free all pages and sglists */ |
267 | for (i = 0; i < num; i++) { |
268 | if (dma[i].sglist) { |
269 | bcm_vk_dma_free(dev, dma: &dma[i]); |
270 | *proc_cnt += 1; |
271 | } |
272 | } |
273 | |
274 | return 0; |
275 | } |
276 | |