1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/slab.h> |
5 | #include <linux/dma-mapping.h> |
6 | |
7 | #include "lima_device.h" |
8 | #include "lima_vm.h" |
9 | #include "lima_gem.h" |
10 | #include "lima_regs.h" |
11 | |
12 | struct lima_bo_va { |
13 | struct list_head list; |
14 | unsigned int ref_count; |
15 | |
16 | struct drm_mm_node node; |
17 | |
18 | struct lima_vm *vm; |
19 | }; |
20 | |
21 | #define LIMA_VM_PD_SHIFT 22 |
22 | #define LIMA_VM_PT_SHIFT 12 |
23 | #define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT) |
24 | #define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT |
25 | |
26 | #define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1) |
27 | #define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1) |
28 | |
29 | #define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT) |
30 | #define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT) |
31 | #define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT) |
32 | #define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT) |
33 | |
34 | |
35 | static void lima_vm_unmap_range(struct lima_vm *vm, u32 start, u32 end) |
36 | { |
37 | u32 addr; |
38 | |
39 | for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) { |
40 | u32 pbe = LIMA_PBE(addr); |
41 | u32 bte = LIMA_BTE(addr); |
42 | |
43 | vm->bts[pbe].cpu[bte] = 0; |
44 | } |
45 | } |
46 | |
47 | static int lima_vm_map_page(struct lima_vm *vm, dma_addr_t pa, u32 va) |
48 | { |
49 | u32 pbe = LIMA_PBE(va); |
50 | u32 bte = LIMA_BTE(va); |
51 | |
52 | if (!vm->bts[pbe].cpu) { |
53 | dma_addr_t pts; |
54 | u32 *pd; |
55 | int j; |
56 | |
57 | vm->bts[pbe].cpu = dma_alloc_wc( |
58 | dev: vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT, |
59 | dma_addr: &vm->bts[pbe].dma, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); |
60 | if (!vm->bts[pbe].cpu) |
61 | return -ENOMEM; |
62 | |
63 | pts = vm->bts[pbe].dma; |
64 | pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT); |
65 | for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) { |
66 | pd[j] = pts | LIMA_VM_FLAG_PRESENT; |
67 | pts += LIMA_PAGE_SIZE; |
68 | } |
69 | } |
70 | |
71 | vm->bts[pbe].cpu[bte] = pa | LIMA_VM_FLAGS_CACHE; |
72 | |
73 | return 0; |
74 | } |
75 | |
76 | static struct lima_bo_va * |
77 | lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo) |
78 | { |
79 | struct lima_bo_va *bo_va, *ret = NULL; |
80 | |
81 | list_for_each_entry(bo_va, &bo->va, list) { |
82 | if (bo_va->vm == vm) { |
83 | ret = bo_va; |
84 | break; |
85 | } |
86 | } |
87 | |
88 | return ret; |
89 | } |
90 | |
91 | int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create) |
92 | { |
93 | struct lima_bo_va *bo_va; |
94 | struct sg_dma_page_iter sg_iter; |
95 | int offset = 0, err; |
96 | |
97 | mutex_lock(&bo->lock); |
98 | |
99 | bo_va = lima_vm_bo_find(vm, bo); |
100 | if (bo_va) { |
101 | bo_va->ref_count++; |
102 | mutex_unlock(lock: &bo->lock); |
103 | return 0; |
104 | } |
105 | |
106 | /* should not create new bo_va if not asked by caller */ |
107 | if (!create) { |
108 | mutex_unlock(lock: &bo->lock); |
109 | return -ENOENT; |
110 | } |
111 | |
112 | bo_va = kzalloc(size: sizeof(*bo_va), GFP_KERNEL); |
113 | if (!bo_va) { |
114 | err = -ENOMEM; |
115 | goto err_out0; |
116 | } |
117 | |
118 | bo_va->vm = vm; |
119 | bo_va->ref_count = 1; |
120 | |
121 | mutex_lock(&vm->lock); |
122 | |
123 | err = drm_mm_insert_node(mm: &vm->mm, node: &bo_va->node, size: lima_bo_size(bo)); |
124 | if (err) |
125 | goto err_out1; |
126 | |
127 | for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, 0) { |
128 | err = lima_vm_map_page(vm, pa: sg_page_iter_dma_address(dma_iter: &sg_iter), |
129 | va: bo_va->node.start + offset); |
130 | if (err) |
131 | goto err_out2; |
132 | |
133 | offset += PAGE_SIZE; |
134 | } |
135 | |
136 | mutex_unlock(lock: &vm->lock); |
137 | |
138 | list_add_tail(new: &bo_va->list, head: &bo->va); |
139 | |
140 | mutex_unlock(lock: &bo->lock); |
141 | return 0; |
142 | |
143 | err_out2: |
144 | if (offset) |
145 | lima_vm_unmap_range(vm, start: bo_va->node.start, end: bo_va->node.start + offset - 1); |
146 | drm_mm_remove_node(node: &bo_va->node); |
147 | err_out1: |
148 | mutex_unlock(lock: &vm->lock); |
149 | kfree(objp: bo_va); |
150 | err_out0: |
151 | mutex_unlock(lock: &bo->lock); |
152 | return err; |
153 | } |
154 | |
155 | void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo) |
156 | { |
157 | struct lima_bo_va *bo_va; |
158 | u32 size; |
159 | |
160 | mutex_lock(&bo->lock); |
161 | |
162 | bo_va = lima_vm_bo_find(vm, bo); |
163 | if (--bo_va->ref_count > 0) { |
164 | mutex_unlock(lock: &bo->lock); |
165 | return; |
166 | } |
167 | |
168 | mutex_lock(&vm->lock); |
169 | |
170 | size = bo->heap_size ? bo->heap_size : bo_va->node.size; |
171 | lima_vm_unmap_range(vm, start: bo_va->node.start, |
172 | end: bo_va->node.start + size - 1); |
173 | |
174 | drm_mm_remove_node(node: &bo_va->node); |
175 | |
176 | mutex_unlock(lock: &vm->lock); |
177 | |
178 | list_del(entry: &bo_va->list); |
179 | |
180 | mutex_unlock(lock: &bo->lock); |
181 | |
182 | kfree(objp: bo_va); |
183 | } |
184 | |
185 | u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo) |
186 | { |
187 | struct lima_bo_va *bo_va; |
188 | u32 ret; |
189 | |
190 | mutex_lock(&bo->lock); |
191 | |
192 | bo_va = lima_vm_bo_find(vm, bo); |
193 | ret = bo_va->node.start; |
194 | |
195 | mutex_unlock(lock: &bo->lock); |
196 | |
197 | return ret; |
198 | } |
199 | |
200 | struct lima_vm *lima_vm_create(struct lima_device *dev) |
201 | { |
202 | struct lima_vm *vm; |
203 | |
204 | vm = kzalloc(size: sizeof(*vm), GFP_KERNEL); |
205 | if (!vm) |
206 | return NULL; |
207 | |
208 | vm->dev = dev; |
209 | mutex_init(&vm->lock); |
210 | kref_init(kref: &vm->refcount); |
211 | |
212 | vm->pd.cpu = dma_alloc_wc(dev: dev->dev, LIMA_PAGE_SIZE, dma_addr: &vm->pd.dma, |
213 | GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); |
214 | if (!vm->pd.cpu) |
215 | goto err_out0; |
216 | |
217 | if (dev->dlbu_cpu) { |
218 | int err = lima_vm_map_page( |
219 | vm, pa: dev->dlbu_dma, LIMA_VA_RESERVE_DLBU); |
220 | if (err) |
221 | goto err_out1; |
222 | } |
223 | |
224 | drm_mm_init(mm: &vm->mm, start: dev->va_start, size: dev->va_end - dev->va_start); |
225 | |
226 | return vm; |
227 | |
228 | err_out1: |
229 | dma_free_wc(dev: dev->dev, LIMA_PAGE_SIZE, cpu_addr: vm->pd.cpu, dma_addr: vm->pd.dma); |
230 | err_out0: |
231 | kfree(objp: vm); |
232 | return NULL; |
233 | } |
234 | |
235 | void lima_vm_release(struct kref *kref) |
236 | { |
237 | struct lima_vm *vm = container_of(kref, struct lima_vm, refcount); |
238 | int i; |
239 | |
240 | drm_mm_takedown(mm: &vm->mm); |
241 | |
242 | for (i = 0; i < LIMA_VM_NUM_BT; i++) { |
243 | if (vm->bts[i].cpu) |
244 | dma_free_wc(dev: vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT, |
245 | cpu_addr: vm->bts[i].cpu, dma_addr: vm->bts[i].dma); |
246 | } |
247 | |
248 | if (vm->pd.cpu) |
249 | dma_free_wc(dev: vm->dev->dev, LIMA_PAGE_SIZE, cpu_addr: vm->pd.cpu, dma_addr: vm->pd.dma); |
250 | |
251 | kfree(objp: vm); |
252 | } |
253 | |
254 | void lima_vm_print(struct lima_vm *vm) |
255 | { |
256 | int i, j, k; |
257 | u32 *pd, *pt; |
258 | |
259 | if (!vm->pd.cpu) |
260 | return; |
261 | |
262 | pd = vm->pd.cpu; |
263 | for (i = 0; i < LIMA_VM_NUM_BT; i++) { |
264 | if (!vm->bts[i].cpu) |
265 | continue; |
266 | |
267 | pt = vm->bts[i].cpu; |
268 | for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) { |
269 | int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j; |
270 | |
271 | printk(KERN_INFO "lima vm pd %03x:%08x\n" , idx, pd[idx]); |
272 | |
273 | for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) { |
274 | u32 pte = *pt++; |
275 | |
276 | if (pte) |
277 | printk(KERN_INFO " pt %03x:%08x\n" , k, pte); |
278 | } |
279 | } |
280 | } |
281 | } |
282 | |
283 | int lima_vm_map_bo(struct lima_vm *vm, struct lima_bo *bo, int pageoff) |
284 | { |
285 | struct lima_bo_va *bo_va; |
286 | struct sg_dma_page_iter sg_iter; |
287 | int offset = 0, err; |
288 | u32 base; |
289 | |
290 | mutex_lock(&bo->lock); |
291 | |
292 | bo_va = lima_vm_bo_find(vm, bo); |
293 | if (!bo_va) { |
294 | err = -ENOENT; |
295 | goto err_out0; |
296 | } |
297 | |
298 | mutex_lock(&vm->lock); |
299 | |
300 | base = bo_va->node.start + (pageoff << PAGE_SHIFT); |
301 | for_each_sgtable_dma_page(bo->base.sgt, &sg_iter, pageoff) { |
302 | err = lima_vm_map_page(vm, pa: sg_page_iter_dma_address(dma_iter: &sg_iter), |
303 | va: base + offset); |
304 | if (err) |
305 | goto err_out1; |
306 | |
307 | offset += PAGE_SIZE; |
308 | } |
309 | |
310 | mutex_unlock(lock: &vm->lock); |
311 | |
312 | mutex_unlock(lock: &bo->lock); |
313 | return 0; |
314 | |
315 | err_out1: |
316 | if (offset) |
317 | lima_vm_unmap_range(vm, start: base, end: base + offset - 1); |
318 | mutex_unlock(lock: &vm->lock); |
319 | err_out0: |
320 | mutex_unlock(lock: &bo->lock); |
321 | return err; |
322 | } |
323 | |