1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright (c) 2007-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA, |
4 | * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA, |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
7 | * copy of this software and associated documentation files (the "Software"), |
8 | * to deal in the Software without restriction, including without limitation |
9 | * the rights to use, copy, modify, merge, publish, distribute, sub license, |
10 | * and/or sell copies of the Software, and to permit persons to whom the |
11 | * Software is furnished to do so, subject to the following conditions: |
12 | * |
13 | * The above copyright notice and this permission notice (including the |
14 | * next paragraph) shall be included in all copies or substantial portions |
15 | * of the Software. |
16 | * |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
19 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
20 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
21 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
22 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
23 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | */ |
25 | |
26 | #include <linux/limits.h> |
27 | |
28 | #include <drm/ttm/ttm_range_manager.h> |
29 | #include <drm/drm_cache.h> |
30 | |
31 | #include "nouveau_drv.h" |
32 | #include "nouveau_gem.h" |
33 | #include "nouveau_mem.h" |
34 | #include "nouveau_ttm.h" |
35 | |
36 | #include <core/tegra.h> |
37 | |
38 | static void |
39 | nouveau_manager_del(struct ttm_resource_manager *man, |
40 | struct ttm_resource *reg) |
41 | { |
42 | nouveau_mem_del(man, reg); |
43 | } |
44 | |
45 | static bool |
46 | nouveau_manager_intersects(struct ttm_resource_manager *man, |
47 | struct ttm_resource *res, |
48 | const struct ttm_place *place, |
49 | size_t size) |
50 | { |
51 | return nouveau_mem_intersects(res, place, size); |
52 | } |
53 | |
54 | static bool |
55 | nouveau_manager_compatible(struct ttm_resource_manager *man, |
56 | struct ttm_resource *res, |
57 | const struct ttm_place *place, |
58 | size_t size) |
59 | { |
60 | return nouveau_mem_compatible(res, place, size); |
61 | } |
62 | |
63 | static int |
64 | nouveau_vram_manager_new(struct ttm_resource_manager *man, |
65 | struct ttm_buffer_object *bo, |
66 | const struct ttm_place *place, |
67 | struct ttm_resource **res) |
68 | { |
69 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
70 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
71 | int ret; |
72 | |
73 | if (drm->client.device.info.ram_size == 0) |
74 | return -ENOMEM; |
75 | |
76 | ret = nouveau_mem_new(&drm->master, kind: nvbo->kind, comp: nvbo->comp, res); |
77 | if (ret) |
78 | return ret; |
79 | |
80 | ttm_resource_init(bo, place, res: *res); |
81 | |
82 | ret = nouveau_mem_vram(*res, contig: nvbo->contig, page: nvbo->page); |
83 | if (ret) { |
84 | nouveau_mem_del(man, *res); |
85 | return ret; |
86 | } |
87 | |
88 | return 0; |
89 | } |
90 | |
91 | const struct ttm_resource_manager_func nouveau_vram_manager = { |
92 | .alloc = nouveau_vram_manager_new, |
93 | .free = nouveau_manager_del, |
94 | .intersects = nouveau_manager_intersects, |
95 | .compatible = nouveau_manager_compatible, |
96 | }; |
97 | |
98 | static int |
99 | nouveau_gart_manager_new(struct ttm_resource_manager *man, |
100 | struct ttm_buffer_object *bo, |
101 | const struct ttm_place *place, |
102 | struct ttm_resource **res) |
103 | { |
104 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
105 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
106 | int ret; |
107 | |
108 | ret = nouveau_mem_new(&drm->master, kind: nvbo->kind, comp: nvbo->comp, res); |
109 | if (ret) |
110 | return ret; |
111 | |
112 | ttm_resource_init(bo, place, res: *res); |
113 | (*res)->start = 0; |
114 | return 0; |
115 | } |
116 | |
117 | const struct ttm_resource_manager_func nouveau_gart_manager = { |
118 | .alloc = nouveau_gart_manager_new, |
119 | .free = nouveau_manager_del, |
120 | .intersects = nouveau_manager_intersects, |
121 | .compatible = nouveau_manager_compatible, |
122 | }; |
123 | |
124 | static int |
125 | nv04_gart_manager_new(struct ttm_resource_manager *man, |
126 | struct ttm_buffer_object *bo, |
127 | const struct ttm_place *place, |
128 | struct ttm_resource **res) |
129 | { |
130 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
131 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
132 | struct nouveau_mem *mem; |
133 | int ret; |
134 | |
135 | ret = nouveau_mem_new(&drm->master, kind: nvbo->kind, comp: nvbo->comp, res); |
136 | if (ret) |
137 | return ret; |
138 | |
139 | mem = nouveau_mem(reg: *res); |
140 | ttm_resource_init(bo, place, res: *res); |
141 | ret = nvif_vmm_get(&mem->cli->vmm.vmm, PTES, false, 12, 0, |
142 | (long)(*res)->size, &mem->vma[0]); |
143 | if (ret) { |
144 | nouveau_mem_del(man, *res); |
145 | return ret; |
146 | } |
147 | |
148 | (*res)->start = mem->vma[0].addr >> PAGE_SHIFT; |
149 | return 0; |
150 | } |
151 | |
152 | const struct ttm_resource_manager_func nv04_gart_manager = { |
153 | .alloc = nv04_gart_manager_new, |
154 | .free = nouveau_manager_del, |
155 | .intersects = nouveau_manager_intersects, |
156 | .compatible = nouveau_manager_compatible, |
157 | }; |
158 | |
159 | static int |
160 | nouveau_ttm_init_host(struct nouveau_drm *drm, u8 kind) |
161 | { |
162 | struct nvif_mmu *mmu = &drm->client.mmu; |
163 | int typei; |
164 | |
165 | typei = nvif_mmu_type(mmu, NVIF_MEM_HOST | NVIF_MEM_MAPPABLE | |
166 | kind | NVIF_MEM_COHERENT); |
167 | if (typei < 0) |
168 | return -ENOSYS; |
169 | |
170 | drm->ttm.type_host[!!kind] = typei; |
171 | |
172 | typei = nvif_mmu_type(mmu, NVIF_MEM_HOST | NVIF_MEM_MAPPABLE | kind); |
173 | if (typei < 0) |
174 | return -ENOSYS; |
175 | |
176 | drm->ttm.type_ncoh[!!kind] = typei; |
177 | return 0; |
178 | } |
179 | |
180 | static int |
181 | nouveau_ttm_init_vram(struct nouveau_drm *drm) |
182 | { |
183 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { |
184 | struct ttm_resource_manager *man = kzalloc(size: sizeof(*man), GFP_KERNEL); |
185 | |
186 | if (!man) |
187 | return -ENOMEM; |
188 | |
189 | man->func = &nouveau_vram_manager; |
190 | |
191 | ttm_resource_manager_init(man, bdev: &drm->ttm.bdev, |
192 | size: drm->gem.vram_available >> PAGE_SHIFT); |
193 | ttm_set_driver_manager(bdev: &drm->ttm.bdev, TTM_PL_VRAM, manager: man); |
194 | ttm_resource_manager_set_used(man, used: true); |
195 | return 0; |
196 | } else { |
197 | return ttm_range_man_init(bdev: &drm->ttm.bdev, TTM_PL_VRAM, use_tt: false, |
198 | p_size: drm->gem.vram_available >> PAGE_SHIFT); |
199 | } |
200 | } |
201 | |
202 | static void |
203 | nouveau_ttm_fini_vram(struct nouveau_drm *drm) |
204 | { |
205 | struct ttm_resource_manager *man = ttm_manager_type(bdev: &drm->ttm.bdev, TTM_PL_VRAM); |
206 | |
207 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { |
208 | ttm_resource_manager_set_used(man, used: false); |
209 | ttm_resource_manager_evict_all(bdev: &drm->ttm.bdev, man); |
210 | ttm_resource_manager_cleanup(man); |
211 | ttm_set_driver_manager(bdev: &drm->ttm.bdev, TTM_PL_VRAM, NULL); |
212 | kfree(objp: man); |
213 | } else |
214 | ttm_range_man_fini(bdev: &drm->ttm.bdev, TTM_PL_VRAM); |
215 | } |
216 | |
217 | static int |
218 | nouveau_ttm_init_gtt(struct nouveau_drm *drm) |
219 | { |
220 | struct ttm_resource_manager *man; |
221 | unsigned long size_pages = drm->gem.gart_available >> PAGE_SHIFT; |
222 | const struct ttm_resource_manager_func *func = NULL; |
223 | |
224 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) |
225 | func = &nouveau_gart_manager; |
226 | else if (!drm->agp.bridge) |
227 | func = &nv04_gart_manager; |
228 | else |
229 | return ttm_range_man_init(bdev: &drm->ttm.bdev, TTM_PL_TT, use_tt: true, |
230 | p_size: size_pages); |
231 | |
232 | man = kzalloc(size: sizeof(*man), GFP_KERNEL); |
233 | if (!man) |
234 | return -ENOMEM; |
235 | |
236 | man->func = func; |
237 | man->use_tt = true; |
238 | ttm_resource_manager_init(man, bdev: &drm->ttm.bdev, size: size_pages); |
239 | ttm_set_driver_manager(bdev: &drm->ttm.bdev, TTM_PL_TT, manager: man); |
240 | ttm_resource_manager_set_used(man, used: true); |
241 | return 0; |
242 | } |
243 | |
244 | static void |
245 | nouveau_ttm_fini_gtt(struct nouveau_drm *drm) |
246 | { |
247 | struct ttm_resource_manager *man = ttm_manager_type(bdev: &drm->ttm.bdev, TTM_PL_TT); |
248 | |
249 | if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA && |
250 | drm->agp.bridge) |
251 | ttm_range_man_fini(bdev: &drm->ttm.bdev, TTM_PL_TT); |
252 | else { |
253 | ttm_resource_manager_set_used(man, used: false); |
254 | ttm_resource_manager_evict_all(bdev: &drm->ttm.bdev, man); |
255 | ttm_resource_manager_cleanup(man); |
256 | ttm_set_driver_manager(bdev: &drm->ttm.bdev, TTM_PL_TT, NULL); |
257 | kfree(objp: man); |
258 | } |
259 | } |
260 | |
261 | int |
262 | nouveau_ttm_init(struct nouveau_drm *drm) |
263 | { |
264 | struct nvkm_device *device = nvxx_device(&drm->client.device); |
265 | struct nvkm_pci *pci = device->pci; |
266 | struct nvif_mmu *mmu = &drm->client.mmu; |
267 | struct drm_device *dev = drm->dev; |
268 | int typei, ret; |
269 | |
270 | ret = nouveau_ttm_init_host(drm, kind: 0); |
271 | if (ret) |
272 | return ret; |
273 | |
274 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && |
275 | drm->client.device.info.chipset != 0x50) { |
276 | ret = nouveau_ttm_init_host(drm, kind: NVIF_MEM_KIND); |
277 | if (ret) |
278 | return ret; |
279 | } |
280 | |
281 | if (drm->client.device.info.platform != NV_DEVICE_INFO_V0_SOC && |
282 | drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { |
283 | typei = nvif_mmu_type(mmu, NVIF_MEM_VRAM | NVIF_MEM_MAPPABLE | |
284 | NVIF_MEM_KIND | |
285 | NVIF_MEM_COMP | |
286 | NVIF_MEM_DISP); |
287 | if (typei < 0) |
288 | return -ENOSYS; |
289 | |
290 | drm->ttm.type_vram = typei; |
291 | } else { |
292 | drm->ttm.type_vram = -1; |
293 | } |
294 | |
295 | if (pci && pci->agp.bridge) { |
296 | drm->agp.bridge = pci->agp.bridge; |
297 | drm->agp.base = pci->agp.base; |
298 | drm->agp.size = pci->agp.size; |
299 | drm->agp.cma = pci->agp.cma; |
300 | } |
301 | |
302 | ret = ttm_device_init(bdev: &drm->ttm.bdev, funcs: &nouveau_bo_driver, dev: drm->dev->dev, |
303 | mapping: dev->anon_inode->i_mapping, |
304 | vma_manager: dev->vma_offset_manager, |
305 | use_dma_alloc: drm_need_swiotlb(dma_bits: drm->client.mmu.dmabits), |
306 | use_dma32: drm->client.mmu.dmabits <= 32); |
307 | if (ret) { |
308 | NV_ERROR(drm, "error initialising bo driver, %d\n" , ret); |
309 | return ret; |
310 | } |
311 | |
312 | /* VRAM init */ |
313 | drm->gem.vram_available = drm->client.device.info.ram_user; |
314 | |
315 | arch_io_reserve_memtype_wc(start: device->func->resource_addr(device, 1), |
316 | size: device->func->resource_size(device, 1)); |
317 | |
318 | ret = nouveau_ttm_init_vram(drm); |
319 | if (ret) { |
320 | NV_ERROR(drm, "VRAM mm init failed, %d\n" , ret); |
321 | return ret; |
322 | } |
323 | |
324 | drm->ttm.mtrr = arch_phys_wc_add(base: device->func->resource_addr(device, 1), |
325 | size: device->func->resource_size(device, 1)); |
326 | |
327 | /* GART init */ |
328 | if (!drm->agp.bridge) { |
329 | drm->gem.gart_available = drm->client.vmm.vmm.limit; |
330 | } else { |
331 | drm->gem.gart_available = drm->agp.size; |
332 | } |
333 | |
334 | ret = nouveau_ttm_init_gtt(drm); |
335 | if (ret) { |
336 | NV_ERROR(drm, "GART mm init failed, %d\n" , ret); |
337 | return ret; |
338 | } |
339 | |
340 | mutex_init(&drm->ttm.io_reserve_mutex); |
341 | INIT_LIST_HEAD(list: &drm->ttm.io_reserve_lru); |
342 | |
343 | NV_INFO(drm, "VRAM: %d MiB\n" , (u32)(drm->gem.vram_available >> 20)); |
344 | NV_INFO(drm, "GART: %d MiB\n" , (u32)(drm->gem.gart_available >> 20)); |
345 | return 0; |
346 | } |
347 | |
348 | void |
349 | nouveau_ttm_fini(struct nouveau_drm *drm) |
350 | { |
351 | struct nvkm_device *device = nvxx_device(&drm->client.device); |
352 | |
353 | nouveau_ttm_fini_vram(drm); |
354 | nouveau_ttm_fini_gtt(drm); |
355 | |
356 | ttm_device_fini(bdev: &drm->ttm.bdev); |
357 | |
358 | arch_phys_wc_del(handle: drm->ttm.mtrr); |
359 | drm->ttm.mtrr = 0; |
360 | arch_io_free_memtype_wc(start: device->func->resource_addr(device, 1), |
361 | size: device->func->resource_size(device, 1)); |
362 | |
363 | } |
364 | |