1 | /* |
2 | * Copyright 2007 Dave Airlied |
3 | * All Rights Reserved. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice (including the next |
13 | * paragraph) shall be included in all copies or substantial portions of the |
14 | * Software. |
15 | * |
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
19 | * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
22 | * OTHER DEALINGS IN THE SOFTWARE. |
23 | */ |
24 | /* |
25 | * Authors: Dave Airlied <airlied@linux.ie> |
26 | * Ben Skeggs <darktama@iinet.net.au> |
27 | * Jeremy Kolb <jkolb@brandeis.edu> |
28 | */ |
29 | |
30 | #include <linux/dma-mapping.h> |
31 | #include <drm/ttm/ttm_tt.h> |
32 | |
33 | #include "nouveau_drv.h" |
34 | #include "nouveau_chan.h" |
35 | #include "nouveau_fence.h" |
36 | |
37 | #include "nouveau_bo.h" |
38 | #include "nouveau_ttm.h" |
39 | #include "nouveau_gem.h" |
40 | #include "nouveau_mem.h" |
41 | #include "nouveau_vmm.h" |
42 | |
43 | #include <nvif/class.h> |
44 | #include <nvif/if500b.h> |
45 | #include <nvif/if900b.h> |
46 | |
47 | static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, |
48 | struct ttm_resource *reg); |
49 | static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); |
50 | |
51 | /* |
52 | * NV10-NV40 tiling helpers |
53 | */ |
54 | |
55 | static void |
56 | nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, |
57 | u32 addr, u32 size, u32 pitch, u32 flags) |
58 | { |
59 | struct nouveau_drm *drm = nouveau_drm(dev); |
60 | int i = reg - drm->tile.reg; |
61 | struct nvkm_fb *fb = nvxx_fb(&drm->client.device); |
62 | struct nvkm_fb_tile *tile = &fb->tile.region[i]; |
63 | |
64 | nouveau_fence_unref(®->fence); |
65 | |
66 | if (tile->pitch) |
67 | nvkm_fb_tile_fini(fb, i, tile); |
68 | |
69 | if (pitch) |
70 | nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); |
71 | |
72 | nvkm_fb_tile_prog(fb, i, tile); |
73 | } |
74 | |
75 | static struct nouveau_drm_tile * |
76 | nv10_bo_get_tile_region(struct drm_device *dev, int i) |
77 | { |
78 | struct nouveau_drm *drm = nouveau_drm(dev); |
79 | struct nouveau_drm_tile *tile = &drm->tile.reg[i]; |
80 | |
81 | spin_lock(lock: &drm->tile.lock); |
82 | |
83 | if (!tile->used && |
84 | (!tile->fence || nouveau_fence_done(tile->fence))) |
85 | tile->used = true; |
86 | else |
87 | tile = NULL; |
88 | |
89 | spin_unlock(lock: &drm->tile.lock); |
90 | return tile; |
91 | } |
92 | |
93 | static void |
94 | nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, |
95 | struct dma_fence *fence) |
96 | { |
97 | struct nouveau_drm *drm = nouveau_drm(dev); |
98 | |
99 | if (tile) { |
100 | spin_lock(lock: &drm->tile.lock); |
101 | tile->fence = (struct nouveau_fence *)dma_fence_get(fence); |
102 | tile->used = false; |
103 | spin_unlock(lock: &drm->tile.lock); |
104 | } |
105 | } |
106 | |
107 | static struct nouveau_drm_tile * |
108 | nv10_bo_set_tiling(struct drm_device *dev, u32 addr, |
109 | u32 size, u32 pitch, u32 zeta) |
110 | { |
111 | struct nouveau_drm *drm = nouveau_drm(dev); |
112 | struct nvkm_fb *fb = nvxx_fb(&drm->client.device); |
113 | struct nouveau_drm_tile *tile, *found = NULL; |
114 | int i; |
115 | |
116 | for (i = 0; i < fb->tile.regions; i++) { |
117 | tile = nv10_bo_get_tile_region(dev, i); |
118 | |
119 | if (pitch && !found) { |
120 | found = tile; |
121 | continue; |
122 | |
123 | } else if (tile && fb->tile.region[i].pitch) { |
124 | /* Kill an unused tile region. */ |
125 | nv10_bo_update_tile_region(dev, reg: tile, addr: 0, size: 0, pitch: 0, flags: 0); |
126 | } |
127 | |
128 | nv10_bo_put_tile_region(dev, tile, NULL); |
129 | } |
130 | |
131 | if (found) |
132 | nv10_bo_update_tile_region(dev, reg: found, addr, size, pitch, flags: zeta); |
133 | return found; |
134 | } |
135 | |
136 | static void |
137 | nouveau_bo_del_ttm(struct ttm_buffer_object *bo) |
138 | { |
139 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
140 | struct drm_device *dev = drm->dev; |
141 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
142 | |
143 | WARN_ON(nvbo->bo.pin_count > 0); |
144 | nouveau_bo_del_io_reserve_lru(bo); |
145 | nv10_bo_put_tile_region(dev, tile: nvbo->tile, NULL); |
146 | |
147 | /* |
148 | * If nouveau_bo_new() allocated this buffer, the GEM object was never |
149 | * initialized, so don't attempt to release it. |
150 | */ |
151 | if (bo->base.dev) |
152 | drm_gem_object_release(obj: &bo->base); |
153 | else |
154 | dma_resv_fini(obj: &bo->base._resv); |
155 | |
156 | kfree(objp: nvbo); |
157 | } |
158 | |
159 | static inline u64 |
160 | roundup_64(u64 x, u32 y) |
161 | { |
162 | x += y - 1; |
163 | do_div(x, y); |
164 | return x * y; |
165 | } |
166 | |
167 | static void |
168 | nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size) |
169 | { |
170 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
171 | struct nvif_device *device = &drm->client.device; |
172 | |
173 | if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { |
174 | if (nvbo->mode) { |
175 | if (device->info.chipset >= 0x40) { |
176 | *align = 65536; |
177 | *size = roundup_64(x: *size, y: 64 * nvbo->mode); |
178 | |
179 | } else if (device->info.chipset >= 0x30) { |
180 | *align = 32768; |
181 | *size = roundup_64(x: *size, y: 64 * nvbo->mode); |
182 | |
183 | } else if (device->info.chipset >= 0x20) { |
184 | *align = 16384; |
185 | *size = roundup_64(x: *size, y: 64 * nvbo->mode); |
186 | |
187 | } else if (device->info.chipset >= 0x10) { |
188 | *align = 16384; |
189 | *size = roundup_64(x: *size, y: 32 * nvbo->mode); |
190 | } |
191 | } |
192 | } else { |
193 | *size = roundup_64(x: *size, y: (1 << nvbo->page)); |
194 | *align = max((1 << nvbo->page), *align); |
195 | } |
196 | |
197 | *size = roundup_64(x: *size, PAGE_SIZE); |
198 | } |
199 | |
200 | struct nouveau_bo * |
201 | nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain, |
202 | u32 tile_mode, u32 tile_flags, bool internal) |
203 | { |
204 | struct nouveau_drm *drm = cli->drm; |
205 | struct nouveau_bo *nvbo; |
206 | struct nvif_mmu *mmu = &cli->mmu; |
207 | struct nvif_vmm *vmm = &nouveau_cli_vmm(cli)->vmm; |
208 | int i, pi = -1; |
209 | |
210 | if (!*size) { |
211 | NV_WARN(drm, "skipped size %016llx\n" , *size); |
212 | return ERR_PTR(error: -EINVAL); |
213 | } |
214 | |
215 | nvbo = kzalloc(size: sizeof(struct nouveau_bo), GFP_KERNEL); |
216 | if (!nvbo) |
217 | return ERR_PTR(error: -ENOMEM); |
218 | |
219 | INIT_LIST_HEAD(list: &nvbo->head); |
220 | INIT_LIST_HEAD(list: &nvbo->entry); |
221 | INIT_LIST_HEAD(list: &nvbo->vma_list); |
222 | nvbo->bo.bdev = &drm->ttm.bdev; |
223 | |
224 | /* This is confusing, and doesn't actually mean we want an uncached |
225 | * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated |
226 | * into in nouveau_gem_new(). |
227 | */ |
228 | if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) { |
229 | /* Determine if we can get a cache-coherent map, forcing |
230 | * uncached mapping if we can't. |
231 | */ |
232 | if (!nouveau_drm_use_coherent_gpu_mapping(drm)) |
233 | nvbo->force_coherent = true; |
234 | } |
235 | |
236 | nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); |
237 | if (!nouveau_cli_uvmm(cli) || internal) { |
238 | /* for BO noVM allocs, don't assign kinds */ |
239 | if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { |
240 | nvbo->kind = (tile_flags & 0x0000ff00) >> 8; |
241 | if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { |
242 | kfree(objp: nvbo); |
243 | return ERR_PTR(error: -EINVAL); |
244 | } |
245 | |
246 | nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; |
247 | } else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { |
248 | nvbo->kind = (tile_flags & 0x00007f00) >> 8; |
249 | nvbo->comp = (tile_flags & 0x00030000) >> 16; |
250 | if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { |
251 | kfree(objp: nvbo); |
252 | return ERR_PTR(error: -EINVAL); |
253 | } |
254 | } else { |
255 | nvbo->zeta = (tile_flags & 0x00000007); |
256 | } |
257 | nvbo->mode = tile_mode; |
258 | |
259 | /* Determine the desirable target GPU page size for the buffer. */ |
260 | for (i = 0; i < vmm->page_nr; i++) { |
261 | /* Because we cannot currently allow VMM maps to fail |
262 | * during buffer migration, we need to determine page |
263 | * size for the buffer up-front, and pre-allocate its |
264 | * page tables. |
265 | * |
266 | * Skip page sizes that can't support needed domains. |
267 | */ |
268 | if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && |
269 | (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) |
270 | continue; |
271 | if ((domain & NOUVEAU_GEM_DOMAIN_GART) && |
272 | (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) |
273 | continue; |
274 | |
275 | /* Select this page size if it's the first that supports |
276 | * the potential memory domains, or when it's compatible |
277 | * with the requested compression settings. |
278 | */ |
279 | if (pi < 0 || !nvbo->comp || vmm->page[i].comp) |
280 | pi = i; |
281 | |
282 | /* Stop once the buffer is larger than the current page size. */ |
283 | if (*size >= 1ULL << vmm->page[i].shift) |
284 | break; |
285 | } |
286 | |
287 | if (WARN_ON(pi < 0)) { |
288 | kfree(objp: nvbo); |
289 | return ERR_PTR(error: -EINVAL); |
290 | } |
291 | |
292 | /* Disable compression if suitable settings couldn't be found. */ |
293 | if (nvbo->comp && !vmm->page[pi].comp) { |
294 | if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) |
295 | nvbo->kind = mmu->kind[nvbo->kind]; |
296 | nvbo->comp = 0; |
297 | } |
298 | nvbo->page = vmm->page[pi].shift; |
299 | } else { |
300 | /* reject other tile flags when in VM mode. */ |
301 | if (tile_mode) |
302 | return ERR_PTR(error: -EINVAL); |
303 | if (tile_flags & ~NOUVEAU_GEM_TILE_NONCONTIG) |
304 | return ERR_PTR(error: -EINVAL); |
305 | |
306 | /* Determine the desirable target GPU page size for the buffer. */ |
307 | for (i = 0; i < vmm->page_nr; i++) { |
308 | /* Because we cannot currently allow VMM maps to fail |
309 | * during buffer migration, we need to determine page |
310 | * size for the buffer up-front, and pre-allocate its |
311 | * page tables. |
312 | * |
313 | * Skip page sizes that can't support needed domains. |
314 | */ |
315 | if ((domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) |
316 | continue; |
317 | if ((domain & NOUVEAU_GEM_DOMAIN_GART) && |
318 | (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) |
319 | continue; |
320 | |
321 | if (pi < 0) |
322 | pi = i; |
323 | /* Stop once the buffer is larger than the current page size. */ |
324 | if (*size >= 1ULL << vmm->page[i].shift) |
325 | break; |
326 | } |
327 | if (WARN_ON(pi < 0)) { |
328 | kfree(objp: nvbo); |
329 | return ERR_PTR(error: -EINVAL); |
330 | } |
331 | nvbo->page = vmm->page[pi].shift; |
332 | } |
333 | |
334 | nouveau_bo_fixup_align(nvbo, align, size); |
335 | |
336 | return nvbo; |
337 | } |
338 | |
339 | int |
340 | nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, |
341 | struct sg_table *sg, struct dma_resv *robj) |
342 | { |
343 | int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; |
344 | int ret; |
345 | struct ttm_operation_ctx ctx = { |
346 | .interruptible = false, |
347 | .no_wait_gpu = false, |
348 | .resv = robj, |
349 | }; |
350 | |
351 | nouveau_bo_placement_set(nvbo, type: domain, busy: 0); |
352 | INIT_LIST_HEAD(list: &nvbo->io_reserve_lru); |
353 | |
354 | ret = ttm_bo_init_reserved(bdev: nvbo->bo.bdev, bo: &nvbo->bo, type, |
355 | placement: &nvbo->placement, alignment: align >> PAGE_SHIFT, ctx: &ctx, |
356 | sg, resv: robj, destroy: nouveau_bo_del_ttm); |
357 | if (ret) { |
358 | /* ttm will call nouveau_bo_del_ttm if it fails.. */ |
359 | return ret; |
360 | } |
361 | |
362 | if (!robj) |
363 | ttm_bo_unreserve(bo: &nvbo->bo); |
364 | |
365 | return 0; |
366 | } |
367 | |
368 | int |
369 | nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, |
370 | uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, |
371 | struct sg_table *sg, struct dma_resv *robj, |
372 | struct nouveau_bo **pnvbo) |
373 | { |
374 | struct nouveau_bo *nvbo; |
375 | int ret; |
376 | |
377 | nvbo = nouveau_bo_alloc(cli, size: &size, align: &align, domain, tile_mode, |
378 | tile_flags, internal: true); |
379 | if (IS_ERR(ptr: nvbo)) |
380 | return PTR_ERR(ptr: nvbo); |
381 | |
382 | nvbo->bo.base.size = size; |
383 | dma_resv_init(obj: &nvbo->bo.base._resv); |
384 | drm_vma_node_reset(node: &nvbo->bo.base.vma_node); |
385 | |
386 | /* This must be called before ttm_bo_init_reserved(). Subsequent |
387 | * bo_move() callbacks might already iterate the GEMs GPUVA list. |
388 | */ |
389 | drm_gem_gpuva_init(obj: &nvbo->bo.base); |
390 | |
391 | ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj); |
392 | if (ret) |
393 | return ret; |
394 | |
395 | *pnvbo = nvbo; |
396 | return 0; |
397 | } |
398 | |
399 | static void |
400 | set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t domain) |
401 | { |
402 | *n = 0; |
403 | |
404 | if (domain & NOUVEAU_GEM_DOMAIN_VRAM) { |
405 | pl[*n].mem_type = TTM_PL_VRAM; |
406 | pl[*n].flags = 0; |
407 | (*n)++; |
408 | } |
409 | if (domain & NOUVEAU_GEM_DOMAIN_GART) { |
410 | pl[*n].mem_type = TTM_PL_TT; |
411 | pl[*n].flags = 0; |
412 | (*n)++; |
413 | } |
414 | if (domain & NOUVEAU_GEM_DOMAIN_CPU) { |
415 | pl[*n].mem_type = TTM_PL_SYSTEM; |
416 | pl[(*n)++].flags = 0; |
417 | } |
418 | } |
419 | |
420 | static void |
421 | set_placement_range(struct nouveau_bo *nvbo, uint32_t domain) |
422 | { |
423 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
424 | u64 vram_size = drm->client.device.info.ram_size; |
425 | unsigned i, fpfn, lpfn; |
426 | |
427 | if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && |
428 | nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) && |
429 | nvbo->bo.base.size < vram_size / 4) { |
430 | /* |
431 | * Make sure that the color and depth buffers are handled |
432 | * by independent memory controller units. Up to a 9x |
433 | * speed up when alpha-blending and depth-test are enabled |
434 | * at the same time. |
435 | */ |
436 | if (nvbo->zeta) { |
437 | fpfn = (vram_size / 2) >> PAGE_SHIFT; |
438 | lpfn = ~0; |
439 | } else { |
440 | fpfn = 0; |
441 | lpfn = (vram_size / 2) >> PAGE_SHIFT; |
442 | } |
443 | for (i = 0; i < nvbo->placement.num_placement; ++i) { |
444 | nvbo->placements[i].fpfn = fpfn; |
445 | nvbo->placements[i].lpfn = lpfn; |
446 | } |
447 | for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { |
448 | nvbo->busy_placements[i].fpfn = fpfn; |
449 | nvbo->busy_placements[i].lpfn = lpfn; |
450 | } |
451 | } |
452 | } |
453 | |
454 | void |
455 | nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain, |
456 | uint32_t busy) |
457 | { |
458 | struct ttm_placement *pl = &nvbo->placement; |
459 | |
460 | pl->placement = nvbo->placements; |
461 | set_placement_list(pl: nvbo->placements, n: &pl->num_placement, domain); |
462 | |
463 | pl->busy_placement = nvbo->busy_placements; |
464 | set_placement_list(pl: nvbo->busy_placements, n: &pl->num_busy_placement, |
465 | domain: domain | busy); |
466 | |
467 | set_placement_range(nvbo, domain); |
468 | } |
469 | |
470 | int |
471 | nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) |
472 | { |
473 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
474 | struct ttm_buffer_object *bo = &nvbo->bo; |
475 | bool force = false, evict = false; |
476 | int ret; |
477 | |
478 | ret = ttm_bo_reserve(bo, interruptible: false, no_wait: false, NULL); |
479 | if (ret) |
480 | return ret; |
481 | |
482 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && |
483 | domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) { |
484 | if (!nvbo->contig) { |
485 | nvbo->contig = true; |
486 | force = true; |
487 | evict = true; |
488 | } |
489 | } |
490 | |
491 | if (nvbo->bo.pin_count) { |
492 | bool error = evict; |
493 | |
494 | switch (bo->resource->mem_type) { |
495 | case TTM_PL_VRAM: |
496 | error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM); |
497 | break; |
498 | case TTM_PL_TT: |
499 | error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); |
500 | break; |
501 | default: |
502 | break; |
503 | } |
504 | |
505 | if (error) { |
506 | NV_ERROR(drm, "bo %p pinned elsewhere: " |
507 | "0x%08x vs 0x%08x\n" , bo, |
508 | bo->resource->mem_type, domain); |
509 | ret = -EBUSY; |
510 | } |
511 | ttm_bo_pin(bo: &nvbo->bo); |
512 | goto out; |
513 | } |
514 | |
515 | if (evict) { |
516 | nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, busy: 0); |
517 | ret = nouveau_bo_validate(nvbo, interruptible: false, no_wait_gpu: false); |
518 | if (ret) |
519 | goto out; |
520 | } |
521 | |
522 | nouveau_bo_placement_set(nvbo, domain, busy: 0); |
523 | ret = nouveau_bo_validate(nvbo, interruptible: false, no_wait_gpu: false); |
524 | if (ret) |
525 | goto out; |
526 | |
527 | ttm_bo_pin(bo: &nvbo->bo); |
528 | |
529 | switch (bo->resource->mem_type) { |
530 | case TTM_PL_VRAM: |
531 | drm->gem.vram_available -= bo->base.size; |
532 | break; |
533 | case TTM_PL_TT: |
534 | drm->gem.gart_available -= bo->base.size; |
535 | break; |
536 | default: |
537 | break; |
538 | } |
539 | |
540 | out: |
541 | if (force && ret) |
542 | nvbo->contig = false; |
543 | ttm_bo_unreserve(bo); |
544 | return ret; |
545 | } |
546 | |
547 | int |
548 | nouveau_bo_unpin(struct nouveau_bo *nvbo) |
549 | { |
550 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
551 | struct ttm_buffer_object *bo = &nvbo->bo; |
552 | int ret; |
553 | |
554 | ret = ttm_bo_reserve(bo, interruptible: false, no_wait: false, NULL); |
555 | if (ret) |
556 | return ret; |
557 | |
558 | ttm_bo_unpin(bo: &nvbo->bo); |
559 | if (!nvbo->bo.pin_count) { |
560 | switch (bo->resource->mem_type) { |
561 | case TTM_PL_VRAM: |
562 | drm->gem.vram_available += bo->base.size; |
563 | break; |
564 | case TTM_PL_TT: |
565 | drm->gem.gart_available += bo->base.size; |
566 | break; |
567 | default: |
568 | break; |
569 | } |
570 | } |
571 | |
572 | ttm_bo_unreserve(bo); |
573 | return 0; |
574 | } |
575 | |
576 | int |
577 | nouveau_bo_map(struct nouveau_bo *nvbo) |
578 | { |
579 | int ret; |
580 | |
581 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: false, no_wait: false, NULL); |
582 | if (ret) |
583 | return ret; |
584 | |
585 | ret = ttm_bo_kmap(bo: &nvbo->bo, start_page: 0, PFN_UP(nvbo->bo.base.size), map: &nvbo->kmap); |
586 | |
587 | ttm_bo_unreserve(bo: &nvbo->bo); |
588 | return ret; |
589 | } |
590 | |
591 | void |
592 | nouveau_bo_unmap(struct nouveau_bo *nvbo) |
593 | { |
594 | if (!nvbo) |
595 | return; |
596 | |
597 | ttm_bo_kunmap(map: &nvbo->kmap); |
598 | } |
599 | |
600 | void |
601 | nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) |
602 | { |
603 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
604 | struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; |
605 | int i, j; |
606 | |
607 | if (!ttm_dma || !ttm_dma->dma_address) |
608 | return; |
609 | if (!ttm_dma->pages) { |
610 | NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n" , ttm_dma); |
611 | return; |
612 | } |
613 | |
614 | /* Don't waste time looping if the object is coherent */ |
615 | if (nvbo->force_coherent) |
616 | return; |
617 | |
618 | i = 0; |
619 | while (i < ttm_dma->num_pages) { |
620 | struct page *p = ttm_dma->pages[i]; |
621 | size_t num_pages = 1; |
622 | |
623 | for (j = i + 1; j < ttm_dma->num_pages; ++j) { |
624 | if (++p != ttm_dma->pages[j]) |
625 | break; |
626 | |
627 | ++num_pages; |
628 | } |
629 | dma_sync_single_for_device(dev: drm->dev->dev, |
630 | addr: ttm_dma->dma_address[i], |
631 | size: num_pages * PAGE_SIZE, dir: DMA_TO_DEVICE); |
632 | i += num_pages; |
633 | } |
634 | } |
635 | |
636 | void |
637 | nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) |
638 | { |
639 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
640 | struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; |
641 | int i, j; |
642 | |
643 | if (!ttm_dma || !ttm_dma->dma_address) |
644 | return; |
645 | if (!ttm_dma->pages) { |
646 | NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n" , ttm_dma); |
647 | return; |
648 | } |
649 | |
650 | /* Don't waste time looping if the object is coherent */ |
651 | if (nvbo->force_coherent) |
652 | return; |
653 | |
654 | i = 0; |
655 | while (i < ttm_dma->num_pages) { |
656 | struct page *p = ttm_dma->pages[i]; |
657 | size_t num_pages = 1; |
658 | |
659 | for (j = i + 1; j < ttm_dma->num_pages; ++j) { |
660 | if (++p != ttm_dma->pages[j]) |
661 | break; |
662 | |
663 | ++num_pages; |
664 | } |
665 | |
666 | dma_sync_single_for_cpu(dev: drm->dev->dev, addr: ttm_dma->dma_address[i], |
667 | size: num_pages * PAGE_SIZE, dir: DMA_FROM_DEVICE); |
668 | i += num_pages; |
669 | } |
670 | } |
671 | |
672 | void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo) |
673 | { |
674 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
675 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
676 | |
677 | mutex_lock(&drm->ttm.io_reserve_mutex); |
678 | list_move_tail(list: &nvbo->io_reserve_lru, head: &drm->ttm.io_reserve_lru); |
679 | mutex_unlock(lock: &drm->ttm.io_reserve_mutex); |
680 | } |
681 | |
682 | void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo) |
683 | { |
684 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
685 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
686 | |
687 | mutex_lock(&drm->ttm.io_reserve_mutex); |
688 | list_del_init(entry: &nvbo->io_reserve_lru); |
689 | mutex_unlock(lock: &drm->ttm.io_reserve_mutex); |
690 | } |
691 | |
692 | int |
693 | nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, |
694 | bool no_wait_gpu) |
695 | { |
696 | struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; |
697 | int ret; |
698 | |
699 | ret = ttm_bo_validate(bo: &nvbo->bo, placement: &nvbo->placement, ctx: &ctx); |
700 | if (ret) |
701 | return ret; |
702 | |
703 | nouveau_bo_sync_for_device(nvbo); |
704 | |
705 | return 0; |
706 | } |
707 | |
708 | void |
709 | nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) |
710 | { |
711 | bool is_iomem; |
712 | u16 *mem = ttm_kmap_obj_virtual(map: &nvbo->kmap, is_iomem: &is_iomem); |
713 | |
714 | mem += index; |
715 | |
716 | if (is_iomem) |
717 | iowrite16_native(val, (void __force __iomem *)mem); |
718 | else |
719 | *mem = val; |
720 | } |
721 | |
722 | u32 |
723 | nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) |
724 | { |
725 | bool is_iomem; |
726 | u32 *mem = ttm_kmap_obj_virtual(map: &nvbo->kmap, is_iomem: &is_iomem); |
727 | |
728 | mem += index; |
729 | |
730 | if (is_iomem) |
731 | return ioread32_native((void __force __iomem *)mem); |
732 | else |
733 | return *mem; |
734 | } |
735 | |
736 | void |
737 | nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) |
738 | { |
739 | bool is_iomem; |
740 | u32 *mem = ttm_kmap_obj_virtual(map: &nvbo->kmap, is_iomem: &is_iomem); |
741 | |
742 | mem += index; |
743 | |
744 | if (is_iomem) |
745 | iowrite32_native(val, (void __force __iomem *)mem); |
746 | else |
747 | *mem = val; |
748 | } |
749 | |
750 | static struct ttm_tt * |
751 | nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) |
752 | { |
753 | #if IS_ENABLED(CONFIG_AGP) |
754 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
755 | |
756 | if (drm->agp.bridge) { |
757 | return ttm_agp_tt_create(bo, bridge: drm->agp.bridge, page_flags); |
758 | } |
759 | #endif |
760 | |
761 | return nouveau_sgdma_create_ttm(bo, page_flags); |
762 | } |
763 | |
764 | static int |
765 | nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, |
766 | struct ttm_resource *reg) |
767 | { |
768 | #if IS_ENABLED(CONFIG_AGP) |
769 | struct nouveau_drm *drm = nouveau_bdev(bd: bdev); |
770 | #endif |
771 | if (!reg) |
772 | return -EINVAL; |
773 | #if IS_ENABLED(CONFIG_AGP) |
774 | if (drm->agp.bridge) |
775 | return ttm_agp_bind(ttm, bo_mem: reg); |
776 | #endif |
777 | return nouveau_sgdma_bind(bdev, ttm, reg); |
778 | } |
779 | |
780 | static void |
781 | nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) |
782 | { |
783 | #if IS_ENABLED(CONFIG_AGP) |
784 | struct nouveau_drm *drm = nouveau_bdev(bd: bdev); |
785 | |
786 | if (drm->agp.bridge) { |
787 | ttm_agp_unbind(ttm); |
788 | return; |
789 | } |
790 | #endif |
791 | nouveau_sgdma_unbind(bdev, ttm); |
792 | } |
793 | |
794 | static void |
795 | nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) |
796 | { |
797 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
798 | |
799 | switch (bo->resource->mem_type) { |
800 | case TTM_PL_VRAM: |
801 | nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, |
802 | NOUVEAU_GEM_DOMAIN_CPU); |
803 | break; |
804 | default: |
805 | nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, busy: 0); |
806 | break; |
807 | } |
808 | |
809 | *pl = nvbo->placement; |
810 | } |
811 | |
812 | static int |
813 | nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, |
814 | struct ttm_resource *reg) |
815 | { |
816 | struct nouveau_mem *old_mem = nouveau_mem(reg: bo->resource); |
817 | struct nouveau_mem *new_mem = nouveau_mem(reg); |
818 | struct nvif_vmm *vmm = &drm->client.vmm.vmm; |
819 | int ret; |
820 | |
821 | ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, |
822 | old_mem->mem.size, &old_mem->vma[0]); |
823 | if (ret) |
824 | return ret; |
825 | |
826 | ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, |
827 | new_mem->mem.size, &old_mem->vma[1]); |
828 | if (ret) |
829 | goto done; |
830 | |
831 | ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); |
832 | if (ret) |
833 | goto done; |
834 | |
835 | ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); |
836 | done: |
837 | if (ret) { |
838 | nvif_vmm_put(vmm, &old_mem->vma[1]); |
839 | nvif_vmm_put(vmm, &old_mem->vma[0]); |
840 | } |
841 | return 0; |
842 | } |
843 | |
844 | static int |
845 | nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, |
846 | struct ttm_operation_ctx *ctx, |
847 | struct ttm_resource *new_reg) |
848 | { |
849 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
850 | struct nouveau_channel *chan = drm->ttm.chan; |
851 | struct nouveau_cli *cli = (void *)chan->user.client; |
852 | struct nouveau_fence *fence; |
853 | int ret; |
854 | |
855 | /* create temporary vmas for the transfer and attach them to the |
856 | * old nvkm_mem node, these will get cleaned up after ttm has |
857 | * destroyed the ttm_resource |
858 | */ |
859 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { |
860 | ret = nouveau_bo_move_prep(drm, bo, reg: new_reg); |
861 | if (ret) |
862 | return ret; |
863 | } |
864 | |
865 | if (drm_drv_uses_atomic_modeset(dev: drm->dev)) |
866 | mutex_lock(&cli->mutex); |
867 | else |
868 | mutex_lock_nested(lock: &cli->mutex, SINGLE_DEPTH_NESTING); |
869 | |
870 | ret = nouveau_fence_sync(nouveau_bo(bo), chan, exclusive: true, intr: ctx->interruptible); |
871 | if (ret) |
872 | goto out_unlock; |
873 | |
874 | ret = drm->ttm.move(chan, bo, bo->resource, new_reg); |
875 | if (ret) |
876 | goto out_unlock; |
877 | |
878 | ret = nouveau_fence_new(&fence, chan); |
879 | if (ret) |
880 | goto out_unlock; |
881 | |
882 | /* TODO: figure out a better solution here |
883 | * |
884 | * wait on the fence here explicitly as going through |
885 | * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. |
886 | * |
887 | * Without this the operation can timeout and we'll fallback to a |
888 | * software copy, which might take several minutes to finish. |
889 | */ |
890 | nouveau_fence_wait(fence, lazy: false, intr: false); |
891 | ret = ttm_bo_move_accel_cleanup(bo, fence: &fence->base, evict, pipeline: false, |
892 | new_mem: new_reg); |
893 | nouveau_fence_unref(&fence); |
894 | |
895 | out_unlock: |
896 | mutex_unlock(lock: &cli->mutex); |
897 | return ret; |
898 | } |
899 | |
900 | void |
901 | nouveau_bo_move_init(struct nouveau_drm *drm) |
902 | { |
903 | static const struct _method_table { |
904 | const char *name; |
905 | int engine; |
906 | s32 oclass; |
907 | int (*exec)(struct nouveau_channel *, |
908 | struct ttm_buffer_object *, |
909 | struct ttm_resource *, struct ttm_resource *); |
910 | int (*init)(struct nouveau_channel *, u32 handle); |
911 | } _methods[] = { |
912 | { "COPY" , 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, |
913 | { "GRCE" , 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
914 | { "COPY" , 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, |
915 | { "GRCE" , 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
916 | { "COPY" , 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, |
917 | { "GRCE" , 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
918 | { "COPY" , 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, |
919 | { "GRCE" , 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
920 | { "COPY" , 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init }, |
921 | { "GRCE" , 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
922 | { "COPY" , 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init }, |
923 | { "GRCE" , 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
924 | { "COPY" , 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, |
925 | { "GRCE" , 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
926 | { "COPY" , 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, |
927 | { "GRCE" , 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, |
928 | { "COPY1" , 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, |
929 | { "COPY0" , 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, |
930 | { "COPY" , 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init }, |
931 | { "CRYPT" , 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init }, |
932 | { "M2MF" , 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init }, |
933 | { "M2MF" , 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init }, |
934 | { "M2MF" , 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init }, |
935 | {}, |
936 | }; |
937 | const struct _method_table *mthd = _methods; |
938 | const char *name = "CPU" ; |
939 | int ret; |
940 | |
941 | do { |
942 | struct nouveau_channel *chan; |
943 | |
944 | if (mthd->engine) |
945 | chan = drm->cechan; |
946 | else |
947 | chan = drm->channel; |
948 | if (chan == NULL) |
949 | continue; |
950 | |
951 | ret = nvif_object_ctor(&chan->user, "ttmBoMove" , |
952 | mthd->oclass | (mthd->engine << 16), |
953 | mthd->oclass, NULL, 0, |
954 | &drm->ttm.copy); |
955 | if (ret == 0) { |
956 | ret = mthd->init(chan, drm->ttm.copy.handle); |
957 | if (ret) { |
958 | nvif_object_dtor(&drm->ttm.copy); |
959 | continue; |
960 | } |
961 | |
962 | drm->ttm.move = mthd->exec; |
963 | drm->ttm.chan = chan; |
964 | name = mthd->name; |
965 | break; |
966 | } |
967 | } while ((++mthd)->exec); |
968 | |
969 | NV_INFO(drm, "MM: using %s for buffer copies\n" , name); |
970 | } |
971 | |
972 | static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, |
973 | struct ttm_resource *new_reg) |
974 | { |
975 | struct nouveau_mem *mem = new_reg ? nouveau_mem(reg: new_reg) : NULL; |
976 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
977 | struct nouveau_vma *vma; |
978 | long ret; |
979 | |
980 | /* ttm can now (stupidly) pass the driver bos it didn't create... */ |
981 | if (bo->destroy != nouveau_bo_del_ttm) |
982 | return; |
983 | |
984 | nouveau_bo_del_io_reserve_lru(bo); |
985 | |
986 | if (mem && new_reg->mem_type != TTM_PL_SYSTEM && |
987 | mem->mem.page == nvbo->page) { |
988 | list_for_each_entry(vma, &nvbo->vma_list, head) { |
989 | nouveau_vma_map(vma, mem); |
990 | } |
991 | nouveau_uvmm_bo_map_all(nvbov: nvbo, mem); |
992 | } else { |
993 | list_for_each_entry(vma, &nvbo->vma_list, head) { |
994 | ret = dma_resv_wait_timeout(obj: bo->base.resv, |
995 | usage: DMA_RESV_USAGE_BOOKKEEP, |
996 | intr: false, timeout: 15 * HZ); |
997 | WARN_ON(ret <= 0); |
998 | nouveau_vma_unmap(vma); |
999 | } |
1000 | nouveau_uvmm_bo_unmap_all(nvbo); |
1001 | } |
1002 | |
1003 | if (new_reg) |
1004 | nvbo->offset = (new_reg->start << PAGE_SHIFT); |
1005 | |
1006 | } |
1007 | |
1008 | static int |
1009 | nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg, |
1010 | struct nouveau_drm_tile **new_tile) |
1011 | { |
1012 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
1013 | struct drm_device *dev = drm->dev; |
1014 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
1015 | u64 offset = new_reg->start << PAGE_SHIFT; |
1016 | |
1017 | *new_tile = NULL; |
1018 | if (new_reg->mem_type != TTM_PL_VRAM) |
1019 | return 0; |
1020 | |
1021 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { |
1022 | *new_tile = nv10_bo_set_tiling(dev, addr: offset, size: bo->base.size, |
1023 | pitch: nvbo->mode, zeta: nvbo->zeta); |
1024 | } |
1025 | |
1026 | return 0; |
1027 | } |
1028 | |
1029 | static void |
1030 | nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, |
1031 | struct nouveau_drm_tile *new_tile, |
1032 | struct nouveau_drm_tile **old_tile) |
1033 | { |
1034 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
1035 | struct drm_device *dev = drm->dev; |
1036 | struct dma_fence *fence; |
1037 | int ret; |
1038 | |
1039 | ret = dma_resv_get_singleton(obj: bo->base.resv, usage: DMA_RESV_USAGE_WRITE, |
1040 | fence: &fence); |
1041 | if (ret) |
1042 | dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_WRITE, |
1043 | intr: false, MAX_SCHEDULE_TIMEOUT); |
1044 | |
1045 | nv10_bo_put_tile_region(dev, tile: *old_tile, fence); |
1046 | *old_tile = new_tile; |
1047 | } |
1048 | |
1049 | static int |
1050 | nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, |
1051 | struct ttm_operation_ctx *ctx, |
1052 | struct ttm_resource *new_reg, |
1053 | struct ttm_place *hop) |
1054 | { |
1055 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
1056 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
1057 | struct ttm_resource *old_reg = bo->resource; |
1058 | struct nouveau_drm_tile *new_tile = NULL; |
1059 | int ret = 0; |
1060 | |
1061 | |
1062 | if (new_reg->mem_type == TTM_PL_TT) { |
1063 | ret = nouveau_ttm_tt_bind(bdev: bo->bdev, ttm: bo->ttm, reg: new_reg); |
1064 | if (ret) |
1065 | return ret; |
1066 | } |
1067 | |
1068 | nouveau_bo_move_ntfy(bo, new_reg); |
1069 | ret = ttm_bo_wait_ctx(bo, ctx); |
1070 | if (ret) |
1071 | goto out_ntfy; |
1072 | |
1073 | if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { |
1074 | ret = nouveau_bo_vm_bind(bo, new_reg, new_tile: &new_tile); |
1075 | if (ret) |
1076 | goto out_ntfy; |
1077 | } |
1078 | |
1079 | /* Fake bo copy. */ |
1080 | if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM && |
1081 | !bo->ttm)) { |
1082 | ttm_bo_move_null(bo, new_mem: new_reg); |
1083 | goto out; |
1084 | } |
1085 | |
1086 | if (old_reg->mem_type == TTM_PL_SYSTEM && |
1087 | new_reg->mem_type == TTM_PL_TT) { |
1088 | ttm_bo_move_null(bo, new_mem: new_reg); |
1089 | goto out; |
1090 | } |
1091 | |
1092 | if (old_reg->mem_type == TTM_PL_TT && |
1093 | new_reg->mem_type == TTM_PL_SYSTEM) { |
1094 | nouveau_ttm_tt_unbind(bdev: bo->bdev, ttm: bo->ttm); |
1095 | ttm_resource_free(bo, res: &bo->resource); |
1096 | ttm_bo_assign_mem(bo, new_mem: new_reg); |
1097 | goto out; |
1098 | } |
1099 | |
1100 | /* Hardware assisted copy. */ |
1101 | if (drm->ttm.move) { |
1102 | if ((old_reg->mem_type == TTM_PL_SYSTEM && |
1103 | new_reg->mem_type == TTM_PL_VRAM) || |
1104 | (old_reg->mem_type == TTM_PL_VRAM && |
1105 | new_reg->mem_type == TTM_PL_SYSTEM)) { |
1106 | hop->fpfn = 0; |
1107 | hop->lpfn = 0; |
1108 | hop->mem_type = TTM_PL_TT; |
1109 | hop->flags = 0; |
1110 | return -EMULTIHOP; |
1111 | } |
1112 | ret = nouveau_bo_move_m2mf(bo, evict, ctx, |
1113 | new_reg); |
1114 | } else |
1115 | ret = -ENODEV; |
1116 | |
1117 | if (ret) { |
1118 | /* Fallback to software copy. */ |
1119 | ret = ttm_bo_move_memcpy(bo, ctx, new_mem: new_reg); |
1120 | } |
1121 | |
1122 | out: |
1123 | if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { |
1124 | if (ret) |
1125 | nouveau_bo_vm_cleanup(bo, NULL, old_tile: &new_tile); |
1126 | else |
1127 | nouveau_bo_vm_cleanup(bo, new_tile, old_tile: &nvbo->tile); |
1128 | } |
1129 | out_ntfy: |
1130 | if (ret) { |
1131 | nouveau_bo_move_ntfy(bo, new_reg: bo->resource); |
1132 | } |
1133 | return ret; |
1134 | } |
1135 | |
1136 | static void |
1137 | nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, |
1138 | struct ttm_resource *reg) |
1139 | { |
1140 | struct nouveau_mem *mem = nouveau_mem(reg); |
1141 | |
1142 | if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { |
1143 | switch (reg->mem_type) { |
1144 | case TTM_PL_TT: |
1145 | if (mem->kind) |
1146 | nvif_object_unmap_handle(&mem->mem.object); |
1147 | break; |
1148 | case TTM_PL_VRAM: |
1149 | nvif_object_unmap_handle(&mem->mem.object); |
1150 | break; |
1151 | default: |
1152 | break; |
1153 | } |
1154 | } |
1155 | } |
1156 | |
1157 | static int |
1158 | nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg) |
1159 | { |
1160 | struct nouveau_drm *drm = nouveau_bdev(bd: bdev); |
1161 | struct nvkm_device *device = nvxx_device(&drm->client.device); |
1162 | struct nouveau_mem *mem = nouveau_mem(reg); |
1163 | struct nvif_mmu *mmu = &drm->client.mmu; |
1164 | int ret; |
1165 | |
1166 | mutex_lock(&drm->ttm.io_reserve_mutex); |
1167 | retry: |
1168 | switch (reg->mem_type) { |
1169 | case TTM_PL_SYSTEM: |
1170 | /* System memory */ |
1171 | ret = 0; |
1172 | goto out; |
1173 | case TTM_PL_TT: |
1174 | #if IS_ENABLED(CONFIG_AGP) |
1175 | if (drm->agp.bridge) { |
1176 | reg->bus.offset = (reg->start << PAGE_SHIFT) + |
1177 | drm->agp.base; |
1178 | reg->bus.is_iomem = !drm->agp.cma; |
1179 | reg->bus.caching = ttm_write_combined; |
1180 | } |
1181 | #endif |
1182 | if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || |
1183 | !mem->kind) { |
1184 | /* untiled */ |
1185 | ret = 0; |
1186 | break; |
1187 | } |
1188 | fallthrough; /* tiled memory */ |
1189 | case TTM_PL_VRAM: |
1190 | reg->bus.offset = (reg->start << PAGE_SHIFT) + |
1191 | device->func->resource_addr(device, 1); |
1192 | reg->bus.is_iomem = true; |
1193 | |
1194 | /* Some BARs do not support being ioremapped WC */ |
1195 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && |
1196 | mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED) |
1197 | reg->bus.caching = ttm_uncached; |
1198 | else |
1199 | reg->bus.caching = ttm_write_combined; |
1200 | |
1201 | if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { |
1202 | union { |
1203 | struct nv50_mem_map_v0 nv50; |
1204 | struct gf100_mem_map_v0 gf100; |
1205 | } args; |
1206 | u64 handle, length; |
1207 | u32 argc = 0; |
1208 | |
1209 | switch (mem->mem.object.oclass) { |
1210 | case NVIF_CLASS_MEM_NV50: |
1211 | args.nv50.version = 0; |
1212 | args.nv50.ro = 0; |
1213 | args.nv50.kind = mem->kind; |
1214 | args.nv50.comp = mem->comp; |
1215 | argc = sizeof(args.nv50); |
1216 | break; |
1217 | case NVIF_CLASS_MEM_GF100: |
1218 | args.gf100.version = 0; |
1219 | args.gf100.ro = 0; |
1220 | args.gf100.kind = mem->kind; |
1221 | argc = sizeof(args.gf100); |
1222 | break; |
1223 | default: |
1224 | WARN_ON(1); |
1225 | break; |
1226 | } |
1227 | |
1228 | ret = nvif_object_map_handle(&mem->mem.object, |
1229 | &args, argc, |
1230 | &handle, &length); |
1231 | if (ret != 1) { |
1232 | if (WARN_ON(ret == 0)) |
1233 | ret = -EINVAL; |
1234 | goto out; |
1235 | } |
1236 | |
1237 | reg->bus.offset = handle; |
1238 | } |
1239 | ret = 0; |
1240 | break; |
1241 | default: |
1242 | ret = -EINVAL; |
1243 | } |
1244 | |
1245 | out: |
1246 | if (ret == -ENOSPC) { |
1247 | struct nouveau_bo *nvbo; |
1248 | |
1249 | nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru, |
1250 | typeof(*nvbo), |
1251 | io_reserve_lru); |
1252 | if (nvbo) { |
1253 | list_del_init(entry: &nvbo->io_reserve_lru); |
1254 | drm_vma_node_unmap(node: &nvbo->bo.base.vma_node, |
1255 | file_mapping: bdev->dev_mapping); |
1256 | nouveau_ttm_io_mem_free_locked(drm, reg: nvbo->bo.resource); |
1257 | goto retry; |
1258 | } |
1259 | |
1260 | } |
1261 | mutex_unlock(lock: &drm->ttm.io_reserve_mutex); |
1262 | return ret; |
1263 | } |
1264 | |
1265 | static void |
1266 | nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg) |
1267 | { |
1268 | struct nouveau_drm *drm = nouveau_bdev(bd: bdev); |
1269 | |
1270 | mutex_lock(&drm->ttm.io_reserve_mutex); |
1271 | nouveau_ttm_io_mem_free_locked(drm, reg); |
1272 | mutex_unlock(lock: &drm->ttm.io_reserve_mutex); |
1273 | } |
1274 | |
1275 | vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) |
1276 | { |
1277 | struct nouveau_drm *drm = nouveau_bdev(bd: bo->bdev); |
1278 | struct nouveau_bo *nvbo = nouveau_bo(bo); |
1279 | struct nvkm_device *device = nvxx_device(&drm->client.device); |
1280 | u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT; |
1281 | int i, ret; |
1282 | |
1283 | /* as long as the bo isn't in vram, and isn't tiled, we've got |
1284 | * nothing to do here. |
1285 | */ |
1286 | if (bo->resource->mem_type != TTM_PL_VRAM) { |
1287 | if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || |
1288 | !nvbo->kind) |
1289 | return 0; |
1290 | |
1291 | if (bo->resource->mem_type != TTM_PL_SYSTEM) |
1292 | return 0; |
1293 | |
1294 | nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, busy: 0); |
1295 | |
1296 | } else { |
1297 | /* make sure bo is in mappable vram */ |
1298 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA || |
1299 | bo->resource->start + PFN_UP(bo->resource->size) < mappable) |
1300 | return 0; |
1301 | |
1302 | for (i = 0; i < nvbo->placement.num_placement; ++i) { |
1303 | nvbo->placements[i].fpfn = 0; |
1304 | nvbo->placements[i].lpfn = mappable; |
1305 | } |
1306 | |
1307 | for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { |
1308 | nvbo->busy_placements[i].fpfn = 0; |
1309 | nvbo->busy_placements[i].lpfn = mappable; |
1310 | } |
1311 | |
1312 | nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, busy: 0); |
1313 | } |
1314 | |
1315 | ret = nouveau_bo_validate(nvbo, interruptible: false, no_wait_gpu: false); |
1316 | if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS)) |
1317 | return VM_FAULT_NOPAGE; |
1318 | else if (unlikely(ret)) |
1319 | return VM_FAULT_SIGBUS; |
1320 | |
1321 | ttm_bo_move_to_lru_tail_unlocked(bo); |
1322 | return 0; |
1323 | } |
1324 | |
1325 | static int |
1326 | nouveau_ttm_tt_populate(struct ttm_device *bdev, |
1327 | struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) |
1328 | { |
1329 | struct ttm_tt *ttm_dma = (void *)ttm; |
1330 | struct nouveau_drm *drm; |
1331 | bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); |
1332 | |
1333 | if (ttm_tt_is_populated(tt: ttm)) |
1334 | return 0; |
1335 | |
1336 | if (slave && ttm->sg) { |
1337 | drm_prime_sg_to_dma_addr_array(sgt: ttm->sg, addrs: ttm_dma->dma_address, |
1338 | max_pages: ttm->num_pages); |
1339 | return 0; |
1340 | } |
1341 | |
1342 | drm = nouveau_bdev(bd: bdev); |
1343 | |
1344 | return ttm_pool_alloc(pool: &drm->ttm.bdev.pool, tt: ttm, ctx); |
1345 | } |
1346 | |
1347 | static void |
1348 | nouveau_ttm_tt_unpopulate(struct ttm_device *bdev, |
1349 | struct ttm_tt *ttm) |
1350 | { |
1351 | struct nouveau_drm *drm; |
1352 | bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); |
1353 | |
1354 | if (slave) |
1355 | return; |
1356 | |
1357 | nouveau_ttm_tt_unbind(bdev, ttm); |
1358 | |
1359 | drm = nouveau_bdev(bd: bdev); |
1360 | |
1361 | return ttm_pool_free(pool: &drm->ttm.bdev.pool, tt: ttm); |
1362 | } |
1363 | |
1364 | static void |
1365 | nouveau_ttm_tt_destroy(struct ttm_device *bdev, |
1366 | struct ttm_tt *ttm) |
1367 | { |
1368 | #if IS_ENABLED(CONFIG_AGP) |
1369 | struct nouveau_drm *drm = nouveau_bdev(bd: bdev); |
1370 | if (drm->agp.bridge) { |
1371 | ttm_agp_destroy(ttm); |
1372 | return; |
1373 | } |
1374 | #endif |
1375 | nouveau_sgdma_destroy(bdev, ttm); |
1376 | } |
1377 | |
1378 | void |
1379 | nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) |
1380 | { |
1381 | struct dma_resv *resv = nvbo->bo.base.resv; |
1382 | |
1383 | if (!fence) |
1384 | return; |
1385 | |
1386 | dma_resv_add_fence(obj: resv, fence: &fence->base, usage: exclusive ? |
1387 | DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); |
1388 | } |
1389 | |
1390 | static void |
1391 | nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo) |
1392 | { |
1393 | nouveau_bo_move_ntfy(bo, NULL); |
1394 | } |
1395 | |
1396 | struct ttm_device_funcs nouveau_bo_driver = { |
1397 | .ttm_tt_create = &nouveau_ttm_tt_create, |
1398 | .ttm_tt_populate = &nouveau_ttm_tt_populate, |
1399 | .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, |
1400 | .ttm_tt_destroy = &nouveau_ttm_tt_destroy, |
1401 | .eviction_valuable = ttm_bo_eviction_valuable, |
1402 | .evict_flags = nouveau_bo_evict_flags, |
1403 | .delete_mem_notify = nouveau_bo_delete_mem_notify, |
1404 | .move = nouveau_bo_move, |
1405 | .io_mem_reserve = &nouveau_ttm_io_mem_reserve, |
1406 | .io_mem_free = &nouveau_ttm_io_mem_free, |
1407 | }; |
1408 | |