1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright 2014-2018 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23	#include <linux/dma-buf.h>
24	#include <linux/list.h>
25	#include <linux/pagemap.h>
26	#include <linux/sched/mm.h>
27	#include <linux/sched/task.h>
28	#include <drm/ttm/ttm_tt.h>
29
30	#include <drm/drm_exec.h>
31
32	#include "amdgpu_object.h"
33	#include "amdgpu_gem.h"
34	#include "amdgpu_vm.h"
35	#include "amdgpu_hmm.h"
36	#include "amdgpu_amdkfd.h"
37	#include "amdgpu_dma_buf.h"
38	#include <uapi/linux/kfd_ioctl.h>
39	#include "amdgpu_xgmi.h"
40	#include "kfd_priv.h"
41	#include "kfd_smi_events.h"
42
43	/* Userptr restore delay, just long enough to allow consecutive VM
44	* changes to accumulate
45	*/
46	#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
47	#define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29)
48
49	/*
50	* Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
51	* BO chunk
52	*/
53	#define VRAM_AVAILABLITY_ALIGN (1 << 21)
54
55	/* Impose limit on how much memory KFD can use */
56	static struct {
57	uint64_t max_system_mem_limit;
58	uint64_t max_ttm_mem_limit;
59	int64_t system_mem_used;
60	int64_t ttm_mem_used;
61	spinlock_t mem_limit_lock;
62	} kfd_mem_limit;
63
64	static const char * const domain_bit_to_string[] = {
65	"CPU",
66	"GTT",
67	"VRAM",
68	"GDS",
69	"GWS",
70	"OA"
71	};
72
73	#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
74
75	static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
76
77	static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
78	struct kgd_mem *mem)
79	{
80	struct kfd_mem_attachment *entry;
81
82	list_for_each_entry(entry, &mem->attachments, list)
83	if (entry->bo_va->base.vm == avm)
84	return true;
85
86	return false;
87	}
88
89	/**
90	* reuse_dmamap() - Check whether adev can share the original
91	* userptr BO
92	*
93	* If both adev and bo_adev are in direct mapping or
94	* in the same iommu group, they can share the original BO.
95	*
96	* @adev: Device to which can or cannot share the original BO
97	* @bo_adev: Device to which allocated BO belongs to
98	*
99	* Return: returns true if adev can share original userptr BO,
100	* false otherwise.
101	*/
102	static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)
103	{
104	return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||
105	(adev->dev->iommu_group == bo_adev->dev->iommu_group);
106	}
107
108	/* Set memory usage limits. Current, limits are
109	* System (TTM + userptr) memory - 15/16th System RAM
110	* TTM memory - 3/8th System RAM
111	*/
112	void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
113	{
114	struct sysinfo si;
115	uint64_t mem;
116
117	if (kfd_mem_limit.max_system_mem_limit)
118	return;
119
120	si_meminfo(val: &si);
121	mem = si.totalram - si.totalhigh;
122	mem *= si.mem_unit;
123
124	spin_lock_init(&kfd_mem_limit.mem_limit_lock);
125	kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);
126	if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT)
127	kfd_mem_limit.max_system_mem_limit >>= 1;
128	else
129	kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT;
130
131	kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT;
132	pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
133	(kfd_mem_limit.max_system_mem_limit >> 20),
134	(kfd_mem_limit.max_ttm_mem_limit >> 20));
135	}
136
137	void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
138	{
139	kfd_mem_limit.system_mem_used += size;
140	}
141
142	/* Estimate page table size needed to represent a given memory size
143	*
144	* With 4KB pages, we need one 8 byte PTE for each 4KB of memory
145	* (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
146	* of memory (factor 256K, >> 18). ROCm user mode tries to optimize
147	* for 2MB pages for TLB efficiency. However, small allocations and
148	* fragmented system memory still need some 4KB pages. We choose a
149	* compromise that should work in most cases without reserving too
150	* much memory for page tables unnecessarily (factor 16K, >> 14).
151	*/
152
153	#define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)
154
155	/**
156	* amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
157	* of buffer.
158	*
159	* @adev: Device to which allocated BO belongs to
160	* @size: Size of buffer, in bytes, encapsulated by B0. This should be
161	* equivalent to amdgpu_bo_size(BO)
162	* @alloc_flag: Flag used in allocating a BO as noted above
163	* @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is
164	* managed as one compute node in driver for app
165	*
166	* Return:
167	* returns -ENOMEM in case of error, ZERO otherwise
168	*/
169	int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
170	uint64_t size, u32 alloc_flag, int8_t xcp_id)
171	{
172	uint64_t reserved_for_pt =
173	ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
174	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
175	uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);
176	size_t system_mem_needed, ttm_mem_needed, vram_needed;
177	int ret = 0;
178	uint64_t vram_size = 0;
179
180	system_mem_needed = 0;
181	ttm_mem_needed = 0;
182	vram_needed = 0;
183	if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
184	system_mem_needed = size;
185	ttm_mem_needed = size;
186	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
187	/*
188	* Conservatively round up the allocation requirement to 2 MB
189	* to avoid fragmentation caused by 4K allocations in the tail
190	* 2M BO chunk.
191	*/
192	vram_needed = size;
193	/*
194	* For GFX 9.4.3, get the VRAM size from XCP structs
195	*/
196	if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
197	return -EINVAL;
198
199	vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
200	if (adev->apu_prefer_gtt) {
201	system_mem_needed = size;
202	ttm_mem_needed = size;
203	}
204	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
205	system_mem_needed = size;
206	} else if (!(alloc_flag &
207	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
208	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
209	pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
210	return -ENOMEM;
211	}
212
213	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
214
215	if (kfd_mem_limit.system_mem_used + system_mem_needed >
216	kfd_mem_limit.max_system_mem_limit) {
217	pr_debug("Set no_system_mem_limit=1 if using shared memory\n");
218	if (!no_system_mem_limit) {
219	ret = -ENOMEM;
220	goto release;
221	}
222	}
223
224	if (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
225	kfd_mem_limit.max_ttm_mem_limit) {
226	ret = -ENOMEM;
227	goto release;
228	}
229
230	/*if is_app_apu is false and apu_prefer_gtt is true, it is an APU with
231	* carve out < gtt. In that case, VRAM allocation will go to gtt domain, skip
232	* VRAM check since ttm_mem_limit check already cover this allocation
233	*/
234
235	if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu)) {
236	uint64_t vram_available =
237	vram_size - reserved_for_pt - reserved_for_ras -
238	atomic64_read(v: &adev->vram_pin_size);
239	if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available) {
240	ret = -ENOMEM;
241	goto release;
242	}
243	}
244
245	/* Update memory accounting by decreasing available system
246	* memory, TTM memory and GPU memory as computed above
247	*/
248	WARN_ONCE(vram_needed && !adev,
249	"adev reference can't be null when vram is used");
250	if (adev && xcp_id >= 0) {
251	adev->kfd.vram_used[xcp_id] += vram_needed;
252	adev->kfd.vram_used_aligned[xcp_id] +=
253	adev->apu_prefer_gtt ?
254	vram_needed :
255	ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
256	}
257	kfd_mem_limit.system_mem_used += system_mem_needed;
258	kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
259
260	release:
261	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
262	return ret;
263	}
264
265	void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
266	uint64_t size, u32 alloc_flag, int8_t xcp_id)
267	{
268	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
269
270	if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
271	kfd_mem_limit.system_mem_used -= size;
272	kfd_mem_limit.ttm_mem_used -= size;
273	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
274	WARN_ONCE(!adev,
275	"adev reference can't be null when alloc mem flags vram is set");
276	if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
277	goto release;
278
279	if (adev) {
280	adev->kfd.vram_used[xcp_id] -= size;
281	if (adev->apu_prefer_gtt) {
282	adev->kfd.vram_used_aligned[xcp_id] -= size;
283	kfd_mem_limit.system_mem_used -= size;
284	kfd_mem_limit.ttm_mem_used -= size;
285	} else {
286	adev->kfd.vram_used_aligned[xcp_id] -=
287	ALIGN(size, VRAM_AVAILABLITY_ALIGN);
288	}
289	}
290	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
291	kfd_mem_limit.system_mem_used -= size;
292	} else if (!(alloc_flag &
293	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
294	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
295	pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
296	goto release;
297	}
298	WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,
299	"KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);
300	WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
301	"KFD TTM memory accounting unbalanced");
302	WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
303	"KFD system memory accounting unbalanced");
304
305	release:
306	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
307	}
308
309	void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
310	{
311	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
312	u32 alloc_flags = bo->kfd_bo->alloc_flags;
313	u64 size = amdgpu_bo_size(bo);
314
315	amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flag: alloc_flags,
316	xcp_id: bo->xcp_id);
317
318	kfree(objp: bo->kfd_bo);
319	}
320
321	/**
322	* create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
323	* about USERPTR or DOOREBELL or MMIO BO.
324	*
325	* @adev: Device for which dmamap BO is being created
326	* @mem: BO of peer device that is being DMA mapped. Provides parameters
327	* in building the dmamap BO
328	* @bo_out: Output parameter updated with handle of dmamap BO
329	*/
330	static int
331	create_dmamap_sg_bo(struct amdgpu_device *adev,
332	struct kgd_mem *mem, struct amdgpu_bo **bo_out)
333	{
334	struct drm_gem_object *gem_obj;
335	int ret;
336	uint64_t flags = 0;
337
338	ret = amdgpu_bo_reserve(bo: mem->bo, no_intr: false);
339	if (ret)
340	return ret;
341
342	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
343	flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
344	AMDGPU_GEM_CREATE_UNCACHED);
345
346	ret = amdgpu_gem_object_create(adev, size: mem->bo->tbo.base.size, alignment: 1,
347	AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
348	type: ttm_bo_type_sg, resv: mem->bo->tbo.base.resv, obj: &gem_obj, xcp_id_plus1: 0);
349
350	amdgpu_bo_unreserve(bo: mem->bo);
351
352	if (ret) {
353	pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
354	return -EINVAL;
355	}
356
357	*bo_out = gem_to_amdgpu_bo(gem_obj);
358	(*bo_out)->parent = amdgpu_bo_ref(bo: mem->bo);
359	return ret;
360	}
361
362	/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
363	* reservation object.
364	*
365	* @bo: [IN] Remove eviction fence(s) from this BO
366	* @ef: [IN] This eviction fence is removed if it
367	* is present in the shared list.
368	*
369	* NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
370	*/
371	static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
372	struct amdgpu_amdkfd_fence *ef)
373	{
374	struct dma_fence *replacement;
375
376	if (!ef)
377	return -EINVAL;
378
379	/* TODO: Instead of block before we should use the fence of the page
380	* table update and TLB flush here directly.
381	*/
382	replacement = dma_fence_get_stub();
383	dma_resv_replace_fences(obj: bo->tbo.base.resv, context: ef->base.context,
384	fence: replacement, usage: DMA_RESV_USAGE_BOOKKEEP);
385	dma_fence_put(fence: replacement);
386	return 0;
387	}
388
389	/**
390	* amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences
391	* @bo: the BO where to remove the evictions fences from.
392	*
393	* This functions should only be used on release when all references to the BO
394	* are already dropped. We remove the eviction fence from the private copy of
395	* the dma_resv object here since that is what is used during release to
396	* determine of the BO is idle or not.
397	*/
398	void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo)
399	{
400	struct dma_resv *resv = &bo->tbo.base._resv;
401	struct dma_fence *fence, *stub;
402	struct dma_resv_iter cursor;
403
404	dma_resv_assert_held(resv);
405
406	stub = dma_fence_get_stub();
407	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
408	if (!to_amdgpu_amdkfd_fence(f: fence))
409	continue;
410
411	dma_resv_replace_fences(obj: resv, context: fence->context, fence: stub,
412	usage: DMA_RESV_USAGE_BOOKKEEP);
413	}
414	dma_fence_put(fence: stub);
415	}
416
417	static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
418	bool wait)
419	{
420	struct ttm_operation_ctx ctx = { false, false };
421	int ret;
422
423	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
424	"Called with userptr BO"))
425	return -EINVAL;
426
427	/* bo has been pinned, not need validate it */
428	if (bo->tbo.pin_count)
429	return 0;
430
431	amdgpu_bo_placement_from_domain(abo: bo, domain);
432
433	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
434	if (ret)
435	goto validate_fail;
436	if (wait)
437	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
438
439	validate_fail:
440	return ret;
441	}
442
443	int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
444	uint32_t domain,
445	struct dma_fence *fence)
446	{
447	int ret = amdgpu_bo_reserve(bo, no_intr: false);
448
449	if (ret)
450	return ret;
451
452	ret = amdgpu_amdkfd_bo_validate(bo, domain, wait: true);
453	if (ret)
454	goto unreserve_out;
455
456	ret = dma_resv_reserve_fences(obj: bo->tbo.base.resv, num_fences: 1);
457	if (ret)
458	goto unreserve_out;
459
460	dma_resv_add_fence(obj: bo->tbo.base.resv, fence,
461	usage: DMA_RESV_USAGE_BOOKKEEP);
462
463	unreserve_out:
464	amdgpu_bo_unreserve(bo);
465
466	return ret;
467	}
468
469	static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
470	{
471	return amdgpu_amdkfd_bo_validate(bo, domain: bo->allowed_domains, wait: false);
472	}
473
474	/* vm_validate_pt_pd_bos - Validate page table and directory BOs
475	*
476	* Page directories are not updated here because huge page handling
477	* during page table updates can invalidate page directory entries
478	* again. Page directories are only updated after updating page
479	* tables.
480	*/
481	static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm,
482	struct ww_acquire_ctx *ticket)
483	{
484	struct amdgpu_bo *pd = vm->root.bo;
485	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
486	int ret;
487
488	ret = amdgpu_vm_validate(adev, vm, ticket,
489	callback: amdgpu_amdkfd_validate_vm_bo, NULL);
490	if (ret) {
491	pr_err("failed to validate PT BOs\n");
492	return ret;
493	}
494
495	vm->pd_phys_addr = amdgpu_gmc_pd_addr(bo: vm->root.bo);
496
497	return 0;
498	}
499
500	static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
501	{
502	struct amdgpu_bo *pd = vm->root.bo;
503	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
504	int ret;
505
506	ret = amdgpu_vm_update_pdes(adev, vm, immediate: false);
507	if (ret)
508	return ret;
509
510	return amdgpu_sync_fence(sync, f: vm->last_update, GFP_KERNEL);
511	}
512
513	static uint64_t get_pte_flags(struct amdgpu_device *adev, struct amdgpu_vm *vm,
514	struct kgd_mem *mem)
515	{
516	uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |
517	AMDGPU_VM_MTYPE_DEFAULT;
518
519	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
520	mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
521	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
522	mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
523
524	return mapping_flags;
525	}
526
527	/**
528	* create_sg_table() - Create an sg_table for a contiguous DMA addr range
529	* @addr: The starting address to point to
530	* @size: Size of memory area in bytes being pointed to
531	*
532	* Allocates an instance of sg_table and initializes it to point to memory
533	* area specified by input parameters. The address used to build is assumed
534	* to be DMA mapped, if needed.
535	*
536	* DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
537	* because they are physically contiguous.
538	*
539	* Return: Initialized instance of SG Table or NULL
540	*/
541	static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
542	{
543	struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
544
545	if (!sg)
546	return NULL;
547	if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
548	kfree(objp: sg);
549	return NULL;
550	}
551	sg_dma_address(sg->sgl) = addr;
552	sg->sgl->length = size;
553	#ifdef CONFIG_NEED_SG_DMA_LENGTH
554	sg->sgl->dma_length = size;
555	#endif
556	return sg;
557	}
558
559	static int
560	kfd_mem_dmamap_userptr(struct kgd_mem *mem,
561	struct kfd_mem_attachment *attachment)
562	{
563	enum dma_data_direction direction =
564	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
565	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
566	struct ttm_operation_ctx ctx = {.interruptible = true};
567	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
568	struct amdgpu_device *adev = attachment->adev;
569	struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
570	struct ttm_tt *ttm = bo->tbo.ttm;
571	int ret;
572
573	if (WARN_ON(ttm->num_pages != src_ttm->num_pages))
574	return -EINVAL;
575
576	ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL);
577	if (unlikely(!ttm->sg))
578	return -ENOMEM;
579
580	/* Same sequence as in amdgpu_ttm_tt_pin_userptr */
581	ret = sg_alloc_table_from_pages(sgt: ttm->sg, pages: src_ttm->pages,
582	n_pages: ttm->num_pages, offset: 0,
583	size: (u64)ttm->num_pages << PAGE_SHIFT,
584	GFP_KERNEL);
585	if (unlikely(ret))
586	goto free_sg;
587
588	ret = dma_map_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
589	if (unlikely(ret))
590	goto release_sg;
591
592	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
593	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
594	if (ret)
595	goto unmap_sg;
596
597	return 0;
598
599	unmap_sg:
600	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
601	release_sg:
602	pr_err("DMA map userptr failed: %d\n", ret);
603	sg_free_table(ttm->sg);
604	free_sg:
605	kfree(objp: ttm->sg);
606	ttm->sg = NULL;
607	return ret;
608	}
609
610	static int
611	kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
612	{
613	struct ttm_operation_ctx ctx = {.interruptible = true};
614	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
615
616	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
617	return ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
618	}
619
620	/**
621	* kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
622	* @mem: SG BO of the DOORBELL or MMIO resource on the owning device
623	* @attachment: Virtual address attachment of the BO on accessing device
624	*
625	* An access request from the device that owns DOORBELL does not require DMA mapping.
626	* This is because the request doesn't go through PCIe root complex i.e. it instead
627	* loops back. The need to DMA map arises only when accessing peer device's DOORBELL
628	*
629	* In contrast, all access requests for MMIO need to be DMA mapped without regard to
630	* device ownership. This is because access requests for MMIO go through PCIe root
631	* complex.
632	*
633	* This is accomplished in two steps:
634	* - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
635	* in updating requesting device's page table
636	* - Signal TTM to mark memory pointed to by requesting device's BO as GPU
637	* accessible. This allows an update of requesting device's page table
638	* with entries associated with DOOREBELL or MMIO memory
639	*
640	* This method is invoked in the following contexts:
641	* - Mapping of DOORBELL or MMIO BO of same or peer device
642	* - Validating an evicted DOOREBELL or MMIO BO on device seeking access
643	*
644	* Return: ZERO if successful, NON-ZERO otherwise
645	*/
646	static int
647	kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
648	struct kfd_mem_attachment *attachment)
649	{
650	struct ttm_operation_ctx ctx = {.interruptible = true};
651	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
652	struct amdgpu_device *adev = attachment->adev;
653	struct ttm_tt *ttm = bo->tbo.ttm;
654	enum dma_data_direction dir;
655	dma_addr_t dma_addr;
656	bool mmio;
657	int ret;
658
659	/* Expect SG Table of dmapmap BO to be NULL */
660	mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
661	if (unlikely(ttm->sg)) {
662	pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
663	return -EINVAL;
664	}
665
666	dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
667	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
668	dma_addr = mem->bo->tbo.sg->sgl->dma_address;
669	pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
670	pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
671	dma_addr = dma_map_resource(dev: adev->dev, phys_addr: dma_addr,
672	size: mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
673	ret = dma_mapping_error(dev: adev->dev, dma_addr);
674	if (unlikely(ret))
675	return ret;
676	pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
677
678	ttm->sg = create_sg_table(addr: dma_addr, size: mem->bo->tbo.sg->sgl->length);
679	if (unlikely(!ttm->sg)) {
680	ret = -ENOMEM;
681	goto unmap_sg;
682	}
683
684	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
685	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
686	if (unlikely(ret))
687	goto free_sg;
688
689	return ret;
690
691	free_sg:
692	sg_free_table(ttm->sg);
693	kfree(objp: ttm->sg);
694	ttm->sg = NULL;
695	unmap_sg:
696	dma_unmap_resource(dev: adev->dev, addr: dma_addr, size: mem->bo->tbo.sg->sgl->length,
697	dir, DMA_ATTR_SKIP_CPU_SYNC);
698	return ret;
699	}
700
701	static int
702	kfd_mem_dmamap_attachment(struct kgd_mem *mem,
703	struct kfd_mem_attachment *attachment)
704	{
705	switch (attachment->type) {
706	case KFD_MEM_ATT_SHARED:
707	return 0;
708	case KFD_MEM_ATT_USERPTR:
709	return kfd_mem_dmamap_userptr(mem, attachment);
710	case KFD_MEM_ATT_DMABUF:
711	return kfd_mem_dmamap_dmabuf(attachment);
712	case KFD_MEM_ATT_SG:
713	return kfd_mem_dmamap_sg_bo(mem, attachment);
714	default:
715	WARN_ON_ONCE(1);
716	}
717	return -EINVAL;
718	}
719
720	static void
721	kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
722	struct kfd_mem_attachment *attachment)
723	{
724	enum dma_data_direction direction =
725	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
726	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
727	struct ttm_operation_ctx ctx = {.interruptible = false};
728	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
729	struct amdgpu_device *adev = attachment->adev;
730	struct ttm_tt *ttm = bo->tbo.ttm;
731
732	if (unlikely(!ttm->sg))
733	return;
734
735	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
736	(void)ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
737
738	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
739	sg_free_table(ttm->sg);
740	kfree(objp: ttm->sg);
741	ttm->sg = NULL;
742	}
743
744	static void
745	kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
746	{
747	/* This is a no-op. We don't want to trigger eviction fences when
748	* unmapping DMABufs. Therefore the invalidation (moving to system
749	* domain) is done in kfd_mem_dmamap_dmabuf.
750	*/
751	}
752
753	/**
754	* kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
755	* @mem: SG BO of the DOORBELL or MMIO resource on the owning device
756	* @attachment: Virtual address attachment of the BO on accessing device
757	*
758	* The method performs following steps:
759	* - Signal TTM to mark memory pointed to by BO as GPU inaccessible
760	* - Free SG Table that is used to encapsulate DMA mapped memory of
761	* peer device's DOORBELL or MMIO memory
762	*
763	* This method is invoked in the following contexts:
764	* UNMapping of DOORBELL or MMIO BO on a device having access to its memory
765	* Eviction of DOOREBELL or MMIO BO on device having access to its memory
766	*
767	* Return: void
768	*/
769	static void
770	kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
771	struct kfd_mem_attachment *attachment)
772	{
773	struct ttm_operation_ctx ctx = {.interruptible = true};
774	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
775	struct amdgpu_device *adev = attachment->adev;
776	struct ttm_tt *ttm = bo->tbo.ttm;
777	enum dma_data_direction dir;
778
779	if (unlikely(!ttm->sg)) {
780	pr_debug("SG Table of BO is NULL");
781	return;
782	}
783
784	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
785	(void)ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
786
787	dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
788	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
789	dma_unmap_resource(dev: adev->dev, addr: ttm->sg->sgl->dma_address,
790	size: ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
791	sg_free_table(ttm->sg);
792	kfree(objp: ttm->sg);
793	ttm->sg = NULL;
794	bo->tbo.sg = NULL;
795	}
796
797	static void
798	kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
799	struct kfd_mem_attachment *attachment)
800	{
801	switch (attachment->type) {
802	case KFD_MEM_ATT_SHARED:
803	break;
804	case KFD_MEM_ATT_USERPTR:
805	kfd_mem_dmaunmap_userptr(mem, attachment);
806	break;
807	case KFD_MEM_ATT_DMABUF:
808	kfd_mem_dmaunmap_dmabuf(attachment);
809	break;
810	case KFD_MEM_ATT_SG:
811	kfd_mem_dmaunmap_sg_bo(mem, attachment);
812	break;
813	default:
814	WARN_ON_ONCE(1);
815	}
816	}
817
818	static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
819	{
820	if (!mem->dmabuf) {
821	struct amdgpu_device *bo_adev;
822	struct dma_buf *dmabuf;
823
824	bo_adev = amdgpu_ttm_adev(bdev: mem->bo->tbo.bdev);
825	dmabuf = drm_gem_prime_handle_to_dmabuf(dev: &bo_adev->ddev, file_priv: bo_adev->kfd.client.file,
826	handle: mem->gem_handle,
827	flags: mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
828	DRM_RDWR : 0);
829	if (IS_ERR(ptr: dmabuf))
830	return PTR_ERR(ptr: dmabuf);
831	mem->dmabuf = dmabuf;
832	}
833
834	return 0;
835	}
836
837	static int
838	kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
839	struct amdgpu_bo **bo)
840	{
841	struct drm_gem_object *gobj;
842	int ret;
843
844	ret = kfd_mem_export_dmabuf(mem);
845	if (ret)
846	return ret;
847
848	gobj = amdgpu_gem_prime_import(dev: adev_to_drm(adev), dma_buf: mem->dmabuf);
849	if (IS_ERR(ptr: gobj))
850	return PTR_ERR(ptr: gobj);
851
852	*bo = gem_to_amdgpu_bo(gobj);
853	(*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
854
855	return 0;
856	}
857
858	/* kfd_mem_attach - Add a BO to a VM
859	*
860	* Everything that needs to bo done only once when a BO is first added
861	* to a VM. It can later be mapped and unmapped many times without
862	* repeating these steps.
863	*
864	* 0. Create BO for DMA mapping, if needed
865	* 1. Allocate and initialize BO VA entry data structure
866	* 2. Add BO to the VM
867	* 3. Determine ASIC-specific PTE flags
868	* 4. Alloc page tables and directories if needed
869	* 4a. Validate new page tables and directories
870	*/
871	static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
872	struct amdgpu_vm *vm, bool is_aql)
873	{
874	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bdev: mem->bo->tbo.bdev);
875	unsigned long bo_size = mem->bo->tbo.base.size;
876	uint64_t va = mem->va;
877	struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
878	struct amdgpu_bo *bo[2] = {NULL, NULL};
879	struct amdgpu_bo_va *bo_va;
880	bool same_hive = false;
881	int i, ret;
882
883	if (!va) {
884	pr_err("Invalid VA when adding BO to VM\n");
885	return -EINVAL;
886	}
887
888	/* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
889	*
890	* The access path of MMIO and DOORBELL BOs of is always over PCIe.
891	* In contrast the access path of VRAM BOs depens upon the type of
892	* link that connects the peer device. Access over PCIe is allowed
893	* if peer device has large BAR. In contrast, access over xGMI is
894	* allowed for both small and large BAR configurations of peer device
895	*/
896	if ((adev != bo_adev && !adev->apu_prefer_gtt) &&
897	((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
898	(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
899	(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
900	if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
901	same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
902	if (!same_hive && !amdgpu_device_is_peer_accessible(adev: bo_adev, peer_adev: adev))
903	return -EINVAL;
904	}
905
906	for (i = 0; i <= is_aql; i++) {
907	attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
908	if (unlikely(!attachment[i])) {
909	ret = -ENOMEM;
910	goto unwind;
911	}
912
913	pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
914	va + bo_size, vm);
915
916	if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
917	(amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
918	(mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||
919	same_hive) {
920	/* Mappings on the local GPU, or VRAM mappings in the
921	* local hive, or userptr, or GTT mapping can reuse dma map
922	* address space share the original BO
923	*/
924	attachment[i]->type = KFD_MEM_ATT_SHARED;
925	bo[i] = mem->bo;
926	drm_gem_object_get(obj: &bo[i]->tbo.base);
927	} else if (i > 0) {
928	/* Multiple mappings on the same GPU share the BO */
929	attachment[i]->type = KFD_MEM_ATT_SHARED;
930	bo[i] = bo[0];
931	drm_gem_object_get(obj: &bo[i]->tbo.base);
932	} else if (amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm)) {
933	/* Create an SG BO to DMA-map userptrs on other GPUs */
934	attachment[i]->type = KFD_MEM_ATT_USERPTR;
935	ret = create_dmamap_sg_bo(adev, mem, bo_out: &bo[i]);
936	if (ret)
937	goto unwind;
938	/* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
939	} else if (mem->bo->tbo.type == ttm_bo_type_sg) {
940	WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
941	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
942	"Handing invalid SG BO in ATTACH request");
943	attachment[i]->type = KFD_MEM_ATT_SG;
944	ret = create_dmamap_sg_bo(adev, mem, bo_out: &bo[i]);
945	if (ret)
946	goto unwind;
947	/* Enable acces to GTT and VRAM BOs of peer devices */
948	} else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
949	mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
950	attachment[i]->type = KFD_MEM_ATT_DMABUF;
951	ret = kfd_mem_attach_dmabuf(adev, mem, bo: &bo[i]);
952	if (ret)
953	goto unwind;
954	pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");
955	} else {
956	WARN_ONCE(true, "Handling invalid ATTACH request");
957	ret = -EINVAL;
958	goto unwind;
959	}
960
961	/* Add BO to VM internal data structures */
962	ret = amdgpu_bo_reserve(bo: bo[i], no_intr: false);
963	if (ret) {
964	pr_debug("Unable to reserve BO during memory attach");
965	goto unwind;
966	}
967	bo_va = amdgpu_vm_bo_find(vm, bo: bo[i]);
968	if (!bo_va)
969	bo_va = amdgpu_vm_bo_add(adev, vm, bo: bo[i]);
970	else
971	++bo_va->ref_count;
972	attachment[i]->bo_va = bo_va;
973	amdgpu_bo_unreserve(bo: bo[i]);
974	if (unlikely(!attachment[i]->bo_va)) {
975	ret = -ENOMEM;
976	pr_err("Failed to add BO object to VM. ret == %d\n",
977	ret);
978	goto unwind;
979	}
980	attachment[i]->va = va;
981	attachment[i]->pte_flags = get_pte_flags(adev, vm, mem);
982	attachment[i]->adev = adev;
983	list_add(new: &attachment[i]->list, head: &mem->attachments);
984
985	va += bo_size;
986	}
987
988	return 0;
989
990	unwind:
991	for (; i >= 0; i--) {
992	if (!attachment[i])
993	continue;
994	if (attachment[i]->bo_va) {
995	(void)amdgpu_bo_reserve(bo: bo[i], no_intr: true);
996	if (--attachment[i]->bo_va->ref_count == 0)
997	amdgpu_vm_bo_del(adev, bo_va: attachment[i]->bo_va);
998	amdgpu_bo_unreserve(bo: bo[i]);
999	list_del(entry: &attachment[i]->list);
1000	}
1001	if (bo[i])
1002	drm_gem_object_put(obj: &bo[i]->tbo.base);
1003	kfree(objp: attachment[i]);
1004	}
1005	return ret;
1006	}
1007
1008	static void kfd_mem_detach(struct kfd_mem_attachment *attachment)
1009	{
1010	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
1011
1012	pr_debug("\t remove VA 0x%llx in entry %p\n",
1013	attachment->va, attachment);
1014	if (--attachment->bo_va->ref_count == 0)
1015	amdgpu_vm_bo_del(adev: attachment->adev, bo_va: attachment->bo_va);
1016	drm_gem_object_put(obj: &bo->tbo.base);
1017	list_del(entry: &attachment->list);
1018	kfree(objp: attachment);
1019	}
1020
1021	static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
1022	struct amdkfd_process_info *process_info,
1023	bool userptr)
1024	{
1025	mutex_lock(&process_info->lock);
1026	if (userptr)
1027	list_add_tail(new: &mem->validate_list,
1028	head: &process_info->userptr_valid_list);
1029	else
1030	list_add_tail(new: &mem->validate_list, head: &process_info->kfd_bo_list);
1031	mutex_unlock(lock: &process_info->lock);
1032	}
1033
1034	static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
1035	struct amdkfd_process_info *process_info)
1036	{
1037	mutex_lock(&process_info->lock);
1038	list_del(entry: &mem->validate_list);
1039	mutex_unlock(lock: &process_info->lock);
1040	}
1041
1042	/* Initializes user pages. It registers the MMU notifier and validates
1043	* the userptr BO in the GTT domain.
1044	*
1045	* The BO must already be on the userptr_valid_list. Otherwise an
1046	* eviction and restore may happen that leaves the new BO unmapped
1047	* with the user mode queues running.
1048	*
1049	* Takes the process_info->lock to protect against concurrent restore
1050	* workers.
1051	*
1052	* Returns 0 for success, negative errno for errors.
1053	*/
1054	static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
1055	bool criu_resume)
1056	{
1057	struct amdkfd_process_info *process_info = mem->process_info;
1058	struct amdgpu_bo *bo = mem->bo;
1059	struct ttm_operation_ctx ctx = { true, false };
1060	struct amdgpu_hmm_range *range;
1061	int ret = 0;
1062
1063	mutex_lock(&process_info->lock);
1064
1065	ret = amdgpu_ttm_tt_set_userptr(bo: &bo->tbo, addr: user_addr, flags: 0);
1066	if (ret) {
1067	pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
1068	goto out;
1069	}
1070
1071	ret = amdgpu_hmm_register(bo, addr: user_addr);
1072	if (ret) {
1073	pr_err("%s: Failed to register MMU notifier: %d\n",
1074	__func__, ret);
1075	goto out;
1076	}
1077
1078	if (criu_resume) {
1079	/*
1080	* During a CRIU restore operation, the userptr buffer objects
1081	* will be validated in the restore_userptr_work worker at a
1082	* later stage when it is scheduled by another ioctl called by
1083	* CRIU master process for the target pid for restore.
1084	*/
1085	mutex_lock(&process_info->notifier_lock);
1086	mem->invalid++;
1087	mutex_unlock(lock: &process_info->notifier_lock);
1088	mutex_unlock(lock: &process_info->lock);
1089	return 0;
1090	}
1091
1092	range = amdgpu_hmm_range_alloc(NULL);
1093	if (unlikely(!range)) {
1094	ret = -ENOMEM;
1095	goto unregister_out;
1096	}
1097
1098	ret = amdgpu_ttm_tt_get_user_pages(bo, range);
1099	if (ret) {
1100	amdgpu_hmm_range_free(range);
1101	if (ret == -EAGAIN)
1102	pr_debug("Failed to get user pages, try again\n");
1103	else
1104	pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
1105	goto unregister_out;
1106	}
1107
1108	ret = amdgpu_bo_reserve(bo, no_intr: true);
1109	if (ret) {
1110	pr_err("%s: Failed to reserve BO\n", __func__);
1111	goto release_out;
1112	}
1113
1114	amdgpu_ttm_tt_set_user_pages(ttm: bo->tbo.ttm, range);
1115
1116	amdgpu_bo_placement_from_domain(abo: bo, domain: mem->domain);
1117	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
1118	if (ret)
1119	pr_err("%s: failed to validate BO\n", __func__);
1120	amdgpu_bo_unreserve(bo);
1121
1122	release_out:
1123	amdgpu_hmm_range_free(range);
1124	unregister_out:
1125	if (ret)
1126	amdgpu_hmm_unregister(bo);
1127	out:
1128	mutex_unlock(lock: &process_info->lock);
1129	return ret;
1130	}
1131
1132	/* Reserving a BO and its page table BOs must happen atomically to
1133	* avoid deadlocks. Some operations update multiple VMs at once. Track
1134	* all the reservation info in a context structure. Optionally a sync
1135	* object can track VM updates.
1136	*/
1137	struct bo_vm_reservation_context {
1138	/* DRM execution context for the reservation */
1139	struct drm_exec exec;
1140	/* Number of VMs reserved */
1141	unsigned int n_vms;
1142	/* Pointer to sync object */
1143	struct amdgpu_sync *sync;
1144	};
1145
1146	enum bo_vm_match {
1147	BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */
1148	BO_VM_MAPPED, /* Match VMs where a BO is mapped */
1149	BO_VM_ALL, /* Match all VMs a BO was added to */
1150	};
1151
1152	/**
1153	* reserve_bo_and_vm - reserve a BO and a VM unconditionally.
1154	* @mem: KFD BO structure.
1155	* @vm: the VM to reserve.
1156	* @ctx: the struct that will be used in unreserve_bo_and_vms().
1157	*/
1158	static int reserve_bo_and_vm(struct kgd_mem *mem,
1159	struct amdgpu_vm *vm,
1160	struct bo_vm_reservation_context *ctx)
1161	{
1162	struct amdgpu_bo *bo = mem->bo;
1163	int ret;
1164
1165	WARN_ON(!vm);
1166
1167	ctx->n_vms = 1;
1168	ctx->sync = &mem->sync;
1169	drm_exec_init(exec: &ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, nr: 0);
1170	drm_exec_until_all_locked(&ctx->exec) {
1171	ret = amdgpu_vm_lock_pd(vm, exec: &ctx->exec, num_fences: 2);
1172	drm_exec_retry_on_contention(&ctx->exec);
1173	if (unlikely(ret))
1174	goto error;
1175
1176	ret = drm_exec_prepare_obj(exec: &ctx->exec, obj: &bo->tbo.base, num_fences: 1);
1177	drm_exec_retry_on_contention(&ctx->exec);
1178	if (unlikely(ret))
1179	goto error;
1180	}
1181	return 0;
1182
1183	error:
1184	pr_err("Failed to reserve buffers in ttm.\n");
1185	drm_exec_fini(exec: &ctx->exec);
1186	return ret;
1187	}
1188
1189	/**
1190	* reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
1191	* @mem: KFD BO structure.
1192	* @vm: the VM to reserve. If NULL, then all VMs associated with the BO
1193	* is used. Otherwise, a single VM associated with the BO.
1194	* @map_type: the mapping status that will be used to filter the VMs.
1195	* @ctx: the struct that will be used in unreserve_bo_and_vms().
1196	*
1197	* Returns 0 for success, negative for failure.
1198	*/
1199	static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1200	struct amdgpu_vm *vm, enum bo_vm_match map_type,
1201	struct bo_vm_reservation_context *ctx)
1202	{
1203	struct kfd_mem_attachment *entry;
1204	struct amdgpu_bo *bo = mem->bo;
1205	int ret;
1206
1207	ctx->sync = &mem->sync;
1208	drm_exec_init(exec: &ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
1209	DRM_EXEC_IGNORE_DUPLICATES, nr: 0);
1210	drm_exec_until_all_locked(&ctx->exec) {
1211	ctx->n_vms = 0;
1212	list_for_each_entry(entry, &mem->attachments, list) {
1213	if ((vm && vm != entry->bo_va->base.vm) ||
1214	(entry->is_mapped != map_type
1215	&& map_type != BO_VM_ALL))
1216	continue;
1217
1218	ret = amdgpu_vm_lock_pd(vm: entry->bo_va->base.vm,
1219	exec: &ctx->exec, num_fences: 2);
1220	drm_exec_retry_on_contention(&ctx->exec);
1221	if (unlikely(ret))
1222	goto error;
1223	++ctx->n_vms;
1224	}
1225
1226	ret = drm_exec_prepare_obj(exec: &ctx->exec, obj: &bo->tbo.base, num_fences: 1);
1227	drm_exec_retry_on_contention(&ctx->exec);
1228	if (unlikely(ret))
1229	goto error;
1230	}
1231	return 0;
1232
1233	error:
1234	pr_err("Failed to reserve buffers in ttm.\n");
1235	drm_exec_fini(exec: &ctx->exec);
1236	return ret;
1237	}
1238
1239	/**
1240	* unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
1241	* @ctx: Reservation context to unreserve
1242	* @wait: Optionally wait for a sync object representing pending VM updates
1243	* @intr: Whether the wait is interruptible
1244	*
1245	* Also frees any resources allocated in
1246	* reserve_bo_and_(cond_)vm(s). Returns the status from
1247	* amdgpu_sync_wait.
1248	*/
1249	static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
1250	bool wait, bool intr)
1251	{
1252	int ret = 0;
1253
1254	if (wait)
1255	ret = amdgpu_sync_wait(sync: ctx->sync, intr);
1256
1257	drm_exec_fini(exec: &ctx->exec);
1258	ctx->sync = NULL;
1259	return ret;
1260	}
1261
1262	static int unmap_bo_from_gpuvm(struct kgd_mem *mem,
1263	struct kfd_mem_attachment *entry,
1264	struct amdgpu_sync *sync)
1265	{
1266	struct amdgpu_bo_va *bo_va = entry->bo_va;
1267	struct amdgpu_device *adev = entry->adev;
1268	struct amdgpu_vm *vm = bo_va->base.vm;
1269
1270	if (bo_va->queue_refcount) {
1271	pr_debug("bo_va->queue_refcount %d\n", bo_va->queue_refcount);
1272	return -EBUSY;
1273	}
1274
1275	(void)amdgpu_vm_bo_unmap(adev, bo_va, addr: entry->va);
1276
1277	/* VM entity stopped if process killed, don't clear freed pt bo */
1278	if (!amdgpu_vm_ready(vm))
1279	return 0;
1280
1281	(void)amdgpu_vm_clear_freed(adev, vm, fence: &bo_va->last_pt_update);
1282
1283	(void)amdgpu_sync_fence(sync, f: bo_va->last_pt_update, GFP_KERNEL);
1284
1285	return 0;
1286	}
1287
1288	static int update_gpuvm_pte(struct kgd_mem *mem,
1289	struct kfd_mem_attachment *entry,
1290	struct amdgpu_sync *sync)
1291	{
1292	struct amdgpu_bo_va *bo_va = entry->bo_va;
1293	struct amdgpu_device *adev = entry->adev;
1294	int ret;
1295
1296	ret = kfd_mem_dmamap_attachment(mem, attachment: entry);
1297	if (ret)
1298	return ret;
1299
1300	/* Update the page tables */
1301	ret = amdgpu_vm_bo_update(adev, bo_va, clear: false);
1302	if (ret) {
1303	pr_err("amdgpu_vm_bo_update failed\n");
1304	return ret;
1305	}
1306
1307	return amdgpu_sync_fence(sync, f: bo_va->last_pt_update, GFP_KERNEL);
1308	}
1309
1310	static int map_bo_to_gpuvm(struct kgd_mem *mem,
1311	struct kfd_mem_attachment *entry,
1312	struct amdgpu_sync *sync,
1313	bool no_update_pte)
1314	{
1315	int ret;
1316
1317	/* Set virtual address for the allocation */
1318	ret = amdgpu_vm_bo_map(adev: entry->adev, bo_va: entry->bo_va, addr: entry->va, offset: 0,
1319	size: amdgpu_bo_size(bo: entry->bo_va->base.bo),
1320	flags: entry->pte_flags);
1321	if (ret) {
1322	pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
1323	entry->va, ret);
1324	return ret;
1325	}
1326
1327	if (no_update_pte)
1328	return 0;
1329
1330	ret = update_gpuvm_pte(mem, entry, sync);
1331	if (ret) {
1332	pr_err("update_gpuvm_pte() failed\n");
1333	goto update_gpuvm_pte_failed;
1334	}
1335
1336	return 0;
1337
1338	update_gpuvm_pte_failed:
1339	unmap_bo_from_gpuvm(mem, entry, sync);
1340	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
1341	return ret;
1342	}
1343
1344	static int process_validate_vms(struct amdkfd_process_info *process_info,
1345	struct ww_acquire_ctx *ticket)
1346	{
1347	struct amdgpu_vm *peer_vm;
1348	int ret;
1349
1350	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1351	vm_list_node) {
1352	ret = vm_validate_pt_pd_bos(vm: peer_vm, ticket);
1353	if (ret)
1354	return ret;
1355	}
1356
1357	return 0;
1358	}
1359
1360	static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
1361	struct amdgpu_sync *sync)
1362	{
1363	struct amdgpu_vm *peer_vm;
1364	int ret;
1365
1366	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1367	vm_list_node) {
1368	struct amdgpu_bo *pd = peer_vm->root.bo;
1369
1370	ret = amdgpu_sync_resv(NULL, sync, resv: pd->tbo.base.resv,
1371	mode: AMDGPU_SYNC_NE_OWNER,
1372	AMDGPU_FENCE_OWNER_KFD);
1373	if (ret)
1374	return ret;
1375	}
1376
1377	return 0;
1378	}
1379
1380	static int process_update_pds(struct amdkfd_process_info *process_info,
1381	struct amdgpu_sync *sync)
1382	{
1383	struct amdgpu_vm *peer_vm;
1384	int ret;
1385
1386	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1387	vm_list_node) {
1388	ret = vm_update_pds(vm: peer_vm, sync);
1389	if (ret)
1390	return ret;
1391	}
1392
1393	return 0;
1394	}
1395
1396	static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
1397	struct dma_fence **ef)
1398	{
1399	struct amdkfd_process_info *info = NULL;
1400	int ret;
1401
1402	if (!*process_info) {
1403	info = kzalloc(sizeof(*info), GFP_KERNEL);
1404	if (!info)
1405	return -ENOMEM;
1406
1407	mutex_init(&info->lock);
1408	mutex_init(&info->notifier_lock);
1409	INIT_LIST_HEAD(list: &info->vm_list_head);
1410	INIT_LIST_HEAD(list: &info->kfd_bo_list);
1411	INIT_LIST_HEAD(list: &info->userptr_valid_list);
1412	INIT_LIST_HEAD(list: &info->userptr_inval_list);
1413
1414	info->eviction_fence =
1415	amdgpu_amdkfd_fence_create(context: dma_fence_context_alloc(num: 1),
1416	current->mm,
1417	NULL);
1418	if (!info->eviction_fence) {
1419	pr_err("Failed to create eviction fence\n");
1420	ret = -ENOMEM;
1421	goto create_evict_fence_fail;
1422	}
1423
1424	info->pid = get_task_pid(current->group_leader, type: PIDTYPE_PID);
1425	INIT_DELAYED_WORK(&info->restore_userptr_work,
1426	amdgpu_amdkfd_restore_userptr_worker);
1427
1428	*process_info = info;
1429	}
1430
1431	vm->process_info = *process_info;
1432
1433	/* Validate page directory and attach eviction fence */
1434	ret = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: true);
1435	if (ret)
1436	goto reserve_pd_fail;
1437	ret = vm_validate_pt_pd_bos(vm, NULL);
1438	if (ret) {
1439	pr_err("validate_pt_pd_bos() failed\n");
1440	goto validate_pd_fail;
1441	}
1442	ret = amdgpu_bo_sync_wait(bo: vm->root.bo,
1443	AMDGPU_FENCE_OWNER_KFD, intr: false);
1444	if (ret)
1445	goto wait_pd_fail;
1446	ret = dma_resv_reserve_fences(obj: vm->root.bo->tbo.base.resv, num_fences: 1);
1447	if (ret)
1448	goto reserve_shared_fail;
1449	dma_resv_add_fence(obj: vm->root.bo->tbo.base.resv,
1450	fence: &vm->process_info->eviction_fence->base,
1451	usage: DMA_RESV_USAGE_BOOKKEEP);
1452	amdgpu_bo_unreserve(bo: vm->root.bo);
1453
1454	/* Update process info */
1455	mutex_lock(&vm->process_info->lock);
1456	list_add_tail(new: &vm->vm_list_node,
1457	head: &(vm->process_info->vm_list_head));
1458	vm->process_info->n_vms++;
1459	if (ef)
1460	*ef = dma_fence_get(fence: &vm->process_info->eviction_fence->base);
1461	mutex_unlock(lock: &vm->process_info->lock);
1462
1463	return 0;
1464
1465	reserve_shared_fail:
1466	wait_pd_fail:
1467	validate_pd_fail:
1468	amdgpu_bo_unreserve(bo: vm->root.bo);
1469	reserve_pd_fail:
1470	vm->process_info = NULL;
1471	if (info) {
1472	dma_fence_put(fence: &info->eviction_fence->base);
1473	*process_info = NULL;
1474	put_pid(pid: info->pid);
1475	create_evict_fence_fail:
1476	mutex_destroy(lock: &info->lock);
1477	mutex_destroy(lock: &info->notifier_lock);
1478	kfree(objp: info);
1479	}
1480	return ret;
1481	}
1482
1483	/**
1484	* amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
1485	* @bo: Handle of buffer object being pinned
1486	* @domain: Domain into which BO should be pinned
1487	*
1488	* - USERPTR BOs are UNPINNABLE and will return error
1489	* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1490	* PIN count incremented. It is valid to PIN a BO multiple times
1491	*
1492	* Return: ZERO if successful in pinning, Non-Zero in case of error.
1493	*/
1494	static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
1495	{
1496	int ret = 0;
1497
1498	ret = amdgpu_bo_reserve(bo, no_intr: false);
1499	if (unlikely(ret))
1500	return ret;
1501
1502	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {
1503	/*
1504	* If bo is not contiguous on VRAM, move to system memory first to ensure
1505	* we can get contiguous VRAM space after evicting other BOs.
1506	*/
1507	if (!(bo->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) {
1508	struct ttm_operation_ctx ctx = { true, false };
1509
1510	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
1511	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
1512	if (unlikely(ret)) {
1513	pr_debug("validate bo 0x%p to GTT failed %d\n", &bo->tbo, ret);
1514	goto out;
1515	}
1516	}
1517	}
1518
1519	ret = amdgpu_bo_pin(bo, domain);
1520	if (ret)
1521	pr_err("Error in Pinning BO to domain: %d\n", domain);
1522
1523	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
1524	out:
1525	amdgpu_bo_unreserve(bo);
1526	return ret;
1527	}
1528
1529	/**
1530	* amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
1531	* @bo: Handle of buffer object being unpinned
1532	*
1533	* - Is a illegal request for USERPTR BOs and is ignored
1534	* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1535	* PIN count decremented. Calls to UNPIN must balance calls to PIN
1536	*/
1537	static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
1538	{
1539	int ret = 0;
1540
1541	ret = amdgpu_bo_reserve(bo, no_intr: false);
1542	if (unlikely(ret))
1543	return;
1544
1545	amdgpu_bo_unpin(bo);
1546	amdgpu_bo_unreserve(bo);
1547	}
1548
1549	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
1550	struct amdgpu_vm *avm,
1551	void **process_info,
1552	struct dma_fence **ef)
1553	{
1554	int ret;
1555
1556	/* Already a compute VM? */
1557	if (avm->process_info)
1558	return -EINVAL;
1559
1560	/* Convert VM into a compute VM */
1561	ret = amdgpu_vm_make_compute(adev, vm: avm);
1562	if (ret)
1563	return ret;
1564
1565	/* Initialize KFD part of the VM and process info */
1566	ret = init_kfd_vm(vm: avm, process_info, ef);
1567	if (ret)
1568	return ret;
1569
1570	amdgpu_vm_set_task_info(vm: avm);
1571
1572	return 0;
1573	}
1574
1575	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
1576	struct amdgpu_vm *vm)
1577	{
1578	struct amdkfd_process_info *process_info = vm->process_info;
1579
1580	if (!process_info)
1581	return;
1582
1583	/* Update process info */
1584	mutex_lock(&process_info->lock);
1585	process_info->n_vms--;
1586	list_del(entry: &vm->vm_list_node);
1587	mutex_unlock(lock: &process_info->lock);
1588
1589	vm->process_info = NULL;
1590
1591	/* Release per-process resources when last compute VM is destroyed */
1592	if (!process_info->n_vms) {
1593	WARN_ON(!list_empty(&process_info->kfd_bo_list));
1594	WARN_ON(!list_empty(&process_info->userptr_valid_list));
1595	WARN_ON(!list_empty(&process_info->userptr_inval_list));
1596
1597	dma_fence_put(fence: &process_info->eviction_fence->base);
1598	cancel_delayed_work_sync(dwork: &process_info->restore_userptr_work);
1599	put_pid(pid: process_info->pid);
1600	mutex_destroy(lock: &process_info->lock);
1601	mutex_destroy(lock: &process_info->notifier_lock);
1602	kfree(objp: process_info);
1603	}
1604	}
1605
1606	uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1607	{
1608	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1609	struct amdgpu_bo *pd = avm->root.bo;
1610	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
1611
1612	if (adev->asic_type < CHIP_VEGA10)
1613	return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1614	return avm->pd_phys_addr;
1615	}
1616
1617	void amdgpu_amdkfd_block_mmu_notifications(void *p)
1618	{
1619	struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1620
1621	mutex_lock(&pinfo->lock);
1622	WRITE_ONCE(pinfo->block_mmu_notifications, true);
1623	mutex_unlock(lock: &pinfo->lock);
1624	}
1625
1626	int amdgpu_amdkfd_criu_resume(void *p)
1627	{
1628	int ret = 0;
1629	struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1630
1631	mutex_lock(&pinfo->lock);
1632	pr_debug("scheduling work\n");
1633	mutex_lock(&pinfo->notifier_lock);
1634	pinfo->evicted_bos++;
1635	mutex_unlock(lock: &pinfo->notifier_lock);
1636	if (!READ_ONCE(pinfo->block_mmu_notifications)) {
1637	ret = -EINVAL;
1638	goto out_unlock;
1639	}
1640	WRITE_ONCE(pinfo->block_mmu_notifications, false);
1641	queue_delayed_work(wq: system_freezable_wq,
1642	dwork: &pinfo->restore_userptr_work, delay: 0);
1643
1644	out_unlock:
1645	mutex_unlock(lock: &pinfo->lock);
1646	return ret;
1647	}
1648
1649	size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
1650	uint8_t xcp_id)
1651	{
1652	uint64_t reserved_for_pt =
1653	ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
1654	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1655	uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0);
1656	ssize_t available;
1657	uint64_t vram_available, system_mem_available, ttm_mem_available;
1658
1659	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
1660	if (adev->apu_prefer_gtt && !adev->gmc.is_app_apu)
1661	vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
1662	- adev->kfd.vram_used_aligned[xcp_id];
1663	else
1664	vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
1665	- adev->kfd.vram_used_aligned[xcp_id]
1666	- atomic64_read(v: &adev->vram_pin_size)
1667	- reserved_for_pt
1668	- reserved_for_ras;
1669
1670	if (adev->apu_prefer_gtt) {
1671	system_mem_available = no_system_mem_limit ?
1672	kfd_mem_limit.max_system_mem_limit :
1673	kfd_mem_limit.max_system_mem_limit -
1674	kfd_mem_limit.system_mem_used;
1675
1676	ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -
1677	kfd_mem_limit.ttm_mem_used;
1678
1679	available = min3(system_mem_available, ttm_mem_available,
1680	vram_available);
1681	available = ALIGN_DOWN(available, PAGE_SIZE);
1682	} else {
1683	available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
1684	}
1685
1686	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
1687
1688	if (available < 0)
1689	available = 0;
1690
1691	return available;
1692	}
1693
1694	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1695	struct amdgpu_device *adev, uint64_t va, uint64_t size,
1696	void *drm_priv, struct kgd_mem **mem,
1697	uint64_t *offset, uint32_t flags, bool criu_resume)
1698	{
1699	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1700	struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
1701	enum ttm_bo_type bo_type = ttm_bo_type_device;
1702	struct sg_table *sg = NULL;
1703	uint64_t user_addr = 0;
1704	struct amdgpu_bo *bo;
1705	struct drm_gem_object *gobj = NULL;
1706	u32 domain, alloc_domain;
1707	uint64_t aligned_size;
1708	int8_t xcp_id = -1;
1709	u64 alloc_flags;
1710	int ret;
1711
1712	/*
1713	* Check on which domain to allocate BO
1714	*/
1715	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1716	domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1717
1718	if (adev->apu_prefer_gtt) {
1719	domain = AMDGPU_GEM_DOMAIN_GTT;
1720	alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1721	alloc_flags = 0;
1722	} else {
1723	alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1724	alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
1725	AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
1726
1727	/* For contiguous VRAM allocation */
1728	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS)
1729	alloc_flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1730	}
1731	xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?
1732	0 : fpriv->xcp_id;
1733	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
1734	domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1735	alloc_flags = 0;
1736	} else {
1737	domain = AMDGPU_GEM_DOMAIN_GTT;
1738	alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1739	alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
1740
1741	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1742	if (!offset || !*offset)
1743	return -EINVAL;
1744	user_addr = untagged_addr(*offset);
1745	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1746	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1747	bo_type = ttm_bo_type_sg;
1748	if (size > UINT_MAX)
1749	return -EINVAL;
1750	sg = create_sg_table(addr: *offset, size);
1751	if (!sg)
1752	return -ENOMEM;
1753	} else {
1754	return -EINVAL;
1755	}
1756	}
1757
1758	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
1759	alloc_flags |= AMDGPU_GEM_CREATE_COHERENT;
1760	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT)
1761	alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT;
1762	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED)
1763	alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED;
1764
1765	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1766	if (!*mem) {
1767	ret = -ENOMEM;
1768	goto err;
1769	}
1770	INIT_LIST_HEAD(list: &(*mem)->attachments);
1771	mutex_init(&(*mem)->lock);
1772	(*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1773
1774	/* Workaround for AQL queue wraparound bug. Map the same
1775	* memory twice. That means we only actually allocate half
1776	* the memory.
1777	*/
1778	if ((*mem)->aql_queue)
1779	size >>= 1;
1780	aligned_size = PAGE_ALIGN(size);
1781
1782	(*mem)->alloc_flags = flags;
1783
1784	amdgpu_sync_create(sync: &(*mem)->sync);
1785
1786	ret = amdgpu_amdkfd_reserve_mem_limit(adev, size: aligned_size, alloc_flag: flags,
1787	xcp_id);
1788	if (ret) {
1789	pr_debug("Insufficient memory\n");
1790	goto err_reserve_limit;
1791	}
1792
1793	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",
1794	va, (*mem)->aql_queue ? size << 1 : size,
1795	domain_string(alloc_domain), xcp_id);
1796
1797	ret = amdgpu_gem_object_create(adev, size: aligned_size, alignment: 1, initial_domain: alloc_domain, flags: alloc_flags,
1798	type: bo_type, NULL, obj: &gobj, xcp_id_plus1: xcp_id + 1);
1799	if (ret) {
1800	pr_debug("Failed to create BO on domain %s. ret %d\n",
1801	domain_string(alloc_domain), ret);
1802	goto err_bo_create;
1803	}
1804	ret = drm_vma_node_allow(node: &gobj->vma_node, tag: drm_priv);
1805	if (ret) {
1806	pr_debug("Failed to allow vma node access. ret %d\n", ret);
1807	goto err_node_allow;
1808	}
1809	ret = drm_gem_handle_create(file_priv: adev->kfd.client.file, obj: gobj, handlep: &(*mem)->gem_handle);
1810	if (ret)
1811	goto err_gem_handle_create;
1812	bo = gem_to_amdgpu_bo(gobj);
1813	if (bo_type == ttm_bo_type_sg) {
1814	bo->tbo.sg = sg;
1815	bo->tbo.ttm->sg = sg;
1816	}
1817	bo->kfd_bo = *mem;
1818	(*mem)->bo = bo;
1819	if (user_addr)
1820	bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1821
1822	(*mem)->va = va;
1823	(*mem)->domain = domain;
1824	(*mem)->mapped_to_gpu_memory = 0;
1825	(*mem)->process_info = avm->process_info;
1826
1827	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info: avm->process_info, userptr: user_addr);
1828
1829	if (user_addr) {
1830	pr_debug("creating userptr BO for user_addr = %llx\n", user_addr);
1831	ret = init_user_pages(mem: *mem, user_addr, criu_resume);
1832	if (ret)
1833	goto allocate_init_user_pages_failed;
1834	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1835	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1836	ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
1837	if (ret) {
1838	pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
1839	goto err_pin_bo;
1840	}
1841	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
1842	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
1843	} else {
1844	mutex_lock(&avm->process_info->lock);
1845	if (avm->process_info->eviction_fence &&
1846	!dma_fence_is_signaled(fence: &avm->process_info->eviction_fence->base))
1847	ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,
1848	fence: &avm->process_info->eviction_fence->base);
1849	mutex_unlock(lock: &avm->process_info->lock);
1850	if (ret)
1851	goto err_validate_bo;
1852	}
1853
1854	if (offset)
1855	*offset = amdgpu_bo_mmap_offset(bo);
1856
1857	return 0;
1858
1859	allocate_init_user_pages_failed:
1860	err_pin_bo:
1861	err_validate_bo:
1862	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info: avm->process_info);
1863	drm_gem_handle_delete(filp: adev->kfd.client.file, handle: (*mem)->gem_handle);
1864	err_gem_handle_create:
1865	drm_vma_node_revoke(node: &gobj->vma_node, tag: drm_priv);
1866	err_node_allow:
1867	/* Don't unreserve system mem limit twice */
1868	goto err_reserve_limit;
1869	err_bo_create:
1870	amdgpu_amdkfd_unreserve_mem_limit(adev, size: aligned_size, alloc_flag: flags, xcp_id);
1871	err_reserve_limit:
1872	amdgpu_sync_free(sync: &(*mem)->sync);
1873	mutex_destroy(lock: &(*mem)->lock);
1874	if (gobj)
1875	drm_gem_object_put(obj: gobj);
1876	else
1877	kfree(objp: *mem);
1878	err:
1879	if (sg) {
1880	sg_free_table(sg);
1881	kfree(objp: sg);
1882	}
1883	return ret;
1884	}
1885
1886	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1887	struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1888	uint64_t *size)
1889	{
1890	struct amdkfd_process_info *process_info = mem->process_info;
1891	unsigned long bo_size = mem->bo->tbo.base.size;
1892	bool use_release_notifier = (mem->bo->kfd_bo == mem);
1893	struct kfd_mem_attachment *entry, *tmp;
1894	struct bo_vm_reservation_context ctx;
1895	unsigned int mapped_to_gpu_memory;
1896	int ret;
1897	bool is_imported = false;
1898
1899	mutex_lock(&mem->lock);
1900
1901	/* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
1902	if (mem->alloc_flags &
1903	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1904	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1905	amdgpu_amdkfd_gpuvm_unpin_bo(bo: mem->bo);
1906	}
1907
1908	mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1909	is_imported = mem->is_imported;
1910	mutex_unlock(lock: &mem->lock);
1911	/* lock is not needed after this, since mem is unused and will
1912	* be freed anyway
1913	*/
1914
1915	if (mapped_to_gpu_memory > 0) {
1916	pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1917	mem->va, bo_size);
1918	return -EBUSY;
1919	}
1920
1921	/* Make sure restore workers don't access the BO any more */
1922	mutex_lock(&process_info->lock);
1923	if (!list_empty(head: &mem->validate_list))
1924	list_del_init(entry: &mem->validate_list);
1925	mutex_unlock(lock: &process_info->lock);
1926
1927	ret = reserve_bo_and_cond_vms(mem, NULL, map_type: BO_VM_ALL, ctx: &ctx);
1928	if (unlikely(ret))
1929	return ret;
1930
1931	/* Cleanup user pages and MMU notifiers */
1932	if (amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm)) {
1933	amdgpu_hmm_unregister(bo: mem->bo);
1934	amdgpu_hmm_range_free(range: mem->range);
1935	mem->range = NULL;
1936	}
1937
1938	amdgpu_amdkfd_remove_eviction_fence(bo: mem->bo,
1939	ef: process_info->eviction_fence);
1940	pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1941	mem->va + bo_size * (1 + mem->aql_queue));
1942
1943	/* Remove from VM internal data structures */
1944	list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {
1945	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
1946	kfd_mem_detach(attachment: entry);
1947	}
1948
1949	ret = unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
1950
1951	/* Free the sync object */
1952	amdgpu_sync_free(sync: &mem->sync);
1953
1954	/* If the SG is not NULL, it's one we created for a doorbell or mmio
1955	* remap BO. We need to free it.
1956	*/
1957	if (mem->bo->tbo.sg) {
1958	sg_free_table(mem->bo->tbo.sg);
1959	kfree(objp: mem->bo->tbo.sg);
1960	}
1961
1962	/* Update the size of the BO being freed if it was allocated from
1963	* VRAM and is not imported. For APP APU VRAM allocations are done
1964	* in GTT domain
1965	*/
1966	if (size) {
1967	if (!is_imported &&
1968	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1969	*size = bo_size;
1970	else
1971	*size = 0;
1972	}
1973
1974	/* Free the BO*/
1975	drm_vma_node_revoke(node: &mem->bo->tbo.base.vma_node, tag: drm_priv);
1976	drm_gem_handle_delete(filp: adev->kfd.client.file, handle: mem->gem_handle);
1977	if (mem->dmabuf) {
1978	dma_buf_put(dmabuf: mem->dmabuf);
1979	mem->dmabuf = NULL;
1980	}
1981	mutex_destroy(lock: &mem->lock);
1982
1983	/* If this releases the last reference, it will end up calling
1984	* amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1985	* this needs to be the last call here.
1986	*/
1987	drm_gem_object_put(obj: &mem->bo->tbo.base);
1988
1989	/*
1990	* For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),
1991	* explicitly free it here.
1992	*/
1993	if (!use_release_notifier)
1994	kfree(objp: mem);
1995
1996	return ret;
1997	}
1998
1999	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
2000	struct amdgpu_device *adev, struct kgd_mem *mem,
2001	void *drm_priv)
2002	{
2003	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2004	int ret;
2005	struct amdgpu_bo *bo;
2006	uint32_t domain;
2007	struct kfd_mem_attachment *entry;
2008	struct bo_vm_reservation_context ctx;
2009	unsigned long bo_size;
2010	bool is_invalid_userptr = false;
2011
2012	bo = mem->bo;
2013	if (!bo) {
2014	pr_err("Invalid BO when mapping memory to GPU\n");
2015	return -EINVAL;
2016	}
2017
2018	/* Make sure restore is not running concurrently. Since we
2019	* don't map invalid userptr BOs, we rely on the next restore
2020	* worker to do the mapping
2021	*/
2022	mutex_lock(&mem->process_info->lock);
2023
2024	/* Lock notifier lock. If we find an invalid userptr BO, we can be
2025	* sure that the MMU notifier is no longer running
2026	* concurrently and the queues are actually stopped
2027	*/
2028	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm)) {
2029	mutex_lock(&mem->process_info->notifier_lock);
2030	is_invalid_userptr = !!mem->invalid;
2031	mutex_unlock(lock: &mem->process_info->notifier_lock);
2032	}
2033
2034	mutex_lock(&mem->lock);
2035
2036	domain = mem->domain;
2037	bo_size = bo->tbo.base.size;
2038
2039	pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
2040	mem->va,
2041	mem->va + bo_size * (1 + mem->aql_queue),
2042	avm, domain_string(domain));
2043
2044	if (!kfd_mem_is_attached(avm, mem)) {
2045	ret = kfd_mem_attach(adev, mem, vm: avm, is_aql: mem->aql_queue);
2046	if (ret)
2047	goto out;
2048	}
2049
2050	ret = reserve_bo_and_vm(mem, vm: avm, ctx: &ctx);
2051	if (unlikely(ret))
2052	goto out;
2053
2054	/* Userptr can be marked as "not invalid", but not actually be
2055	* validated yet (still in the system domain). In that case
2056	* the queues are still stopped and we can leave mapping for
2057	* the next restore worker
2058	*/
2059	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm) &&
2060	bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
2061	is_invalid_userptr = true;
2062
2063	ret = vm_validate_pt_pd_bos(vm: avm, NULL);
2064	if (unlikely(ret))
2065	goto out_unreserve;
2066
2067	list_for_each_entry(entry, &mem->attachments, list) {
2068	if (entry->bo_va->base.vm != avm || entry->is_mapped)
2069	continue;
2070
2071	pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
2072	entry->va, entry->va + bo_size, entry);
2073
2074	ret = map_bo_to_gpuvm(mem, entry, sync: ctx.sync,
2075	no_update_pte: is_invalid_userptr);
2076	if (ret) {
2077	pr_err("Failed to map bo to gpuvm\n");
2078	goto out_unreserve;
2079	}
2080
2081	ret = vm_update_pds(vm: avm, sync: ctx.sync);
2082	if (ret) {
2083	pr_err("Failed to update page directories\n");
2084	goto out_unreserve;
2085	}
2086
2087	entry->is_mapped = true;
2088	mem->mapped_to_gpu_memory++;
2089	pr_debug("\t INC mapping count %d\n",
2090	mem->mapped_to_gpu_memory);
2091	}
2092
2093	ret = unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2094
2095	goto out;
2096
2097	out_unreserve:
2098	unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2099	out:
2100	mutex_unlock(lock: &mem->process_info->lock);
2101	mutex_unlock(lock: &mem->lock);
2102	return ret;
2103	}
2104
2105	int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)
2106	{
2107	struct kfd_mem_attachment *entry;
2108	struct amdgpu_vm *vm;
2109	int ret;
2110
2111	vm = drm_priv_to_vm(drm_priv);
2112
2113	mutex_lock(&mem->lock);
2114
2115	ret = amdgpu_bo_reserve(bo: mem->bo, no_intr: true);
2116	if (ret)
2117	goto out;
2118
2119	list_for_each_entry(entry, &mem->attachments, list) {
2120	if (entry->bo_va->base.vm != vm)
2121	continue;
2122	if (entry->bo_va->base.bo->tbo.ttm &&
2123	!entry->bo_va->base.bo->tbo.ttm->sg)
2124	continue;
2125
2126	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
2127	}
2128
2129	amdgpu_bo_unreserve(bo: mem->bo);
2130	out:
2131	mutex_unlock(lock: &mem->lock);
2132
2133	return ret;
2134	}
2135
2136	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
2137	struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2138	{
2139	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2140	unsigned long bo_size = mem->bo->tbo.base.size;
2141	struct kfd_mem_attachment *entry;
2142	struct bo_vm_reservation_context ctx;
2143	int ret;
2144
2145	mutex_lock(&mem->lock);
2146
2147	ret = reserve_bo_and_cond_vms(mem, vm: avm, map_type: BO_VM_MAPPED, ctx: &ctx);
2148	if (unlikely(ret))
2149	goto out;
2150	/* If no VMs were reserved, it means the BO wasn't actually mapped */
2151	if (ctx.n_vms == 0) {
2152	ret = -EINVAL;
2153	goto unreserve_out;
2154	}
2155
2156	ret = vm_validate_pt_pd_bos(vm: avm, NULL);
2157	if (unlikely(ret))
2158	goto unreserve_out;
2159
2160	pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
2161	mem->va,
2162	mem->va + bo_size * (1 + mem->aql_queue),
2163	avm);
2164
2165	list_for_each_entry(entry, &mem->attachments, list) {
2166	if (entry->bo_va->base.vm != avm || !entry->is_mapped)
2167	continue;
2168
2169	pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
2170	entry->va, entry->va + bo_size, entry);
2171
2172	ret = unmap_bo_from_gpuvm(mem, entry, sync: ctx.sync);
2173	if (ret)
2174	goto unreserve_out;
2175
2176	entry->is_mapped = false;
2177
2178	mem->mapped_to_gpu_memory--;
2179	pr_debug("\t DEC mapping count %d\n",
2180	mem->mapped_to_gpu_memory);
2181	}
2182
2183	unreserve_out:
2184	unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2185	out:
2186	mutex_unlock(lock: &mem->lock);
2187	return ret;
2188	}
2189
2190	int amdgpu_amdkfd_gpuvm_sync_memory(
2191	struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2192	{
2193	struct amdgpu_sync sync;
2194	int ret;
2195
2196	amdgpu_sync_create(sync: &sync);
2197
2198	mutex_lock(&mem->lock);
2199	amdgpu_sync_clone(source: &mem->sync, clone: &sync);
2200	mutex_unlock(lock: &mem->lock);
2201
2202	ret = amdgpu_sync_wait(sync: &sync, intr);
2203	amdgpu_sync_free(sync: &sync);
2204	return ret;
2205	}
2206
2207	/**
2208	* amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count
2209	* @bo: Buffer object to be mapped
2210	* @bo_gart: Return bo reference
2211	*
2212	* Before return, bo reference count is incremented. To release the reference and unpin/
2213	* unmap the BO, call amdgpu_amdkfd_free_gtt_mem.
2214	*/
2215	int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart)
2216	{
2217	int ret;
2218
2219	ret = amdgpu_bo_reserve(bo, no_intr: true);
2220	if (ret) {
2221	pr_err("Failed to reserve bo. ret %d\n", ret);
2222	goto err_reserve_bo_failed;
2223	}
2224
2225	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2226	if (ret) {
2227	pr_err("Failed to pin bo. ret %d\n", ret);
2228	goto err_pin_bo_failed;
2229	}
2230
2231	ret = amdgpu_ttm_alloc_gart(bo: &bo->tbo);
2232	if (ret) {
2233	pr_err("Failed to bind bo to GART. ret %d\n", ret);
2234	goto err_map_bo_gart_failed;
2235	}
2236
2237	amdgpu_amdkfd_remove_eviction_fence(
2238	bo, ef: bo->vm_bo->vm->process_info->eviction_fence);
2239
2240	amdgpu_bo_unreserve(bo);
2241
2242	*bo_gart = amdgpu_bo_ref(bo);
2243
2244	return 0;
2245
2246	err_map_bo_gart_failed:
2247	amdgpu_bo_unpin(bo);
2248	err_pin_bo_failed:
2249	amdgpu_bo_unreserve(bo);
2250	err_reserve_bo_failed:
2251
2252	return ret;
2253	}
2254
2255	/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
2256	*
2257	* @mem: Buffer object to be mapped for CPU access
2258	* @kptr[out]: pointer in kernel CPU address space
2259	* @size[out]: size of the buffer
2260	*
2261	* Pins the BO and maps it for kernel CPU access. The eviction fence is removed
2262	* from the BO, since pinned BOs cannot be evicted. The bo must remain on the
2263	* validate_list, so the GPU mapping can be restored after a page table was
2264	* evicted.
2265	*
2266	* Return: 0 on success, error code on failure
2267	*/
2268	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2269	void **kptr, uint64_t *size)
2270	{
2271	int ret;
2272	struct amdgpu_bo *bo = mem->bo;
2273
2274	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm)) {
2275	pr_err("userptr can't be mapped to kernel\n");
2276	return -EINVAL;
2277	}
2278
2279	mutex_lock(&mem->process_info->lock);
2280
2281	ret = amdgpu_bo_reserve(bo, no_intr: true);
2282	if (ret) {
2283	pr_err("Failed to reserve bo. ret %d\n", ret);
2284	goto bo_reserve_failed;
2285	}
2286
2287	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2288	if (ret) {
2289	pr_err("Failed to pin bo. ret %d\n", ret);
2290	goto pin_failed;
2291	}
2292
2293	ret = amdgpu_bo_kmap(bo, ptr: kptr);
2294	if (ret) {
2295	pr_err("Failed to map bo to kernel. ret %d\n", ret);
2296	goto kmap_failed;
2297	}
2298
2299	amdgpu_amdkfd_remove_eviction_fence(
2300	bo, ef: mem->process_info->eviction_fence);
2301
2302	if (size)
2303	*size = amdgpu_bo_size(bo);
2304
2305	amdgpu_bo_unreserve(bo);
2306
2307	mutex_unlock(lock: &mem->process_info->lock);
2308	return 0;
2309
2310	kmap_failed:
2311	amdgpu_bo_unpin(bo);
2312	pin_failed:
2313	amdgpu_bo_unreserve(bo);
2314	bo_reserve_failed:
2315	mutex_unlock(lock: &mem->process_info->lock);
2316
2317	return ret;
2318	}
2319
2320	/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
2321	*
2322	* @mem: Buffer object to be unmapped for CPU access
2323	*
2324	* Removes the kernel CPU mapping and unpins the BO. It does not restore the
2325	* eviction fence, so this function should only be used for cleanup before the
2326	* BO is destroyed.
2327	*/
2328	void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2329	{
2330	struct amdgpu_bo *bo = mem->bo;
2331
2332	(void)amdgpu_bo_reserve(bo, no_intr: true);
2333	amdgpu_bo_kunmap(bo);
2334	amdgpu_bo_unpin(bo);
2335	amdgpu_bo_unreserve(bo);
2336	}
2337
2338	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
2339	struct kfd_vm_fault_info *mem)
2340	{
2341	if (atomic_read_acquire(v: &adev->gmc.vm_fault_info_updated) == 1) {
2342	*mem = *adev->gmc.vm_fault_info;
2343	atomic_set_release(v: &adev->gmc.vm_fault_info_updated, i: 0);
2344	}
2345	return 0;
2346	}
2347
2348	static int import_obj_create(struct amdgpu_device *adev,
2349	struct dma_buf *dma_buf,
2350	struct drm_gem_object *obj,
2351	uint64_t va, void *drm_priv,
2352	struct kgd_mem **mem, uint64_t *size,
2353	uint64_t *mmap_offset)
2354	{
2355	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2356	struct amdgpu_bo *bo;
2357	int ret;
2358
2359	bo = gem_to_amdgpu_bo(obj);
2360	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
2361	AMDGPU_GEM_DOMAIN_GTT)))
2362	/* Only VRAM and GTT BOs are supported */
2363	return -EINVAL;
2364
2365	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2366	if (!*mem)
2367	return -ENOMEM;
2368
2369	ret = drm_vma_node_allow(node: &obj->vma_node, tag: drm_priv);
2370	if (ret)
2371	goto err_free_mem;
2372
2373	if (size)
2374	*size = amdgpu_bo_size(bo);
2375
2376	if (mmap_offset)
2377	*mmap_offset = amdgpu_bo_mmap_offset(bo);
2378
2379	INIT_LIST_HEAD(list: &(*mem)->attachments);
2380	mutex_init(&(*mem)->lock);
2381
2382	(*mem)->alloc_flags =
2383	((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2384	KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)
2385	| KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
2386	| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
2387
2388	get_dma_buf(dmabuf: dma_buf);
2389	(*mem)->dmabuf = dma_buf;
2390	(*mem)->bo = bo;
2391	(*mem)->va = va;
2392	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) &&
2393	!adev->apu_prefer_gtt ?
2394	AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
2395
2396	(*mem)->mapped_to_gpu_memory = 0;
2397	(*mem)->process_info = avm->process_info;
2398	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info: avm->process_info, userptr: false);
2399	amdgpu_sync_create(sync: &(*mem)->sync);
2400	(*mem)->is_imported = true;
2401
2402	mutex_lock(&avm->process_info->lock);
2403	if (avm->process_info->eviction_fence &&
2404	!dma_fence_is_signaled(fence: &avm->process_info->eviction_fence->base))
2405	ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain: (*mem)->domain,
2406	fence: &avm->process_info->eviction_fence->base);
2407	mutex_unlock(lock: &avm->process_info->lock);
2408	if (ret)
2409	goto err_remove_mem;
2410
2411	return 0;
2412
2413	err_remove_mem:
2414	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info: avm->process_info);
2415	drm_vma_node_revoke(node: &obj->vma_node, tag: drm_priv);
2416	err_free_mem:
2417	kfree(objp: *mem);
2418	return ret;
2419	}
2420
2421	int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd,
2422	uint64_t va, void *drm_priv,
2423	struct kgd_mem **mem, uint64_t *size,
2424	uint64_t *mmap_offset)
2425	{
2426	struct drm_gem_object *obj;
2427	uint32_t handle;
2428	int ret;
2429
2430	ret = drm_gem_prime_fd_to_handle(dev: &adev->ddev, file_priv: adev->kfd.client.file, prime_fd: fd,
2431	handle: &handle);
2432	if (ret)
2433	return ret;
2434	obj = drm_gem_object_lookup(filp: adev->kfd.client.file, handle);
2435	if (!obj) {
2436	ret = -EINVAL;
2437	goto err_release_handle;
2438	}
2439
2440	ret = import_obj_create(adev, dma_buf: obj->dma_buf, obj, va, drm_priv, mem, size,
2441	mmap_offset);
2442	if (ret)
2443	goto err_put_obj;
2444
2445	(*mem)->gem_handle = handle;
2446
2447	return 0;
2448
2449	err_put_obj:
2450	drm_gem_object_put(obj);
2451	err_release_handle:
2452	drm_gem_handle_delete(filp: adev->kfd.client.file, handle);
2453	return ret;
2454	}
2455
2456	int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
2457	struct dma_buf **dma_buf)
2458	{
2459	int ret;
2460
2461	mutex_lock(&mem->lock);
2462	ret = kfd_mem_export_dmabuf(mem);
2463	if (ret)
2464	goto out;
2465
2466	get_dma_buf(dmabuf: mem->dmabuf);
2467	*dma_buf = mem->dmabuf;
2468	out:
2469	mutex_unlock(lock: &mem->lock);
2470	return ret;
2471	}
2472
2473	/* Evict a userptr BO by stopping the queues if necessary
2474	*
2475	* Runs in MMU notifier, may be in RECLAIM_FS context. This means it
2476	* cannot do any memory allocations, and cannot take any locks that
2477	* are held elsewhere while allocating memory.
2478	*
2479	* It doesn't do anything to the BO itself. The real work happens in
2480	* restore, where we get updated page addresses. This function only
2481	* ensures that GPU access to the BO is stopped.
2482	*/
2483	int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,
2484	unsigned long cur_seq, struct kgd_mem *mem)
2485	{
2486	struct amdkfd_process_info *process_info = mem->process_info;
2487	int r = 0;
2488
2489	/* Do not process MMU notifications during CRIU restore until
2490	* KFD_CRIU_OP_RESUME IOCTL is received
2491	*/
2492	if (READ_ONCE(process_info->block_mmu_notifications))
2493	return 0;
2494
2495	mutex_lock(&process_info->notifier_lock);
2496	mmu_interval_set_seq(interval_sub: mni, cur_seq);
2497
2498	mem->invalid++;
2499	if (++process_info->evicted_bos == 1) {
2500	/* First eviction, stop the queues */
2501	r = kgd2kfd_quiesce_mm(mm: mni->mm,
2502	trigger: KFD_QUEUE_EVICTION_TRIGGER_USERPTR);
2503
2504	if (r && r != -ESRCH)
2505	pr_err("Failed to quiesce KFD\n");
2506
2507	if (r != -ESRCH)
2508	queue_delayed_work(wq: system_freezable_wq,
2509	dwork: &process_info->restore_userptr_work,
2510	delay: msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2511	}
2512	mutex_unlock(lock: &process_info->notifier_lock);
2513
2514	return r;
2515	}
2516
2517	/* Update invalid userptr BOs
2518	*
2519	* Moves invalidated (evicted) userptr BOs from userptr_valid_list to
2520	* userptr_inval_list and updates user pages for all BOs that have
2521	* been invalidated since their last update.
2522	*/
2523	static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
2524	struct mm_struct *mm)
2525	{
2526	struct kgd_mem *mem, *tmp_mem;
2527	struct amdgpu_bo *bo;
2528	struct ttm_operation_ctx ctx = { false, false };
2529	uint32_t invalid;
2530	int ret = 0;
2531
2532	mutex_lock(&process_info->notifier_lock);
2533
2534	/* Move all invalidated BOs to the userptr_inval_list */
2535	list_for_each_entry_safe(mem, tmp_mem,
2536	&process_info->userptr_valid_list,
2537	validate_list)
2538	if (mem->invalid)
2539	list_move_tail(list: &mem->validate_list,
2540	head: &process_info->userptr_inval_list);
2541
2542	/* Go through userptr_inval_list and update any invalid user_pages */
2543	list_for_each_entry(mem, &process_info->userptr_inval_list,
2544	validate_list) {
2545	invalid = mem->invalid;
2546	if (!invalid)
2547	/* BO hasn't been invalidated since the last
2548	* revalidation attempt. Keep its page list.
2549	*/
2550	continue;
2551
2552	bo = mem->bo;
2553
2554	amdgpu_hmm_range_free(range: mem->range);
2555	mem->range = NULL;
2556
2557	/* BO reservations and getting user pages (hmm_range_fault)
2558	* must happen outside the notifier lock
2559	*/
2560	mutex_unlock(lock: &process_info->notifier_lock);
2561
2562	/* Move the BO to system (CPU) domain if necessary to unmap
2563	* and free the SG table
2564	*/
2565	if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) {
2566	if (amdgpu_bo_reserve(bo, no_intr: true))
2567	return -EAGAIN;
2568	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
2569	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
2570	amdgpu_bo_unreserve(bo);
2571	if (ret) {
2572	pr_err("%s: Failed to invalidate userptr BO\n",
2573	__func__);
2574	return -EAGAIN;
2575	}
2576	}
2577
2578	mem->range = amdgpu_hmm_range_alloc(NULL);
2579	if (unlikely(!mem->range))
2580	return -ENOMEM;
2581	/* Get updated user pages */
2582	ret = amdgpu_ttm_tt_get_user_pages(bo, range: mem->range);
2583	if (ret) {
2584	amdgpu_hmm_range_free(range: mem->range);
2585	mem->range = NULL;
2586	pr_debug("Failed %d to get user pages\n", ret);
2587
2588	/* Return -EFAULT bad address error as success. It will
2589	* fail later with a VM fault if the GPU tries to access
2590	* it. Better than hanging indefinitely with stalled
2591	* user mode queues.
2592	*
2593	* Return other error -EBUSY or -ENOMEM to retry restore
2594	*/
2595	if (ret != -EFAULT)
2596	return ret;
2597
2598	/* If applications unmap memory before destroying the userptr
2599	* from the KFD, trigger a segmentation fault in VM debug mode.
2600	*/
2601	if (amdgpu_ttm_adev(bdev: bo->tbo.bdev)->debug_vm_userptr) {
2602	struct kfd_process *p;
2603
2604	pr_err("Pid %d unmapped memory before destroying userptr at GPU addr 0x%llx\n",
2605	pid_nr(process_info->pid), mem->va);
2606
2607	// Send GPU VM fault to user space
2608	p = kfd_lookup_process_by_pid(pid: process_info->pid);
2609	if (p) {
2610	kfd_signal_vm_fault_event_with_userptr(p, gpu_va: mem->va);
2611	kfd_unref_process(p);
2612	}
2613	}
2614
2615	ret = 0;
2616	}
2617
2618	amdgpu_ttm_tt_set_user_pages(ttm: bo->tbo.ttm, range: mem->range);
2619
2620	mutex_lock(&process_info->notifier_lock);
2621
2622	/* Mark the BO as valid unless it was invalidated
2623	* again concurrently.
2624	*/
2625	if (mem->invalid != invalid) {
2626	ret = -EAGAIN;
2627	goto unlock_out;
2628	}
2629	/* set mem valid if mem has hmm range associated */
2630	if (mem->range)
2631	mem->invalid = 0;
2632	}
2633
2634	unlock_out:
2635	mutex_unlock(lock: &process_info->notifier_lock);
2636
2637	return ret;
2638	}
2639
2640	/* Validate invalid userptr BOs
2641	*
2642	* Validates BOs on the userptr_inval_list. Also updates GPUVM page tables
2643	* with new page addresses and waits for the page table updates to complete.
2644	*/
2645	static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
2646	{
2647	struct ttm_operation_ctx ctx = { false, false };
2648	struct amdgpu_sync sync;
2649	struct drm_exec exec;
2650
2651	struct amdgpu_vm *peer_vm;
2652	struct kgd_mem *mem, *tmp_mem;
2653	struct amdgpu_bo *bo;
2654	int ret;
2655
2656	amdgpu_sync_create(sync: &sync);
2657
2658	drm_exec_init(exec: &exec, flags: 0, nr: 0);
2659	/* Reserve all BOs and page tables for validation */
2660	drm_exec_until_all_locked(&exec) {
2661	/* Reserve all the page directories */
2662	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2663	vm_list_node) {
2664	ret = amdgpu_vm_lock_pd(vm: peer_vm, exec: &exec, num_fences: 2);
2665	drm_exec_retry_on_contention(&exec);
2666	if (unlikely(ret))
2667	goto unreserve_out;
2668	}
2669
2670	/* Reserve the userptr_inval_list entries to resv_list */
2671	list_for_each_entry(mem, &process_info->userptr_inval_list,
2672	validate_list) {
2673	struct drm_gem_object *gobj;
2674
2675	gobj = &mem->bo->tbo.base;
2676	ret = drm_exec_prepare_obj(exec: &exec, obj: gobj, num_fences: 1);
2677	drm_exec_retry_on_contention(&exec);
2678	if (unlikely(ret))
2679	goto unreserve_out;
2680	}
2681	}
2682
2683	ret = process_validate_vms(process_info, NULL);
2684	if (ret)
2685	goto unreserve_out;
2686
2687	/* Validate BOs and update GPUVM page tables */
2688	list_for_each_entry_safe(mem, tmp_mem,
2689	&process_info->userptr_inval_list,
2690	validate_list) {
2691	struct kfd_mem_attachment *attachment;
2692
2693	bo = mem->bo;
2694
2695	/* Validate the BO if we got user pages */
2696	if (bo->tbo.ttm->pages[0]) {
2697	amdgpu_bo_placement_from_domain(abo: bo, domain: mem->domain);
2698	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
2699	if (ret) {
2700	pr_err("%s: failed to validate BO\n", __func__);
2701	goto unreserve_out;
2702	}
2703	}
2704
2705	/* Update mapping. If the BO was not validated
2706	* (because we couldn't get user pages), this will
2707	* clear the page table entries, which will result in
2708	* VM faults if the GPU tries to access the invalid
2709	* memory.
2710	*/
2711	list_for_each_entry(attachment, &mem->attachments, list) {
2712	if (!attachment->is_mapped)
2713	continue;
2714
2715	kfd_mem_dmaunmap_attachment(mem, attachment);
2716	ret = update_gpuvm_pte(mem, entry: attachment, sync: &sync);
2717	if (ret) {
2718	pr_err("%s: update PTE failed\n", __func__);
2719	/* make sure this gets validated again */
2720	mutex_lock(&process_info->notifier_lock);
2721	mem->invalid++;
2722	mutex_unlock(lock: &process_info->notifier_lock);
2723	goto unreserve_out;
2724	}
2725	}
2726	}
2727
2728	/* Update page directories */
2729	ret = process_update_pds(process_info, sync: &sync);
2730
2731	unreserve_out:
2732	drm_exec_fini(exec: &exec);
2733	amdgpu_sync_wait(sync: &sync, intr: false);
2734	amdgpu_sync_free(sync: &sync);
2735
2736	return ret;
2737	}
2738
2739	/* Confirm that all user pages are valid while holding the notifier lock
2740	*
2741	* Moves valid BOs from the userptr_inval_list back to userptr_val_list.
2742	*/
2743	static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)
2744	{
2745	struct kgd_mem *mem, *tmp_mem;
2746	int ret = 0;
2747
2748	list_for_each_entry_safe(mem, tmp_mem,
2749	&process_info->userptr_inval_list,
2750	validate_list) {
2751	bool valid;
2752
2753	/* keep mem without hmm range at userptr_inval_list */
2754	if (!mem->range)
2755	continue;
2756
2757	/* Only check mem with hmm range associated */
2758	valid = amdgpu_hmm_range_valid(range: mem->range);
2759	amdgpu_hmm_range_free(range: mem->range);
2760
2761	mem->range = NULL;
2762	if (!valid) {
2763	WARN(!mem->invalid, "Invalid BO not marked invalid");
2764	ret = -EAGAIN;
2765	continue;
2766	}
2767
2768	if (mem->invalid) {
2769	WARN(1, "Valid BO is marked invalid");
2770	ret = -EAGAIN;
2771	continue;
2772	}
2773
2774	list_move_tail(list: &mem->validate_list,
2775	head: &process_info->userptr_valid_list);
2776	}
2777
2778	return ret;
2779	}
2780
2781	/* Worker callback to restore evicted userptr BOs
2782	*
2783	* Tries to update and validate all userptr BOs. If successful and no
2784	* concurrent evictions happened, the queues are restarted. Otherwise,
2785	* reschedule for another attempt later.
2786	*/
2787	static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
2788	{
2789	struct delayed_work *dwork = to_delayed_work(work);
2790	struct amdkfd_process_info *process_info =
2791	container_of(dwork, struct amdkfd_process_info,
2792	restore_userptr_work);
2793	struct task_struct *usertask;
2794	struct mm_struct *mm;
2795	uint32_t evicted_bos;
2796
2797	mutex_lock(&process_info->notifier_lock);
2798	evicted_bos = process_info->evicted_bos;
2799	mutex_unlock(lock: &process_info->notifier_lock);
2800	if (!evicted_bos)
2801	return;
2802
2803	/* Reference task and mm in case of concurrent process termination */
2804	usertask = get_pid_task(pid: process_info->pid, PIDTYPE_PID);
2805	if (!usertask)
2806	return;
2807	mm = get_task_mm(task: usertask);
2808	if (!mm) {
2809	put_task_struct(t: usertask);
2810	return;
2811	}
2812
2813	mutex_lock(&process_info->lock);
2814
2815	if (update_invalid_user_pages(process_info, mm))
2816	goto unlock_out;
2817	/* userptr_inval_list can be empty if all evicted userptr BOs
2818	* have been freed. In that case there is nothing to validate
2819	* and we can just restart the queues.
2820	*/
2821	if (!list_empty(head: &process_info->userptr_inval_list)) {
2822	if (validate_invalid_user_pages(process_info))
2823	goto unlock_out;
2824	}
2825	/* Final check for concurrent evicton and atomic update. If
2826	* another eviction happens after successful update, it will
2827	* be a first eviction that calls quiesce_mm. The eviction
2828	* reference counting inside KFD will handle this case.
2829	*/
2830	mutex_lock(&process_info->notifier_lock);
2831	if (process_info->evicted_bos != evicted_bos)
2832	goto unlock_notifier_out;
2833
2834	if (confirm_valid_user_pages_locked(process_info)) {
2835	WARN(1, "User pages unexpectedly invalid");
2836	goto unlock_notifier_out;
2837	}
2838
2839	process_info->evicted_bos = evicted_bos = 0;
2840
2841	if (kgd2kfd_resume_mm(mm)) {
2842	pr_err("%s: Failed to resume KFD\n", __func__);
2843	/* No recovery from this failure. Probably the CP is
2844	* hanging. No point trying again.
2845	*/
2846	}
2847
2848	unlock_notifier_out:
2849	mutex_unlock(lock: &process_info->notifier_lock);
2850	unlock_out:
2851	mutex_unlock(lock: &process_info->lock);
2852
2853	/* If validation failed, reschedule another attempt */
2854	if (evicted_bos) {
2855	queue_delayed_work(wq: system_freezable_wq,
2856	dwork: &process_info->restore_userptr_work,
2857	delay: msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2858
2859	kfd_smi_event_queue_restore_rescheduled(mm);
2860	}
2861	mmput(mm);
2862	put_task_struct(t: usertask);
2863	}
2864
2865	static void replace_eviction_fence(struct dma_fence __rcu **ef,
2866	struct dma_fence *new_ef)
2867	{
2868	struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true
2869	/* protected by process_info->lock */);
2870
2871	/* If we're replacing an unsignaled eviction fence, that fence will
2872	* never be signaled, and if anyone is still waiting on that fence,
2873	* they will hang forever. This should never happen. We should only
2874	* replace the fence in restore_work that only gets scheduled after
2875	* eviction work signaled the fence.
2876	*/
2877	WARN_ONCE(!dma_fence_is_signaled(old_ef),
2878	"Replacing unsignaled eviction fence");
2879	dma_fence_put(fence: old_ef);
2880	}
2881
2882	/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
2883	* KFD process identified by process_info
2884	*
2885	* @process_info: amdkfd_process_info of the KFD process
2886	*
2887	* After memory eviction, restore thread calls this function. The function
2888	* should be called when the Process is still valid. BO restore involves -
2889	*
2890	* 1. Release old eviction fence and create new one
2891	* 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
2892	* 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
2893	* BOs that need to be reserved.
2894	* 4. Reserve all the BOs
2895	* 5. Validate of PD and PT BOs.
2896	* 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
2897	* 7. Add fence to all PD and PT BOs.
2898	* 8. Unreserve all BOs
2899	*/
2900	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef)
2901	{
2902	struct amdkfd_process_info *process_info = info;
2903	struct amdgpu_vm *peer_vm;
2904	struct kgd_mem *mem;
2905	struct list_head duplicate_save;
2906	struct amdgpu_sync sync_obj;
2907	unsigned long failed_size = 0;
2908	unsigned long total_size = 0;
2909	struct drm_exec exec;
2910	int ret;
2911
2912	INIT_LIST_HEAD(list: &duplicate_save);
2913
2914	mutex_lock(&process_info->lock);
2915
2916	drm_exec_init(exec: &exec, DRM_EXEC_IGNORE_DUPLICATES, nr: 0);
2917	drm_exec_until_all_locked(&exec) {
2918	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2919	vm_list_node) {
2920	ret = amdgpu_vm_lock_pd(vm: peer_vm, exec: &exec, num_fences: 2);
2921	drm_exec_retry_on_contention(&exec);
2922	if (unlikely(ret)) {
2923	pr_err("Locking VM PD failed, ret: %d\n", ret);
2924	goto ttm_reserve_fail;
2925	}
2926	}
2927
2928	/* Reserve all BOs and page tables/directory. Add all BOs from
2929	* kfd_bo_list to ctx.list
2930	*/
2931	list_for_each_entry(mem, &process_info->kfd_bo_list,
2932	validate_list) {
2933	struct drm_gem_object *gobj;
2934
2935	gobj = &mem->bo->tbo.base;
2936	ret = drm_exec_prepare_obj(exec: &exec, obj: gobj, num_fences: 1);
2937	drm_exec_retry_on_contention(&exec);
2938	if (unlikely(ret)) {
2939	pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret);
2940	goto ttm_reserve_fail;
2941	}
2942	}
2943	}
2944
2945	amdgpu_sync_create(sync: &sync_obj);
2946
2947	/* Validate BOs managed by KFD */
2948	list_for_each_entry(mem, &process_info->kfd_bo_list,
2949	validate_list) {
2950
2951	struct amdgpu_bo *bo = mem->bo;
2952	uint32_t domain = mem->domain;
2953	struct dma_resv_iter cursor;
2954	struct dma_fence *fence;
2955
2956	total_size += amdgpu_bo_size(bo);
2957
2958	ret = amdgpu_amdkfd_bo_validate(bo, domain, wait: false);
2959	if (ret) {
2960	pr_debug("Memory eviction: Validate BOs failed\n");
2961	failed_size += amdgpu_bo_size(bo);
2962	ret = amdgpu_amdkfd_bo_validate(bo,
2963	AMDGPU_GEM_DOMAIN_GTT, wait: false);
2964	if (ret) {
2965	pr_debug("Memory eviction: Try again\n");
2966	goto validate_map_fail;
2967	}
2968	}
2969	dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
2970	DMA_RESV_USAGE_KERNEL, fence) {
2971	ret = amdgpu_sync_fence(sync: &sync_obj, f: fence, GFP_KERNEL);
2972	if (ret) {
2973	pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2974	goto validate_map_fail;
2975	}
2976	}
2977	}
2978
2979	if (failed_size)
2980	pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
2981
2982	/* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO
2983	* validations above would invalidate DMABuf imports again.
2984	*/
2985	ret = process_validate_vms(process_info, ticket: &exec.ticket);
2986	if (ret) {
2987	pr_debug("Validating VMs failed, ret: %d\n", ret);
2988	goto validate_map_fail;
2989	}
2990
2991	/* Update mappings managed by KFD. */
2992	list_for_each_entry(mem, &process_info->kfd_bo_list,
2993	validate_list) {
2994	struct kfd_mem_attachment *attachment;
2995
2996	list_for_each_entry(attachment, &mem->attachments, list) {
2997	if (!attachment->is_mapped)
2998	continue;
2999
3000	kfd_mem_dmaunmap_attachment(mem, attachment);
3001	ret = update_gpuvm_pte(mem, entry: attachment, sync: &sync_obj);
3002	if (ret) {
3003	pr_debug("Memory eviction: update PTE failed. Try again\n");
3004	goto validate_map_fail;
3005	}
3006	}
3007	}
3008
3009	/* Update mappings not managed by KFD */
3010	list_for_each_entry(peer_vm, &process_info->vm_list_head,
3011	vm_list_node) {
3012	struct amdgpu_device *adev = amdgpu_ttm_adev(
3013	bdev: peer_vm->root.bo->tbo.bdev);
3014
3015	struct amdgpu_fpriv *fpriv =
3016	container_of(peer_vm, struct amdgpu_fpriv, vm);
3017
3018	ret = amdgpu_vm_bo_update(adev, bo_va: fpriv->prt_va, clear: false);
3019	if (ret) {
3020	dev_dbg(adev->dev,
3021	"Memory eviction: handle PRT moved failed, pid %8d. Try again.\n",
3022	pid_nr(process_info->pid));
3023	goto validate_map_fail;
3024	}
3025
3026	ret = amdgpu_vm_handle_moved(adev, vm: peer_vm, ticket: &exec.ticket);
3027	if (ret) {
3028	dev_dbg(adev->dev,
3029	"Memory eviction: handle moved failed, pid %8d. Try again.\n",
3030	pid_nr(process_info->pid));
3031	goto validate_map_fail;
3032	}
3033	}
3034
3035	/* Update page directories */
3036	ret = process_update_pds(process_info, sync: &sync_obj);
3037	if (ret) {
3038	pr_debug("Memory eviction: update PDs failed. Try again\n");
3039	goto validate_map_fail;
3040	}
3041
3042	/* Sync with fences on all the page tables. They implicitly depend on any
3043	* move fences from amdgpu_vm_handle_moved above.
3044	*/
3045	ret = process_sync_pds_resv(process_info, sync: &sync_obj);
3046	if (ret) {
3047	pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
3048	goto validate_map_fail;
3049	}
3050
3051	/* Wait for validate and PT updates to finish */
3052	amdgpu_sync_wait(sync: &sync_obj, intr: false);
3053
3054	/* The old eviction fence may be unsignaled if restore happens
3055	* after a GPU reset or suspend/resume. Keep the old fence in that
3056	* case. Otherwise release the old eviction fence and create new
3057	* one, because fence only goes from unsignaled to signaled once
3058	* and cannot be reused. Use context and mm from the old fence.
3059	*
3060	* If an old eviction fence signals after this check, that's OK.
3061	* Anyone signaling an eviction fence must stop the queues first
3062	* and schedule another restore worker.
3063	*/
3064	if (dma_fence_is_signaled(fence: &process_info->eviction_fence->base)) {
3065	struct amdgpu_amdkfd_fence *new_fence =
3066	amdgpu_amdkfd_fence_create(
3067	context: process_info->eviction_fence->base.context,
3068	mm: process_info->eviction_fence->mm,
3069	NULL);
3070
3071	if (!new_fence) {
3072	pr_err("Failed to create eviction fence\n");
3073	ret = -ENOMEM;
3074	goto validate_map_fail;
3075	}
3076	dma_fence_put(fence: &process_info->eviction_fence->base);
3077	process_info->eviction_fence = new_fence;
3078	replace_eviction_fence(ef, new_ef: dma_fence_get(fence: &new_fence->base));
3079	} else {
3080	WARN_ONCE(*ef != &process_info->eviction_fence->base,
3081	"KFD eviction fence doesn't match KGD process_info");
3082	}
3083
3084	/* Attach new eviction fence to all BOs except pinned ones */
3085	list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {
3086	if (mem->bo->tbo.pin_count)
3087	continue;
3088
3089	dma_resv_add_fence(obj: mem->bo->tbo.base.resv,
3090	fence: &process_info->eviction_fence->base,
3091	usage: DMA_RESV_USAGE_BOOKKEEP);
3092	}
3093	/* Attach eviction fence to PD / PT BOs and DMABuf imports */
3094	list_for_each_entry(peer_vm, &process_info->vm_list_head,
3095	vm_list_node) {
3096	struct amdgpu_bo *bo = peer_vm->root.bo;
3097
3098	dma_resv_add_fence(obj: bo->tbo.base.resv,
3099	fence: &process_info->eviction_fence->base,
3100	usage: DMA_RESV_USAGE_BOOKKEEP);
3101	}
3102
3103	validate_map_fail:
3104	amdgpu_sync_free(sync: &sync_obj);
3105	ttm_reserve_fail:
3106	drm_exec_fini(exec: &exec);
3107	mutex_unlock(lock: &process_info->lock);
3108	return ret;
3109	}
3110
3111	int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
3112	{
3113	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3114	struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
3115	int ret;
3116
3117	if (!info || !gws)
3118	return -EINVAL;
3119
3120	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
3121	if (!*mem)
3122	return -ENOMEM;
3123
3124	mutex_init(&(*mem)->lock);
3125	INIT_LIST_HEAD(list: &(*mem)->attachments);
3126	(*mem)->bo = amdgpu_bo_ref(bo: gws_bo);
3127	(*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
3128	(*mem)->process_info = process_info;
3129	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info, userptr: false);
3130	amdgpu_sync_create(sync: &(*mem)->sync);
3131
3132
3133	/* Validate gws bo the first time it is added to process */
3134	mutex_lock(&(*mem)->process_info->lock);
3135	ret = amdgpu_bo_reserve(bo: gws_bo, no_intr: false);
3136	if (unlikely(ret)) {
3137	pr_err("Reserve gws bo failed %d\n", ret);
3138	goto bo_reservation_failure;
3139	}
3140
3141	ret = amdgpu_amdkfd_bo_validate(bo: gws_bo, AMDGPU_GEM_DOMAIN_GWS, wait: true);
3142	if (ret) {
3143	pr_err("GWS BO validate failed %d\n", ret);
3144	goto bo_validation_failure;
3145	}
3146	/* GWS resource is shared b/t amdgpu and amdkfd
3147	* Add process eviction fence to bo so they can
3148	* evict each other.
3149	*/
3150	ret = dma_resv_reserve_fences(obj: gws_bo->tbo.base.resv, num_fences: 1);
3151	if (ret)
3152	goto reserve_shared_fail;
3153	dma_resv_add_fence(obj: gws_bo->tbo.base.resv,
3154	fence: &process_info->eviction_fence->base,
3155	usage: DMA_RESV_USAGE_BOOKKEEP);
3156	amdgpu_bo_unreserve(bo: gws_bo);
3157	mutex_unlock(lock: &(*mem)->process_info->lock);
3158
3159	return ret;
3160
3161	reserve_shared_fail:
3162	bo_validation_failure:
3163	amdgpu_bo_unreserve(bo: gws_bo);
3164	bo_reservation_failure:
3165	mutex_unlock(lock: &(*mem)->process_info->lock);
3166	amdgpu_sync_free(sync: &(*mem)->sync);
3167	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info);
3168	amdgpu_bo_unref(bo: &gws_bo);
3169	mutex_destroy(lock: &(*mem)->lock);
3170	kfree(objp: *mem);
3171	*mem = NULL;
3172	return ret;
3173	}
3174
3175	int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
3176	{
3177	int ret;
3178	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3179	struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
3180	struct amdgpu_bo *gws_bo = kgd_mem->bo;
3181
3182	/* Remove BO from process's validate list so restore worker won't touch
3183	* it anymore
3184	*/
3185	remove_kgd_mem_from_kfd_bo_list(mem: kgd_mem, process_info);
3186
3187	ret = amdgpu_bo_reserve(bo: gws_bo, no_intr: false);
3188	if (unlikely(ret)) {
3189	pr_err("Reserve gws bo failed %d\n", ret);
3190	//TODO add BO back to validate_list?
3191	return ret;
3192	}
3193	amdgpu_amdkfd_remove_eviction_fence(bo: gws_bo,
3194	ef: process_info->eviction_fence);
3195	amdgpu_bo_unreserve(bo: gws_bo);
3196	amdgpu_sync_free(sync: &kgd_mem->sync);
3197	amdgpu_bo_unref(bo: &gws_bo);
3198	mutex_destroy(lock: &kgd_mem->lock);
3199	kfree(objp: mem);
3200	return 0;
3201	}
3202
3203	/* Returns GPU-specific tiling mode information */
3204	int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
3205	struct tile_config *config)
3206	{
3207	config->gb_addr_config = adev->gfx.config.gb_addr_config;
3208	config->tile_config_ptr = adev->gfx.config.tile_mode_array;
3209	config->num_tile_configs =
3210	ARRAY_SIZE(adev->gfx.config.tile_mode_array);
3211	config->macro_tile_config_ptr =
3212	adev->gfx.config.macrotile_mode_array;
3213	config->num_macro_tile_configs =
3214	ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
3215
3216	/* Those values are not set from GFX9 onwards */
3217	config->num_banks = adev->gfx.config.num_banks;
3218	config->num_ranks = adev->gfx.config.num_ranks;
3219
3220	return 0;
3221	}
3222
3223	bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem)
3224	{
3225	struct amdgpu_vm *vm = drm_priv_to_vm(drm_priv);
3226	struct kfd_mem_attachment *entry;
3227
3228	list_for_each_entry(entry, &mem->attachments, list) {
3229	if (entry->is_mapped && entry->bo_va->base.vm == vm)
3230	return true;
3231	}
3232	return false;
3233	}
3234
3235	#if defined(CONFIG_DEBUG_FS)
3236
3237	int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)
3238	{
3239
3240	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
3241	seq_printf(m, fmt: "System mem used %lldM out of %lluM\n",
3242	(kfd_mem_limit.system_mem_used >> 20),
3243	(kfd_mem_limit.max_system_mem_limit >> 20));
3244	seq_printf(m, fmt: "TTM mem used %lldM out of %lluM\n",
3245	(kfd_mem_limit.ttm_mem_used >> 20),
3246	(kfd_mem_limit.max_ttm_mem_limit >> 20));
3247	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
3248
3249	return 0;
3250	}
3251
3252	#endif
3253