1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright 2014-2018 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23	#include <linux/dma-buf.h>
24	#include <linux/list.h>
25	#include <linux/pagemap.h>
26	#include <linux/sched/mm.h>
27	#include <linux/sched/task.h>
28	#include <linux/fdtable.h>
29	#include <drm/ttm/ttm_tt.h>
30
31	#include <drm/drm_exec.h>
32
33	#include "amdgpu_object.h"
34	#include "amdgpu_gem.h"
35	#include "amdgpu_vm.h"
36	#include "amdgpu_hmm.h"
37	#include "amdgpu_amdkfd.h"
38	#include "amdgpu_dma_buf.h"
39	#include <uapi/linux/kfd_ioctl.h>
40	#include "amdgpu_xgmi.h"
41	#include "kfd_priv.h"
42	#include "kfd_smi_events.h"
43
44	/* Userptr restore delay, just long enough to allow consecutive VM
45	* changes to accumulate
46	*/
47	#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
48	#define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29)
49
50	/*
51	* Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB
52	* BO chunk
53	*/
54	#define VRAM_AVAILABLITY_ALIGN (1 << 21)
55
56	/* Impose limit on how much memory KFD can use */
57	static struct {
58	uint64_t max_system_mem_limit;
59	uint64_t max_ttm_mem_limit;
60	int64_t system_mem_used;
61	int64_t ttm_mem_used;
62	spinlock_t mem_limit_lock;
63	} kfd_mem_limit;
64
65	static const char * const domain_bit_to_string[] = {
66	"CPU",
67	"GTT",
68	"VRAM",
69	"GDS",
70	"GWS",
71	"OA"
72	};
73
74	#define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
75
76	static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
77
78	static bool kfd_mem_is_attached(struct amdgpu_vm *avm,
79	struct kgd_mem *mem)
80	{
81	struct kfd_mem_attachment *entry;
82
83	list_for_each_entry(entry, &mem->attachments, list)
84	if (entry->bo_va->base.vm == avm)
85	return true;
86
87	return false;
88	}
89
90	/**
91	* reuse_dmamap() - Check whether adev can share the original
92	* userptr BO
93	*
94	* If both adev and bo_adev are in direct mapping or
95	* in the same iommu group, they can share the original BO.
96	*
97	* @adev: Device to which can or cannot share the original BO
98	* @bo_adev: Device to which allocated BO belongs to
99	*
100	* Return: returns true if adev can share original userptr BO,
101	* false otherwise.
102	*/
103	static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev)
104	{
105	return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) ||
106	(adev->dev->iommu_group == bo_adev->dev->iommu_group);
107	}
108
109	/* Set memory usage limits. Current, limits are
110	* System (TTM + userptr) memory - 15/16th System RAM
111	* TTM memory - 3/8th System RAM
112	*/
113	void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
114	{
115	struct sysinfo si;
116	uint64_t mem;
117
118	if (kfd_mem_limit.max_system_mem_limit)
119	return;
120
121	si_meminfo(val: &si);
122	mem = si.totalram - si.totalhigh;
123	mem *= si.mem_unit;
124
125	spin_lock_init(&kfd_mem_limit.mem_limit_lock);
126	kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6);
127	if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT)
128	kfd_mem_limit.max_system_mem_limit >>= 1;
129	else
130	kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT;
131
132	kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT;
133	pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
134	(kfd_mem_limit.max_system_mem_limit >> 20),
135	(kfd_mem_limit.max_ttm_mem_limit >> 20));
136	}
137
138	void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
139	{
140	kfd_mem_limit.system_mem_used += size;
141	}
142
143	/* Estimate page table size needed to represent a given memory size
144	*
145	* With 4KB pages, we need one 8 byte PTE for each 4KB of memory
146	* (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
147	* of memory (factor 256K, >> 18). ROCm user mode tries to optimize
148	* for 2MB pages for TLB efficiency. However, small allocations and
149	* fragmented system memory still need some 4KB pages. We choose a
150	* compromise that should work in most cases without reserving too
151	* much memory for page tables unnecessarily (factor 16K, >> 14).
152	*/
153
154	#define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM)
155
156	/**
157	* amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size
158	* of buffer.
159	*
160	* @adev: Device to which allocated BO belongs to
161	* @size: Size of buffer, in bytes, encapsulated by B0. This should be
162	* equivalent to amdgpu_bo_size(BO)
163	* @alloc_flag: Flag used in allocating a BO as noted above
164	* @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is
165	* managed as one compute node in driver for app
166	*
167	* Return:
168	* returns -ENOMEM in case of error, ZERO otherwise
169	*/
170	int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
171	uint64_t size, u32 alloc_flag, int8_t xcp_id)
172	{
173	uint64_t reserved_for_pt =
174	ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
175	size_t system_mem_needed, ttm_mem_needed, vram_needed;
176	int ret = 0;
177	uint64_t vram_size = 0;
178
179	system_mem_needed = 0;
180	ttm_mem_needed = 0;
181	vram_needed = 0;
182	if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
183	system_mem_needed = size;
184	ttm_mem_needed = size;
185	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
186	/*
187	* Conservatively round up the allocation requirement to 2 MB
188	* to avoid fragmentation caused by 4K allocations in the tail
189	* 2M BO chunk.
190	*/
191	vram_needed = size;
192	/*
193	* For GFX 9.4.3, get the VRAM size from XCP structs
194	*/
195	if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
196	return -EINVAL;
197
198	vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id);
199	if (adev->gmc.is_app_apu) {
200	system_mem_needed = size;
201	ttm_mem_needed = size;
202	}
203	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
204	system_mem_needed = size;
205	} else if (!(alloc_flag &
206	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
207	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
208	pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
209	return -ENOMEM;
210	}
211
212	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
213
214	if (kfd_mem_limit.system_mem_used + system_mem_needed >
215	kfd_mem_limit.max_system_mem_limit)
216	pr_debug("Set no_system_mem_limit=1 if using shared memory\n");
217
218	if ((kfd_mem_limit.system_mem_used + system_mem_needed >
219	kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||
220	(kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
221	kfd_mem_limit.max_ttm_mem_limit) ||
222	(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed >
223	vram_size - reserved_for_pt)) {
224	ret = -ENOMEM;
225	goto release;
226	}
227
228	/* Update memory accounting by decreasing available system
229	* memory, TTM memory and GPU memory as computed above
230	*/
231	WARN_ONCE(vram_needed && !adev,
232	"adev reference can't be null when vram is used");
233	if (adev && xcp_id >= 0) {
234	adev->kfd.vram_used[xcp_id] += vram_needed;
235	adev->kfd.vram_used_aligned[xcp_id] += adev->gmc.is_app_apu ?
236	vram_needed :
237	ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN);
238	}
239	kfd_mem_limit.system_mem_used += system_mem_needed;
240	kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
241
242	release:
243	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
244	return ret;
245	}
246
247	void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
248	uint64_t size, u32 alloc_flag, int8_t xcp_id)
249	{
250	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
251
252	if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
253	kfd_mem_limit.system_mem_used -= size;
254	kfd_mem_limit.ttm_mem_used -= size;
255	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
256	WARN_ONCE(!adev,
257	"adev reference can't be null when alloc mem flags vram is set");
258	if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id))
259	goto release;
260
261	if (adev) {
262	adev->kfd.vram_used[xcp_id] -= size;
263	if (adev->gmc.is_app_apu) {
264	adev->kfd.vram_used_aligned[xcp_id] -= size;
265	kfd_mem_limit.system_mem_used -= size;
266	kfd_mem_limit.ttm_mem_used -= size;
267	} else {
268	adev->kfd.vram_used_aligned[xcp_id] -=
269	ALIGN(size, VRAM_AVAILABLITY_ALIGN);
270	}
271	}
272	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
273	kfd_mem_limit.system_mem_used -= size;
274	} else if (!(alloc_flag &
275	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
276	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
277	pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);
278	goto release;
279	}
280	WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0,
281	"KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id);
282	WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
283	"KFD TTM memory accounting unbalanced");
284	WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
285	"KFD system memory accounting unbalanced");
286
287	release:
288	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
289	}
290
291	void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
292	{
293	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
294	u32 alloc_flags = bo->kfd_bo->alloc_flags;
295	u64 size = amdgpu_bo_size(bo);
296
297	amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flag: alloc_flags,
298	xcp_id: bo->xcp_id);
299
300	kfree(objp: bo->kfd_bo);
301	}
302
303	/**
304	* create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information
305	* about USERPTR or DOOREBELL or MMIO BO.
306	*
307	* @adev: Device for which dmamap BO is being created
308	* @mem: BO of peer device that is being DMA mapped. Provides parameters
309	* in building the dmamap BO
310	* @bo_out: Output parameter updated with handle of dmamap BO
311	*/
312	static int
313	create_dmamap_sg_bo(struct amdgpu_device *adev,
314	struct kgd_mem *mem, struct amdgpu_bo **bo_out)
315	{
316	struct drm_gem_object *gem_obj;
317	int ret;
318	uint64_t flags = 0;
319
320	ret = amdgpu_bo_reserve(bo: mem->bo, no_intr: false);
321	if (ret)
322	return ret;
323
324	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
325	flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
326	AMDGPU_GEM_CREATE_UNCACHED);
327
328	ret = amdgpu_gem_object_create(adev, size: mem->bo->tbo.base.size, alignment: 1,
329	AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags,
330	type: ttm_bo_type_sg, resv: mem->bo->tbo.base.resv, obj: &gem_obj, xcp_id_plus1: 0);
331
332	amdgpu_bo_unreserve(bo: mem->bo);
333
334	if (ret) {
335	pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
336	return -EINVAL;
337	}
338
339	*bo_out = gem_to_amdgpu_bo(gem_obj);
340	(*bo_out)->parent = amdgpu_bo_ref(bo: mem->bo);
341	return ret;
342	}
343
344	/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
345	* reservation object.
346	*
347	* @bo: [IN] Remove eviction fence(s) from this BO
348	* @ef: [IN] This eviction fence is removed if it
349	* is present in the shared list.
350	*
351	* NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
352	*/
353	static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
354	struct amdgpu_amdkfd_fence *ef)
355	{
356	struct dma_fence *replacement;
357
358	if (!ef)
359	return -EINVAL;
360
361	/* TODO: Instead of block before we should use the fence of the page
362	* table update and TLB flush here directly.
363	*/
364	replacement = dma_fence_get_stub();
365	dma_resv_replace_fences(obj: bo->tbo.base.resv, context: ef->base.context,
366	fence: replacement, usage: DMA_RESV_USAGE_BOOKKEEP);
367	dma_fence_put(fence: replacement);
368	return 0;
369	}
370
371	int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
372	{
373	struct amdgpu_bo *root = bo;
374	struct amdgpu_vm_bo_base *vm_bo;
375	struct amdgpu_vm *vm;
376	struct amdkfd_process_info *info;
377	struct amdgpu_amdkfd_fence *ef;
378	int ret;
379
380	/* we can always get vm_bo from root PD bo.*/
381	while (root->parent)
382	root = root->parent;
383
384	vm_bo = root->vm_bo;
385	if (!vm_bo)
386	return 0;
387
388	vm = vm_bo->vm;
389	if (!vm)
390	return 0;
391
392	info = vm->process_info;
393	if (!info || !info->eviction_fence)
394	return 0;
395
396	ef = container_of(dma_fence_get(&info->eviction_fence->base),
397	struct amdgpu_amdkfd_fence, base);
398
399	BUG_ON(!dma_resv_trylock(bo->tbo.base.resv));
400	ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef);
401	dma_resv_unlock(obj: bo->tbo.base.resv);
402
403	dma_fence_put(fence: &ef->base);
404	return ret;
405	}
406
407	static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
408	bool wait)
409	{
410	struct ttm_operation_ctx ctx = { false, false };
411	int ret;
412
413	if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
414	"Called with userptr BO"))
415	return -EINVAL;
416
417	amdgpu_bo_placement_from_domain(abo: bo, domain);
418
419	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
420	if (ret)
421	goto validate_fail;
422	if (wait)
423	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
424
425	validate_fail:
426	return ret;
427	}
428
429	int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
430	uint32_t domain,
431	struct dma_fence *fence)
432	{
433	int ret = amdgpu_bo_reserve(bo, no_intr: false);
434
435	if (ret)
436	return ret;
437
438	ret = amdgpu_amdkfd_bo_validate(bo, domain, wait: true);
439	if (ret)
440	goto unreserve_out;
441
442	ret = dma_resv_reserve_fences(obj: bo->tbo.base.resv, num_fences: 1);
443	if (ret)
444	goto unreserve_out;
445
446	dma_resv_add_fence(obj: bo->tbo.base.resv, fence,
447	usage: DMA_RESV_USAGE_BOOKKEEP);
448
449	unreserve_out:
450	amdgpu_bo_unreserve(bo);
451
452	return ret;
453	}
454
455	static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
456	{
457	return amdgpu_amdkfd_bo_validate(bo, domain: bo->allowed_domains, wait: false);
458	}
459
460	/* vm_validate_pt_pd_bos - Validate page table and directory BOs
461	*
462	* Page directories are not updated here because huge page handling
463	* during page table updates can invalidate page directory entries
464	* again. Page directories are only updated after updating page
465	* tables.
466	*/
467	static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm,
468	struct ww_acquire_ctx *ticket)
469	{
470	struct amdgpu_bo *pd = vm->root.bo;
471	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
472	int ret;
473
474	ret = amdgpu_vm_validate(adev, vm, ticket,
475	callback: amdgpu_amdkfd_validate_vm_bo, NULL);
476	if (ret) {
477	pr_err("failed to validate PT BOs\n");
478	return ret;
479	}
480
481	vm->pd_phys_addr = amdgpu_gmc_pd_addr(bo: vm->root.bo);
482
483	return 0;
484	}
485
486	static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
487	{
488	struct amdgpu_bo *pd = vm->root.bo;
489	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
490	int ret;
491
492	ret = amdgpu_vm_update_pdes(adev, vm, immediate: false);
493	if (ret)
494	return ret;
495
496	return amdgpu_sync_fence(sync, f: vm->last_update);
497	}
498
499	static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
500	{
501	uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |
502	AMDGPU_VM_MTYPE_DEFAULT;
503
504	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)
505	mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
506	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
507	mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
508
509	return amdgpu_gem_va_map_flags(adev, flags: mapping_flags);
510	}
511
512	/**
513	* create_sg_table() - Create an sg_table for a contiguous DMA addr range
514	* @addr: The starting address to point to
515	* @size: Size of memory area in bytes being pointed to
516	*
517	* Allocates an instance of sg_table and initializes it to point to memory
518	* area specified by input parameters. The address used to build is assumed
519	* to be DMA mapped, if needed.
520	*
521	* DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
522	* because they are physically contiguous.
523	*
524	* Return: Initialized instance of SG Table or NULL
525	*/
526	static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
527	{
528	struct sg_table *sg = kmalloc(size: sizeof(*sg), GFP_KERNEL);
529
530	if (!sg)
531	return NULL;
532	if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
533	kfree(objp: sg);
534	return NULL;
535	}
536	sg_dma_address(sg->sgl) = addr;
537	sg->sgl->length = size;
538	#ifdef CONFIG_NEED_SG_DMA_LENGTH
539	sg->sgl->dma_length = size;
540	#endif
541	return sg;
542	}
543
544	static int
545	kfd_mem_dmamap_userptr(struct kgd_mem *mem,
546	struct kfd_mem_attachment *attachment)
547	{
548	enum dma_data_direction direction =
549	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
550	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
551	struct ttm_operation_ctx ctx = {.interruptible = true};
552	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
553	struct amdgpu_device *adev = attachment->adev;
554	struct ttm_tt *src_ttm = mem->bo->tbo.ttm;
555	struct ttm_tt *ttm = bo->tbo.ttm;
556	int ret;
557
558	if (WARN_ON(ttm->num_pages != src_ttm->num_pages))
559	return -EINVAL;
560
561	ttm->sg = kmalloc(size: sizeof(*ttm->sg), GFP_KERNEL);
562	if (unlikely(!ttm->sg))
563	return -ENOMEM;
564
565	/* Same sequence as in amdgpu_ttm_tt_pin_userptr */
566	ret = sg_alloc_table_from_pages(sgt: ttm->sg, pages: src_ttm->pages,
567	n_pages: ttm->num_pages, offset: 0,
568	size: (u64)ttm->num_pages << PAGE_SHIFT,
569	GFP_KERNEL);
570	if (unlikely(ret))
571	goto free_sg;
572
573	ret = dma_map_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
574	if (unlikely(ret))
575	goto release_sg;
576
577	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
578	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
579	if (ret)
580	goto unmap_sg;
581
582	return 0;
583
584	unmap_sg:
585	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
586	release_sg:
587	pr_err("DMA map userptr failed: %d\n", ret);
588	sg_free_table(ttm->sg);
589	free_sg:
590	kfree(objp: ttm->sg);
591	ttm->sg = NULL;
592	return ret;
593	}
594
595	static int
596	kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
597	{
598	struct ttm_operation_ctx ctx = {.interruptible = true};
599	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
600	int ret;
601
602	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
603	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
604	if (ret)
605	return ret;
606
607	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
608	return ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
609	}
610
611	/**
612	* kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
613	* @mem: SG BO of the DOORBELL or MMIO resource on the owning device
614	* @attachment: Virtual address attachment of the BO on accessing device
615	*
616	* An access request from the device that owns DOORBELL does not require DMA mapping.
617	* This is because the request doesn't go through PCIe root complex i.e. it instead
618	* loops back. The need to DMA map arises only when accessing peer device's DOORBELL
619	*
620	* In contrast, all access requests for MMIO need to be DMA mapped without regard to
621	* device ownership. This is because access requests for MMIO go through PCIe root
622	* complex.
623	*
624	* This is accomplished in two steps:
625	* - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
626	* in updating requesting device's page table
627	* - Signal TTM to mark memory pointed to by requesting device's BO as GPU
628	* accessible. This allows an update of requesting device's page table
629	* with entries associated with DOOREBELL or MMIO memory
630	*
631	* This method is invoked in the following contexts:
632	* - Mapping of DOORBELL or MMIO BO of same or peer device
633	* - Validating an evicted DOOREBELL or MMIO BO on device seeking access
634	*
635	* Return: ZERO if successful, NON-ZERO otherwise
636	*/
637	static int
638	kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
639	struct kfd_mem_attachment *attachment)
640	{
641	struct ttm_operation_ctx ctx = {.interruptible = true};
642	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
643	struct amdgpu_device *adev = attachment->adev;
644	struct ttm_tt *ttm = bo->tbo.ttm;
645	enum dma_data_direction dir;
646	dma_addr_t dma_addr;
647	bool mmio;
648	int ret;
649
650	/* Expect SG Table of dmapmap BO to be NULL */
651	mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
652	if (unlikely(ttm->sg)) {
653	pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
654	return -EINVAL;
655	}
656
657	dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
658	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
659	dma_addr = mem->bo->tbo.sg->sgl->dma_address;
660	pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
661	pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
662	dma_addr = dma_map_resource(dev: adev->dev, phys_addr: dma_addr,
663	size: mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
664	ret = dma_mapping_error(dev: adev->dev, dma_addr);
665	if (unlikely(ret))
666	return ret;
667	pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
668
669	ttm->sg = create_sg_table(addr: dma_addr, size: mem->bo->tbo.sg->sgl->length);
670	if (unlikely(!ttm->sg)) {
671	ret = -ENOMEM;
672	goto unmap_sg;
673	}
674
675	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_GTT);
676	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
677	if (unlikely(ret))
678	goto free_sg;
679
680	return ret;
681
682	free_sg:
683	sg_free_table(ttm->sg);
684	kfree(objp: ttm->sg);
685	ttm->sg = NULL;
686	unmap_sg:
687	dma_unmap_resource(dev: adev->dev, addr: dma_addr, size: mem->bo->tbo.sg->sgl->length,
688	dir, DMA_ATTR_SKIP_CPU_SYNC);
689	return ret;
690	}
691
692	static int
693	kfd_mem_dmamap_attachment(struct kgd_mem *mem,
694	struct kfd_mem_attachment *attachment)
695	{
696	switch (attachment->type) {
697	case KFD_MEM_ATT_SHARED:
698	return 0;
699	case KFD_MEM_ATT_USERPTR:
700	return kfd_mem_dmamap_userptr(mem, attachment);
701	case KFD_MEM_ATT_DMABUF:
702	return kfd_mem_dmamap_dmabuf(attachment);
703	case KFD_MEM_ATT_SG:
704	return kfd_mem_dmamap_sg_bo(mem, attachment);
705	default:
706	WARN_ON_ONCE(1);
707	}
708	return -EINVAL;
709	}
710
711	static void
712	kfd_mem_dmaunmap_userptr(struct kgd_mem *mem,
713	struct kfd_mem_attachment *attachment)
714	{
715	enum dma_data_direction direction =
716	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
717	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
718	struct ttm_operation_ctx ctx = {.interruptible = false};
719	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
720	struct amdgpu_device *adev = attachment->adev;
721	struct ttm_tt *ttm = bo->tbo.ttm;
722
723	if (unlikely(!ttm->sg))
724	return;
725
726	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
727	ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
728
729	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
730	sg_free_table(ttm->sg);
731	kfree(objp: ttm->sg);
732	ttm->sg = NULL;
733	}
734
735	static void
736	kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
737	{
738	/* This is a no-op. We don't want to trigger eviction fences when
739	* unmapping DMABufs. Therefore the invalidation (moving to system
740	* domain) is done in kfd_mem_dmamap_dmabuf.
741	*/
742	}
743
744	/**
745	* kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
746	* @mem: SG BO of the DOORBELL or MMIO resource on the owning device
747	* @attachment: Virtual address attachment of the BO on accessing device
748	*
749	* The method performs following steps:
750	* - Signal TTM to mark memory pointed to by BO as GPU inaccessible
751	* - Free SG Table that is used to encapsulate DMA mapped memory of
752	* peer device's DOORBELL or MMIO memory
753	*
754	* This method is invoked in the following contexts:
755	* UNMapping of DOORBELL or MMIO BO on a device having access to its memory
756	* Eviction of DOOREBELL or MMIO BO on device having access to its memory
757	*
758	* Return: void
759	*/
760	static void
761	kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
762	struct kfd_mem_attachment *attachment)
763	{
764	struct ttm_operation_ctx ctx = {.interruptible = true};
765	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
766	struct amdgpu_device *adev = attachment->adev;
767	struct ttm_tt *ttm = bo->tbo.ttm;
768	enum dma_data_direction dir;
769
770	if (unlikely(!ttm->sg)) {
771	pr_debug("SG Table of BO is NULL");
772	return;
773	}
774
775	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
776	ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
777
778	dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
779	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
780	dma_unmap_resource(dev: adev->dev, addr: ttm->sg->sgl->dma_address,
781	size: ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
782	sg_free_table(ttm->sg);
783	kfree(objp: ttm->sg);
784	ttm->sg = NULL;
785	bo->tbo.sg = NULL;
786	}
787
788	static void
789	kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
790	struct kfd_mem_attachment *attachment)
791	{
792	switch (attachment->type) {
793	case KFD_MEM_ATT_SHARED:
794	break;
795	case KFD_MEM_ATT_USERPTR:
796	kfd_mem_dmaunmap_userptr(mem, attachment);
797	break;
798	case KFD_MEM_ATT_DMABUF:
799	kfd_mem_dmaunmap_dmabuf(attachment);
800	break;
801	case KFD_MEM_ATT_SG:
802	kfd_mem_dmaunmap_sg_bo(mem, attachment);
803	break;
804	default:
805	WARN_ON_ONCE(1);
806	}
807	}
808
809	static int kfd_mem_export_dmabuf(struct kgd_mem *mem)
810	{
811	if (!mem->dmabuf) {
812	struct amdgpu_device *bo_adev;
813	struct dma_buf *dmabuf;
814	int r, fd;
815
816	bo_adev = amdgpu_ttm_adev(bdev: mem->bo->tbo.bdev);
817	r = drm_gem_prime_handle_to_fd(dev: &bo_adev->ddev, file_priv: bo_adev->kfd.client.file,
818	handle: mem->gem_handle,
819	flags: mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
820	DRM_RDWR : 0, prime_fd: &fd);
821	if (r)
822	return r;
823	dmabuf = dma_buf_get(fd);
824	close_fd(fd);
825	if (WARN_ON_ONCE(IS_ERR(dmabuf)))
826	return PTR_ERR(ptr: dmabuf);
827	mem->dmabuf = dmabuf;
828	}
829
830	return 0;
831	}
832
833	static int
834	kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
835	struct amdgpu_bo **bo)
836	{
837	struct drm_gem_object *gobj;
838	int ret;
839
840	ret = kfd_mem_export_dmabuf(mem);
841	if (ret)
842	return ret;
843
844	gobj = amdgpu_gem_prime_import(dev: adev_to_drm(adev), dma_buf: mem->dmabuf);
845	if (IS_ERR(ptr: gobj))
846	return PTR_ERR(ptr: gobj);
847
848	*bo = gem_to_amdgpu_bo(gobj);
849	(*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE;
850
851	return 0;
852	}
853
854	/* kfd_mem_attach - Add a BO to a VM
855	*
856	* Everything that needs to bo done only once when a BO is first added
857	* to a VM. It can later be mapped and unmapped many times without
858	* repeating these steps.
859	*
860	* 0. Create BO for DMA mapping, if needed
861	* 1. Allocate and initialize BO VA entry data structure
862	* 2. Add BO to the VM
863	* 3. Determine ASIC-specific PTE flags
864	* 4. Alloc page tables and directories if needed
865	* 4a. Validate new page tables and directories
866	*/
867	static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
868	struct amdgpu_vm *vm, bool is_aql)
869	{
870	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bdev: mem->bo->tbo.bdev);
871	unsigned long bo_size = mem->bo->tbo.base.size;
872	uint64_t va = mem->va;
873	struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
874	struct amdgpu_bo *bo[2] = {NULL, NULL};
875	struct amdgpu_bo_va *bo_va;
876	bool same_hive = false;
877	int i, ret;
878
879	if (!va) {
880	pr_err("Invalid VA when adding BO to VM\n");
881	return -EINVAL;
882	}
883
884	/* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
885	*
886	* The access path of MMIO and DOORBELL BOs of is always over PCIe.
887	* In contrast the access path of VRAM BOs depens upon the type of
888	* link that connects the peer device. Access over PCIe is allowed
889	* if peer device has large BAR. In contrast, access over xGMI is
890	* allowed for both small and large BAR configurations of peer device
891	*/
892	if ((adev != bo_adev && !adev->gmc.is_app_apu) &&
893	((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
894	(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
895	(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
896	if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
897	same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
898	if (!same_hive && !amdgpu_device_is_peer_accessible(adev: bo_adev, peer_adev: adev))
899	return -EINVAL;
900	}
901
902	for (i = 0; i <= is_aql; i++) {
903	attachment[i] = kzalloc(size: sizeof(*attachment[i]), GFP_KERNEL);
904	if (unlikely(!attachment[i])) {
905	ret = -ENOMEM;
906	goto unwind;
907	}
908
909	pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
910	va + bo_size, vm);
911
912	if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
913	(amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) ||
914	(mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) ||
915	same_hive) {
916	/* Mappings on the local GPU, or VRAM mappings in the
917	* local hive, or userptr, or GTT mapping can reuse dma map
918	* address space share the original BO
919	*/
920	attachment[i]->type = KFD_MEM_ATT_SHARED;
921	bo[i] = mem->bo;
922	drm_gem_object_get(obj: &bo[i]->tbo.base);
923	} else if (i > 0) {
924	/* Multiple mappings on the same GPU share the BO */
925	attachment[i]->type = KFD_MEM_ATT_SHARED;
926	bo[i] = bo[0];
927	drm_gem_object_get(obj: &bo[i]->tbo.base);
928	} else if (amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm)) {
929	/* Create an SG BO to DMA-map userptrs on other GPUs */
930	attachment[i]->type = KFD_MEM_ATT_USERPTR;
931	ret = create_dmamap_sg_bo(adev, mem, bo_out: &bo[i]);
932	if (ret)
933	goto unwind;
934	/* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
935	} else if (mem->bo->tbo.type == ttm_bo_type_sg) {
936	WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
937	mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
938	"Handing invalid SG BO in ATTACH request");
939	attachment[i]->type = KFD_MEM_ATT_SG;
940	ret = create_dmamap_sg_bo(adev, mem, bo_out: &bo[i]);
941	if (ret)
942	goto unwind;
943	/* Enable acces to GTT and VRAM BOs of peer devices */
944	} else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
945	mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
946	attachment[i]->type = KFD_MEM_ATT_DMABUF;
947	ret = kfd_mem_attach_dmabuf(adev, mem, bo: &bo[i]);
948	if (ret)
949	goto unwind;
950	pr_debug("Employ DMABUF mechanism to enable peer GPU access\n");
951	} else {
952	WARN_ONCE(true, "Handling invalid ATTACH request");
953	ret = -EINVAL;
954	goto unwind;
955	}
956
957	/* Add BO to VM internal data structures */
958	ret = amdgpu_bo_reserve(bo: bo[i], no_intr: false);
959	if (ret) {
960	pr_debug("Unable to reserve BO during memory attach");
961	goto unwind;
962	}
963	bo_va = amdgpu_vm_bo_find(vm, bo: bo[i]);
964	if (!bo_va)
965	bo_va = amdgpu_vm_bo_add(adev, vm, bo: bo[i]);
966	else
967	++bo_va->ref_count;
968	attachment[i]->bo_va = bo_va;
969	amdgpu_bo_unreserve(bo: bo[i]);
970	if (unlikely(!attachment[i]->bo_va)) {
971	ret = -ENOMEM;
972	pr_err("Failed to add BO object to VM. ret == %d\n",
973	ret);
974	goto unwind;
975	}
976	attachment[i]->va = va;
977	attachment[i]->pte_flags = get_pte_flags(adev, mem);
978	attachment[i]->adev = adev;
979	list_add(new: &attachment[i]->list, head: &mem->attachments);
980
981	va += bo_size;
982	}
983
984	return 0;
985
986	unwind:
987	for (; i >= 0; i--) {
988	if (!attachment[i])
989	continue;
990	if (attachment[i]->bo_va) {
991	amdgpu_bo_reserve(bo: bo[i], no_intr: true);
992	if (--attachment[i]->bo_va->ref_count == 0)
993	amdgpu_vm_bo_del(adev, bo_va: attachment[i]->bo_va);
994	amdgpu_bo_unreserve(bo: bo[i]);
995	list_del(entry: &attachment[i]->list);
996	}
997	if (bo[i])
998	drm_gem_object_put(obj: &bo[i]->tbo.base);
999	kfree(objp: attachment[i]);
1000	}
1001	return ret;
1002	}
1003
1004	static void kfd_mem_detach(struct kfd_mem_attachment *attachment)
1005	{
1006	struct amdgpu_bo *bo = attachment->bo_va->base.bo;
1007
1008	pr_debug("\t remove VA 0x%llx in entry %p\n",
1009	attachment->va, attachment);
1010	if (--attachment->bo_va->ref_count == 0)
1011	amdgpu_vm_bo_del(adev: attachment->adev, bo_va: attachment->bo_va);
1012	drm_gem_object_put(obj: &bo->tbo.base);
1013	list_del(entry: &attachment->list);
1014	kfree(objp: attachment);
1015	}
1016
1017	static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
1018	struct amdkfd_process_info *process_info,
1019	bool userptr)
1020	{
1021	mutex_lock(&process_info->lock);
1022	if (userptr)
1023	list_add_tail(new: &mem->validate_list,
1024	head: &process_info->userptr_valid_list);
1025	else
1026	list_add_tail(new: &mem->validate_list, head: &process_info->kfd_bo_list);
1027	mutex_unlock(lock: &process_info->lock);
1028	}
1029
1030	static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
1031	struct amdkfd_process_info *process_info)
1032	{
1033	mutex_lock(&process_info->lock);
1034	list_del(entry: &mem->validate_list);
1035	mutex_unlock(lock: &process_info->lock);
1036	}
1037
1038	/* Initializes user pages. It registers the MMU notifier and validates
1039	* the userptr BO in the GTT domain.
1040	*
1041	* The BO must already be on the userptr_valid_list. Otherwise an
1042	* eviction and restore may happen that leaves the new BO unmapped
1043	* with the user mode queues running.
1044	*
1045	* Takes the process_info->lock to protect against concurrent restore
1046	* workers.
1047	*
1048	* Returns 0 for success, negative errno for errors.
1049	*/
1050	static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
1051	bool criu_resume)
1052	{
1053	struct amdkfd_process_info *process_info = mem->process_info;
1054	struct amdgpu_bo *bo = mem->bo;
1055	struct ttm_operation_ctx ctx = { true, false };
1056	struct hmm_range *range;
1057	int ret = 0;
1058
1059	mutex_lock(&process_info->lock);
1060
1061	ret = amdgpu_ttm_tt_set_userptr(bo: &bo->tbo, addr: user_addr, flags: 0);
1062	if (ret) {
1063	pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
1064	goto out;
1065	}
1066
1067	ret = amdgpu_hmm_register(bo, addr: user_addr);
1068	if (ret) {
1069	pr_err("%s: Failed to register MMU notifier: %d\n",
1070	__func__, ret);
1071	goto out;
1072	}
1073
1074	if (criu_resume) {
1075	/*
1076	* During a CRIU restore operation, the userptr buffer objects
1077	* will be validated in the restore_userptr_work worker at a
1078	* later stage when it is scheduled by another ioctl called by
1079	* CRIU master process for the target pid for restore.
1080	*/
1081	mutex_lock(&process_info->notifier_lock);
1082	mem->invalid++;
1083	mutex_unlock(lock: &process_info->notifier_lock);
1084	mutex_unlock(lock: &process_info->lock);
1085	return 0;
1086	}
1087
1088	ret = amdgpu_ttm_tt_get_user_pages(bo, pages: bo->tbo.ttm->pages, range: &range);
1089	if (ret) {
1090	pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
1091	goto unregister_out;
1092	}
1093
1094	ret = amdgpu_bo_reserve(bo, no_intr: true);
1095	if (ret) {
1096	pr_err("%s: Failed to reserve BO\n", __func__);
1097	goto release_out;
1098	}
1099	amdgpu_bo_placement_from_domain(abo: bo, domain: mem->domain);
1100	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
1101	if (ret)
1102	pr_err("%s: failed to validate BO\n", __func__);
1103	amdgpu_bo_unreserve(bo);
1104
1105	release_out:
1106	amdgpu_ttm_tt_get_user_pages_done(ttm: bo->tbo.ttm, range);
1107	unregister_out:
1108	if (ret)
1109	amdgpu_hmm_unregister(bo);
1110	out:
1111	mutex_unlock(lock: &process_info->lock);
1112	return ret;
1113	}
1114
1115	/* Reserving a BO and its page table BOs must happen atomically to
1116	* avoid deadlocks. Some operations update multiple VMs at once. Track
1117	* all the reservation info in a context structure. Optionally a sync
1118	* object can track VM updates.
1119	*/
1120	struct bo_vm_reservation_context {
1121	/* DRM execution context for the reservation */
1122	struct drm_exec exec;
1123	/* Number of VMs reserved */
1124	unsigned int n_vms;
1125	/* Pointer to sync object */
1126	struct amdgpu_sync *sync;
1127	};
1128
1129	enum bo_vm_match {
1130	BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */
1131	BO_VM_MAPPED, /* Match VMs where a BO is mapped */
1132	BO_VM_ALL, /* Match all VMs a BO was added to */
1133	};
1134
1135	/**
1136	* reserve_bo_and_vm - reserve a BO and a VM unconditionally.
1137	* @mem: KFD BO structure.
1138	* @vm: the VM to reserve.
1139	* @ctx: the struct that will be used in unreserve_bo_and_vms().
1140	*/
1141	static int reserve_bo_and_vm(struct kgd_mem *mem,
1142	struct amdgpu_vm *vm,
1143	struct bo_vm_reservation_context *ctx)
1144	{
1145	struct amdgpu_bo *bo = mem->bo;
1146	int ret;
1147
1148	WARN_ON(!vm);
1149
1150	ctx->n_vms = 1;
1151	ctx->sync = &mem->sync;
1152	drm_exec_init(exec: &ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, nr: 0);
1153	drm_exec_until_all_locked(&ctx->exec) {
1154	ret = amdgpu_vm_lock_pd(vm, exec: &ctx->exec, num_fences: 2);
1155	drm_exec_retry_on_contention(&ctx->exec);
1156	if (unlikely(ret))
1157	goto error;
1158
1159	ret = drm_exec_prepare_obj(exec: &ctx->exec, obj: &bo->tbo.base, num_fences: 1);
1160	drm_exec_retry_on_contention(&ctx->exec);
1161	if (unlikely(ret))
1162	goto error;
1163	}
1164	return 0;
1165
1166	error:
1167	pr_err("Failed to reserve buffers in ttm.\n");
1168	drm_exec_fini(exec: &ctx->exec);
1169	return ret;
1170	}
1171
1172	/**
1173	* reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally
1174	* @mem: KFD BO structure.
1175	* @vm: the VM to reserve. If NULL, then all VMs associated with the BO
1176	* is used. Otherwise, a single VM associated with the BO.
1177	* @map_type: the mapping status that will be used to filter the VMs.
1178	* @ctx: the struct that will be used in unreserve_bo_and_vms().
1179	*
1180	* Returns 0 for success, negative for failure.
1181	*/
1182	static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
1183	struct amdgpu_vm *vm, enum bo_vm_match map_type,
1184	struct bo_vm_reservation_context *ctx)
1185	{
1186	struct kfd_mem_attachment *entry;
1187	struct amdgpu_bo *bo = mem->bo;
1188	int ret;
1189
1190	ctx->sync = &mem->sync;
1191	drm_exec_init(exec: &ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, nr: 0);
1192	drm_exec_until_all_locked(&ctx->exec) {
1193	ctx->n_vms = 0;
1194	list_for_each_entry(entry, &mem->attachments, list) {
1195	if ((vm && vm != entry->bo_va->base.vm) ||
1196	(entry->is_mapped != map_type
1197	&& map_type != BO_VM_ALL))
1198	continue;
1199
1200	ret = amdgpu_vm_lock_pd(vm: entry->bo_va->base.vm,
1201	exec: &ctx->exec, num_fences: 2);
1202	drm_exec_retry_on_contention(&ctx->exec);
1203	if (unlikely(ret))
1204	goto error;
1205	++ctx->n_vms;
1206	}
1207
1208	ret = drm_exec_prepare_obj(exec: &ctx->exec, obj: &bo->tbo.base, num_fences: 1);
1209	drm_exec_retry_on_contention(&ctx->exec);
1210	if (unlikely(ret))
1211	goto error;
1212	}
1213	return 0;
1214
1215	error:
1216	pr_err("Failed to reserve buffers in ttm.\n");
1217	drm_exec_fini(exec: &ctx->exec);
1218	return ret;
1219	}
1220
1221	/**
1222	* unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context
1223	* @ctx: Reservation context to unreserve
1224	* @wait: Optionally wait for a sync object representing pending VM updates
1225	* @intr: Whether the wait is interruptible
1226	*
1227	* Also frees any resources allocated in
1228	* reserve_bo_and_(cond_)vm(s). Returns the status from
1229	* amdgpu_sync_wait.
1230	*/
1231	static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
1232	bool wait, bool intr)
1233	{
1234	int ret = 0;
1235
1236	if (wait)
1237	ret = amdgpu_sync_wait(sync: ctx->sync, intr);
1238
1239	drm_exec_fini(exec: &ctx->exec);
1240	ctx->sync = NULL;
1241	return ret;
1242	}
1243
1244	static void unmap_bo_from_gpuvm(struct kgd_mem *mem,
1245	struct kfd_mem_attachment *entry,
1246	struct amdgpu_sync *sync)
1247	{
1248	struct amdgpu_bo_va *bo_va = entry->bo_va;
1249	struct amdgpu_device *adev = entry->adev;
1250	struct amdgpu_vm *vm = bo_va->base.vm;
1251
1252	amdgpu_vm_bo_unmap(adev, bo_va, addr: entry->va);
1253
1254	amdgpu_vm_clear_freed(adev, vm, fence: &bo_va->last_pt_update);
1255
1256	amdgpu_sync_fence(sync, f: bo_va->last_pt_update);
1257	}
1258
1259	static int update_gpuvm_pte(struct kgd_mem *mem,
1260	struct kfd_mem_attachment *entry,
1261	struct amdgpu_sync *sync)
1262	{
1263	struct amdgpu_bo_va *bo_va = entry->bo_va;
1264	struct amdgpu_device *adev = entry->adev;
1265	int ret;
1266
1267	ret = kfd_mem_dmamap_attachment(mem, attachment: entry);
1268	if (ret)
1269	return ret;
1270
1271	/* Update the page tables */
1272	ret = amdgpu_vm_bo_update(adev, bo_va, clear: false);
1273	if (ret) {
1274	pr_err("amdgpu_vm_bo_update failed\n");
1275	return ret;
1276	}
1277
1278	return amdgpu_sync_fence(sync, f: bo_va->last_pt_update);
1279	}
1280
1281	static int map_bo_to_gpuvm(struct kgd_mem *mem,
1282	struct kfd_mem_attachment *entry,
1283	struct amdgpu_sync *sync,
1284	bool no_update_pte)
1285	{
1286	int ret;
1287
1288	/* Set virtual address for the allocation */
1289	ret = amdgpu_vm_bo_map(adev: entry->adev, bo_va: entry->bo_va, addr: entry->va, offset: 0,
1290	size: amdgpu_bo_size(bo: entry->bo_va->base.bo),
1291	flags: entry->pte_flags);
1292	if (ret) {
1293	pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
1294	entry->va, ret);
1295	return ret;
1296	}
1297
1298	if (no_update_pte)
1299	return 0;
1300
1301	ret = update_gpuvm_pte(mem, entry, sync);
1302	if (ret) {
1303	pr_err("update_gpuvm_pte() failed\n");
1304	goto update_gpuvm_pte_failed;
1305	}
1306
1307	return 0;
1308
1309	update_gpuvm_pte_failed:
1310	unmap_bo_from_gpuvm(mem, entry, sync);
1311	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
1312	return ret;
1313	}
1314
1315	static int process_validate_vms(struct amdkfd_process_info *process_info,
1316	struct ww_acquire_ctx *ticket)
1317	{
1318	struct amdgpu_vm *peer_vm;
1319	int ret;
1320
1321	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1322	vm_list_node) {
1323	ret = vm_validate_pt_pd_bos(vm: peer_vm, ticket);
1324	if (ret)
1325	return ret;
1326	}
1327
1328	return 0;
1329	}
1330
1331	static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
1332	struct amdgpu_sync *sync)
1333	{
1334	struct amdgpu_vm *peer_vm;
1335	int ret;
1336
1337	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1338	vm_list_node) {
1339	struct amdgpu_bo *pd = peer_vm->root.bo;
1340
1341	ret = amdgpu_sync_resv(NULL, sync, resv: pd->tbo.base.resv,
1342	mode: AMDGPU_SYNC_NE_OWNER,
1343	AMDGPU_FENCE_OWNER_KFD);
1344	if (ret)
1345	return ret;
1346	}
1347
1348	return 0;
1349	}
1350
1351	static int process_update_pds(struct amdkfd_process_info *process_info,
1352	struct amdgpu_sync *sync)
1353	{
1354	struct amdgpu_vm *peer_vm;
1355	int ret;
1356
1357	list_for_each_entry(peer_vm, &process_info->vm_list_head,
1358	vm_list_node) {
1359	ret = vm_update_pds(vm: peer_vm, sync);
1360	if (ret)
1361	return ret;
1362	}
1363
1364	return 0;
1365	}
1366
1367	static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
1368	struct dma_fence **ef)
1369	{
1370	struct amdkfd_process_info *info = NULL;
1371	int ret;
1372
1373	if (!*process_info) {
1374	info = kzalloc(size: sizeof(*info), GFP_KERNEL);
1375	if (!info)
1376	return -ENOMEM;
1377
1378	mutex_init(&info->lock);
1379	mutex_init(&info->notifier_lock);
1380	INIT_LIST_HEAD(list: &info->vm_list_head);
1381	INIT_LIST_HEAD(list: &info->kfd_bo_list);
1382	INIT_LIST_HEAD(list: &info->userptr_valid_list);
1383	INIT_LIST_HEAD(list: &info->userptr_inval_list);
1384
1385	info->eviction_fence =
1386	amdgpu_amdkfd_fence_create(context: dma_fence_context_alloc(num: 1),
1387	current->mm,
1388	NULL);
1389	if (!info->eviction_fence) {
1390	pr_err("Failed to create eviction fence\n");
1391	ret = -ENOMEM;
1392	goto create_evict_fence_fail;
1393	}
1394
1395	info->pid = get_task_pid(current->group_leader, type: PIDTYPE_PID);
1396	INIT_DELAYED_WORK(&info->restore_userptr_work,
1397	amdgpu_amdkfd_restore_userptr_worker);
1398
1399	*process_info = info;
1400	}
1401
1402	vm->process_info = *process_info;
1403
1404	/* Validate page directory and attach eviction fence */
1405	ret = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: true);
1406	if (ret)
1407	goto reserve_pd_fail;
1408	ret = vm_validate_pt_pd_bos(vm, NULL);
1409	if (ret) {
1410	pr_err("validate_pt_pd_bos() failed\n");
1411	goto validate_pd_fail;
1412	}
1413	ret = amdgpu_bo_sync_wait(bo: vm->root.bo,
1414	AMDGPU_FENCE_OWNER_KFD, intr: false);
1415	if (ret)
1416	goto wait_pd_fail;
1417	ret = dma_resv_reserve_fences(obj: vm->root.bo->tbo.base.resv, num_fences: 1);
1418	if (ret)
1419	goto reserve_shared_fail;
1420	dma_resv_add_fence(obj: vm->root.bo->tbo.base.resv,
1421	fence: &vm->process_info->eviction_fence->base,
1422	usage: DMA_RESV_USAGE_BOOKKEEP);
1423	amdgpu_bo_unreserve(bo: vm->root.bo);
1424
1425	/* Update process info */
1426	mutex_lock(&vm->process_info->lock);
1427	list_add_tail(new: &vm->vm_list_node,
1428	head: &(vm->process_info->vm_list_head));
1429	vm->process_info->n_vms++;
1430
1431	*ef = dma_fence_get(fence: &vm->process_info->eviction_fence->base);
1432	mutex_unlock(lock: &vm->process_info->lock);
1433
1434	return 0;
1435
1436	reserve_shared_fail:
1437	wait_pd_fail:
1438	validate_pd_fail:
1439	amdgpu_bo_unreserve(bo: vm->root.bo);
1440	reserve_pd_fail:
1441	vm->process_info = NULL;
1442	if (info) {
1443	dma_fence_put(fence: &info->eviction_fence->base);
1444	*process_info = NULL;
1445	put_pid(pid: info->pid);
1446	create_evict_fence_fail:
1447	mutex_destroy(lock: &info->lock);
1448	mutex_destroy(lock: &info->notifier_lock);
1449	kfree(objp: info);
1450	}
1451	return ret;
1452	}
1453
1454	/**
1455	* amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria
1456	* @bo: Handle of buffer object being pinned
1457	* @domain: Domain into which BO should be pinned
1458	*
1459	* - USERPTR BOs are UNPINNABLE and will return error
1460	* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1461	* PIN count incremented. It is valid to PIN a BO multiple times
1462	*
1463	* Return: ZERO if successful in pinning, Non-Zero in case of error.
1464	*/
1465	static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain)
1466	{
1467	int ret = 0;
1468
1469	ret = amdgpu_bo_reserve(bo, no_intr: false);
1470	if (unlikely(ret))
1471	return ret;
1472
1473	ret = amdgpu_bo_pin_restricted(bo, domain, min_offset: 0, max_offset: 0);
1474	if (ret)
1475	pr_err("Error in Pinning BO to domain: %d\n", domain);
1476
1477	amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
1478	amdgpu_bo_unreserve(bo);
1479
1480	return ret;
1481	}
1482
1483	/**
1484	* amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria
1485	* @bo: Handle of buffer object being unpinned
1486	*
1487	* - Is a illegal request for USERPTR BOs and is ignored
1488	* - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their
1489	* PIN count decremented. Calls to UNPIN must balance calls to PIN
1490	*/
1491	static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo)
1492	{
1493	int ret = 0;
1494
1495	ret = amdgpu_bo_reserve(bo, no_intr: false);
1496	if (unlikely(ret))
1497	return;
1498
1499	amdgpu_bo_unpin(bo);
1500	amdgpu_bo_unreserve(bo);
1501	}
1502
1503	int amdgpu_amdkfd_gpuvm_set_vm_pasid(struct amdgpu_device *adev,
1504	struct amdgpu_vm *avm, u32 pasid)
1505
1506	{
1507	int ret;
1508
1509	/* Free the original amdgpu allocated pasid,
1510	* will be replaced with kfd allocated pasid.
1511	*/
1512	if (avm->pasid) {
1513	amdgpu_pasid_free(pasid: avm->pasid);
1514	amdgpu_vm_set_pasid(adev, vm: avm, pasid: 0);
1515	}
1516
1517	ret = amdgpu_vm_set_pasid(adev, vm: avm, pasid);
1518	if (ret)
1519	return ret;
1520
1521	return 0;
1522	}
1523
1524	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
1525	struct amdgpu_vm *avm,
1526	void **process_info,
1527	struct dma_fence **ef)
1528	{
1529	int ret;
1530
1531	/* Already a compute VM? */
1532	if (avm->process_info)
1533	return -EINVAL;
1534
1535	/* Convert VM into a compute VM */
1536	ret = amdgpu_vm_make_compute(adev, vm: avm);
1537	if (ret)
1538	return ret;
1539
1540	/* Initialize KFD part of the VM and process info */
1541	ret = init_kfd_vm(vm: avm, process_info, ef);
1542	if (ret)
1543	return ret;
1544
1545	amdgpu_vm_set_task_info(vm: avm);
1546
1547	return 0;
1548	}
1549
1550	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
1551	struct amdgpu_vm *vm)
1552	{
1553	struct amdkfd_process_info *process_info = vm->process_info;
1554
1555	if (!process_info)
1556	return;
1557
1558	/* Update process info */
1559	mutex_lock(&process_info->lock);
1560	process_info->n_vms--;
1561	list_del(entry: &vm->vm_list_node);
1562	mutex_unlock(lock: &process_info->lock);
1563
1564	vm->process_info = NULL;
1565
1566	/* Release per-process resources when last compute VM is destroyed */
1567	if (!process_info->n_vms) {
1568	WARN_ON(!list_empty(&process_info->kfd_bo_list));
1569	WARN_ON(!list_empty(&process_info->userptr_valid_list));
1570	WARN_ON(!list_empty(&process_info->userptr_inval_list));
1571
1572	dma_fence_put(fence: &process_info->eviction_fence->base);
1573	cancel_delayed_work_sync(dwork: &process_info->restore_userptr_work);
1574	put_pid(pid: process_info->pid);
1575	mutex_destroy(lock: &process_info->lock);
1576	mutex_destroy(lock: &process_info->notifier_lock);
1577	kfree(objp: process_info);
1578	}
1579	}
1580
1581	void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
1582	void *drm_priv)
1583	{
1584	struct amdgpu_vm *avm;
1585
1586	if (WARN_ON(!adev || !drm_priv))
1587	return;
1588
1589	avm = drm_priv_to_vm(drm_priv);
1590
1591	pr_debug("Releasing process vm %p\n", avm);
1592
1593	/* The original pasid of amdgpu vm has already been
1594	* released during making a amdgpu vm to a compute vm
1595	* The current pasid is managed by kfd and will be
1596	* released on kfd process destroy. Set amdgpu pasid
1597	* to 0 to avoid duplicate release.
1598	*/
1599	amdgpu_vm_release_compute(adev, vm: avm);
1600	}
1601
1602	uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1603	{
1604	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1605	struct amdgpu_bo *pd = avm->root.bo;
1606	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: pd->tbo.bdev);
1607
1608	if (adev->asic_type < CHIP_VEGA10)
1609	return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1610	return avm->pd_phys_addr;
1611	}
1612
1613	void amdgpu_amdkfd_block_mmu_notifications(void *p)
1614	{
1615	struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1616
1617	mutex_lock(&pinfo->lock);
1618	WRITE_ONCE(pinfo->block_mmu_notifications, true);
1619	mutex_unlock(lock: &pinfo->lock);
1620	}
1621
1622	int amdgpu_amdkfd_criu_resume(void *p)
1623	{
1624	int ret = 0;
1625	struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p;
1626
1627	mutex_lock(&pinfo->lock);
1628	pr_debug("scheduling work\n");
1629	mutex_lock(&pinfo->notifier_lock);
1630	pinfo->evicted_bos++;
1631	mutex_unlock(lock: &pinfo->notifier_lock);
1632	if (!READ_ONCE(pinfo->block_mmu_notifications)) {
1633	ret = -EINVAL;
1634	goto out_unlock;
1635	}
1636	WRITE_ONCE(pinfo->block_mmu_notifications, false);
1637	queue_delayed_work(wq: system_freezable_wq,
1638	dwork: &pinfo->restore_userptr_work, delay: 0);
1639
1640	out_unlock:
1641	mutex_unlock(lock: &pinfo->lock);
1642	return ret;
1643	}
1644
1645	size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev,
1646	uint8_t xcp_id)
1647	{
1648	uint64_t reserved_for_pt =
1649	ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
1650	ssize_t available;
1651	uint64_t vram_available, system_mem_available, ttm_mem_available;
1652
1653	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
1654	vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id)
1655	- adev->kfd.vram_used_aligned[xcp_id]
1656	- atomic64_read(v: &adev->vram_pin_size)
1657	- reserved_for_pt;
1658
1659	if (adev->gmc.is_app_apu) {
1660	system_mem_available = no_system_mem_limit ?
1661	kfd_mem_limit.max_system_mem_limit :
1662	kfd_mem_limit.max_system_mem_limit -
1663	kfd_mem_limit.system_mem_used;
1664
1665	ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit -
1666	kfd_mem_limit.ttm_mem_used;
1667
1668	available = min3(system_mem_available, ttm_mem_available,
1669	vram_available);
1670	available = ALIGN_DOWN(available, PAGE_SIZE);
1671	} else {
1672	available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN);
1673	}
1674
1675	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
1676
1677	if (available < 0)
1678	available = 0;
1679
1680	return available;
1681	}
1682
1683	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1684	struct amdgpu_device *adev, uint64_t va, uint64_t size,
1685	void *drm_priv, struct kgd_mem **mem,
1686	uint64_t *offset, uint32_t flags, bool criu_resume)
1687	{
1688	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1689	struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm);
1690	enum ttm_bo_type bo_type = ttm_bo_type_device;
1691	struct sg_table *sg = NULL;
1692	uint64_t user_addr = 0;
1693	struct amdgpu_bo *bo;
1694	struct drm_gem_object *gobj = NULL;
1695	u32 domain, alloc_domain;
1696	uint64_t aligned_size;
1697	int8_t xcp_id = -1;
1698	u64 alloc_flags;
1699	int ret;
1700
1701	/*
1702	* Check on which domain to allocate BO
1703	*/
1704	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1705	domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1706
1707	if (adev->gmc.is_app_apu) {
1708	domain = AMDGPU_GEM_DOMAIN_GTT;
1709	alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1710	alloc_flags = 0;
1711	} else {
1712	alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1713	alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ?
1714	AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0;
1715	}
1716	xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ?
1717	0 : fpriv->xcp_id;
1718	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
1719	domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1720	alloc_flags = 0;
1721	} else {
1722	domain = AMDGPU_GEM_DOMAIN_GTT;
1723	alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1724	alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
1725
1726	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1727	if (!offset || !*offset)
1728	return -EINVAL;
1729	user_addr = untagged_addr(*offset);
1730	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1731	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1732	bo_type = ttm_bo_type_sg;
1733	if (size > UINT_MAX)
1734	return -EINVAL;
1735	sg = create_sg_table(addr: *offset, size);
1736	if (!sg)
1737	return -ENOMEM;
1738	} else {
1739	return -EINVAL;
1740	}
1741	}
1742
1743	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)
1744	alloc_flags |= AMDGPU_GEM_CREATE_COHERENT;
1745	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT)
1746	alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT;
1747	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED)
1748	alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED;
1749
1750	*mem = kzalloc(size: sizeof(struct kgd_mem), GFP_KERNEL);
1751	if (!*mem) {
1752	ret = -ENOMEM;
1753	goto err;
1754	}
1755	INIT_LIST_HEAD(list: &(*mem)->attachments);
1756	mutex_init(&(*mem)->lock);
1757	(*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1758
1759	/* Workaround for AQL queue wraparound bug. Map the same
1760	* memory twice. That means we only actually allocate half
1761	* the memory.
1762	*/
1763	if ((*mem)->aql_queue)
1764	size >>= 1;
1765	aligned_size = PAGE_ALIGN(size);
1766
1767	(*mem)->alloc_flags = flags;
1768
1769	amdgpu_sync_create(sync: &(*mem)->sync);
1770
1771	ret = amdgpu_amdkfd_reserve_mem_limit(adev, size: aligned_size, alloc_flag: flags,
1772	xcp_id);
1773	if (ret) {
1774	pr_debug("Insufficient memory\n");
1775	goto err_reserve_limit;
1776	}
1777
1778	pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n",
1779	va, (*mem)->aql_queue ? size << 1 : size,
1780	domain_string(alloc_domain), xcp_id);
1781
1782	ret = amdgpu_gem_object_create(adev, size: aligned_size, alignment: 1, initial_domain: alloc_domain, flags: alloc_flags,
1783	type: bo_type, NULL, obj: &gobj, xcp_id_plus1: xcp_id + 1);
1784	if (ret) {
1785	pr_debug("Failed to create BO on domain %s. ret %d\n",
1786	domain_string(alloc_domain), ret);
1787	goto err_bo_create;
1788	}
1789	ret = drm_vma_node_allow(node: &gobj->vma_node, tag: drm_priv);
1790	if (ret) {
1791	pr_debug("Failed to allow vma node access. ret %d\n", ret);
1792	goto err_node_allow;
1793	}
1794	ret = drm_gem_handle_create(file_priv: adev->kfd.client.file, obj: gobj, handlep: &(*mem)->gem_handle);
1795	if (ret)
1796	goto err_gem_handle_create;
1797	bo = gem_to_amdgpu_bo(gobj);
1798	if (bo_type == ttm_bo_type_sg) {
1799	bo->tbo.sg = sg;
1800	bo->tbo.ttm->sg = sg;
1801	}
1802	bo->kfd_bo = *mem;
1803	(*mem)->bo = bo;
1804	if (user_addr)
1805	bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1806
1807	(*mem)->va = va;
1808	(*mem)->domain = domain;
1809	(*mem)->mapped_to_gpu_memory = 0;
1810	(*mem)->process_info = avm->process_info;
1811
1812	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info: avm->process_info, userptr: user_addr);
1813
1814	if (user_addr) {
1815	pr_debug("creating userptr BO for user_addr = %llx\n", user_addr);
1816	ret = init_user_pages(mem: *mem, user_addr, criu_resume);
1817	if (ret)
1818	goto allocate_init_user_pages_failed;
1819	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1820	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1821	ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT);
1822	if (ret) {
1823	pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n");
1824	goto err_pin_bo;
1825	}
1826	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
1827	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
1828	} else {
1829	mutex_lock(&avm->process_info->lock);
1830	if (avm->process_info->eviction_fence &&
1831	!dma_fence_is_signaled(fence: &avm->process_info->eviction_fence->base))
1832	ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,
1833	fence: &avm->process_info->eviction_fence->base);
1834	mutex_unlock(lock: &avm->process_info->lock);
1835	if (ret)
1836	goto err_validate_bo;
1837	}
1838
1839	if (offset)
1840	*offset = amdgpu_bo_mmap_offset(bo);
1841
1842	return 0;
1843
1844	allocate_init_user_pages_failed:
1845	err_pin_bo:
1846	err_validate_bo:
1847	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info: avm->process_info);
1848	drm_gem_handle_delete(filp: adev->kfd.client.file, handle: (*mem)->gem_handle);
1849	err_gem_handle_create:
1850	drm_vma_node_revoke(node: &gobj->vma_node, tag: drm_priv);
1851	err_node_allow:
1852	/* Don't unreserve system mem limit twice */
1853	goto err_reserve_limit;
1854	err_bo_create:
1855	amdgpu_amdkfd_unreserve_mem_limit(adev, size: aligned_size, alloc_flag: flags, xcp_id);
1856	err_reserve_limit:
1857	mutex_destroy(lock: &(*mem)->lock);
1858	if (gobj)
1859	drm_gem_object_put(obj: gobj);
1860	else
1861	kfree(objp: *mem);
1862	err:
1863	if (sg) {
1864	sg_free_table(sg);
1865	kfree(objp: sg);
1866	}
1867	return ret;
1868	}
1869
1870	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1871	struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
1872	uint64_t *size)
1873	{
1874	struct amdkfd_process_info *process_info = mem->process_info;
1875	unsigned long bo_size = mem->bo->tbo.base.size;
1876	bool use_release_notifier = (mem->bo->kfd_bo == mem);
1877	struct kfd_mem_attachment *entry, *tmp;
1878	struct bo_vm_reservation_context ctx;
1879	unsigned int mapped_to_gpu_memory;
1880	int ret;
1881	bool is_imported = false;
1882
1883	mutex_lock(&mem->lock);
1884
1885	/* Unpin MMIO/DOORBELL BO's that were pinned during allocation */
1886	if (mem->alloc_flags &
1887	(KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
1888	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1889	amdgpu_amdkfd_gpuvm_unpin_bo(bo: mem->bo);
1890	}
1891
1892	mapped_to_gpu_memory = mem->mapped_to_gpu_memory;
1893	is_imported = mem->is_imported;
1894	mutex_unlock(lock: &mem->lock);
1895	/* lock is not needed after this, since mem is unused and will
1896	* be freed anyway
1897	*/
1898
1899	if (mapped_to_gpu_memory > 0) {
1900	pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1901	mem->va, bo_size);
1902	return -EBUSY;
1903	}
1904
1905	/* Make sure restore workers don't access the BO any more */
1906	mutex_lock(&process_info->lock);
1907	list_del(entry: &mem->validate_list);
1908	mutex_unlock(lock: &process_info->lock);
1909
1910	/* Cleanup user pages and MMU notifiers */
1911	if (amdgpu_ttm_tt_get_usermm(ttm: mem->bo->tbo.ttm)) {
1912	amdgpu_hmm_unregister(bo: mem->bo);
1913	mutex_lock(&process_info->notifier_lock);
1914	amdgpu_ttm_tt_discard_user_pages(ttm: mem->bo->tbo.ttm, range: mem->range);
1915	mutex_unlock(lock: &process_info->notifier_lock);
1916	}
1917
1918	ret = reserve_bo_and_cond_vms(mem, NULL, map_type: BO_VM_ALL, ctx: &ctx);
1919	if (unlikely(ret))
1920	return ret;
1921
1922	amdgpu_amdkfd_remove_eviction_fence(bo: mem->bo,
1923	ef: process_info->eviction_fence);
1924	pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1925	mem->va + bo_size * (1 + mem->aql_queue));
1926
1927	/* Remove from VM internal data structures */
1928	list_for_each_entry_safe(entry, tmp, &mem->attachments, list) {
1929	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
1930	kfd_mem_detach(attachment: entry);
1931	}
1932
1933	ret = unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
1934
1935	/* Free the sync object */
1936	amdgpu_sync_free(sync: &mem->sync);
1937
1938	/* If the SG is not NULL, it's one we created for a doorbell or mmio
1939	* remap BO. We need to free it.
1940	*/
1941	if (mem->bo->tbo.sg) {
1942	sg_free_table(mem->bo->tbo.sg);
1943	kfree(objp: mem->bo->tbo.sg);
1944	}
1945
1946	/* Update the size of the BO being freed if it was allocated from
1947	* VRAM and is not imported. For APP APU VRAM allocations are done
1948	* in GTT domain
1949	*/
1950	if (size) {
1951	if (!is_imported &&
1952	(mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM ||
1953	(adev->gmc.is_app_apu &&
1954	mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT)))
1955	*size = bo_size;
1956	else
1957	*size = 0;
1958	}
1959
1960	/* Free the BO*/
1961	drm_vma_node_revoke(node: &mem->bo->tbo.base.vma_node, tag: drm_priv);
1962	drm_gem_handle_delete(filp: adev->kfd.client.file, handle: mem->gem_handle);
1963	if (mem->dmabuf) {
1964	dma_buf_put(dmabuf: mem->dmabuf);
1965	mem->dmabuf = NULL;
1966	}
1967	mutex_destroy(lock: &mem->lock);
1968
1969	/* If this releases the last reference, it will end up calling
1970	* amdgpu_amdkfd_release_notify and kfree the mem struct. That's why
1971	* this needs to be the last call here.
1972	*/
1973	drm_gem_object_put(obj: &mem->bo->tbo.base);
1974
1975	/*
1976	* For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(),
1977	* explicitly free it here.
1978	*/
1979	if (!use_release_notifier)
1980	kfree(objp: mem);
1981
1982	return ret;
1983	}
1984
1985	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1986	struct amdgpu_device *adev, struct kgd_mem *mem,
1987	void *drm_priv)
1988	{
1989	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1990	int ret;
1991	struct amdgpu_bo *bo;
1992	uint32_t domain;
1993	struct kfd_mem_attachment *entry;
1994	struct bo_vm_reservation_context ctx;
1995	unsigned long bo_size;
1996	bool is_invalid_userptr = false;
1997
1998	bo = mem->bo;
1999	if (!bo) {
2000	pr_err("Invalid BO when mapping memory to GPU\n");
2001	return -EINVAL;
2002	}
2003
2004	/* Make sure restore is not running concurrently. Since we
2005	* don't map invalid userptr BOs, we rely on the next restore
2006	* worker to do the mapping
2007	*/
2008	mutex_lock(&mem->process_info->lock);
2009
2010	/* Lock notifier lock. If we find an invalid userptr BO, we can be
2011	* sure that the MMU notifier is no longer running
2012	* concurrently and the queues are actually stopped
2013	*/
2014	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm)) {
2015	mutex_lock(&mem->process_info->notifier_lock);
2016	is_invalid_userptr = !!mem->invalid;
2017	mutex_unlock(lock: &mem->process_info->notifier_lock);
2018	}
2019
2020	mutex_lock(&mem->lock);
2021
2022	domain = mem->domain;
2023	bo_size = bo->tbo.base.size;
2024
2025	pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
2026	mem->va,
2027	mem->va + bo_size * (1 + mem->aql_queue),
2028	avm, domain_string(domain));
2029
2030	if (!kfd_mem_is_attached(avm, mem)) {
2031	ret = kfd_mem_attach(adev, mem, vm: avm, is_aql: mem->aql_queue);
2032	if (ret)
2033	goto out;
2034	}
2035
2036	ret = reserve_bo_and_vm(mem, vm: avm, ctx: &ctx);
2037	if (unlikely(ret))
2038	goto out;
2039
2040	/* Userptr can be marked as "not invalid", but not actually be
2041	* validated yet (still in the system domain). In that case
2042	* the queues are still stopped and we can leave mapping for
2043	* the next restore worker
2044	*/
2045	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm) &&
2046	bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
2047	is_invalid_userptr = true;
2048
2049	ret = vm_validate_pt_pd_bos(vm: avm, NULL);
2050	if (unlikely(ret))
2051	goto out_unreserve;
2052
2053	list_for_each_entry(entry, &mem->attachments, list) {
2054	if (entry->bo_va->base.vm != avm || entry->is_mapped)
2055	continue;
2056
2057	pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
2058	entry->va, entry->va + bo_size, entry);
2059
2060	ret = map_bo_to_gpuvm(mem, entry, sync: ctx.sync,
2061	no_update_pte: is_invalid_userptr);
2062	if (ret) {
2063	pr_err("Failed to map bo to gpuvm\n");
2064	goto out_unreserve;
2065	}
2066
2067	ret = vm_update_pds(vm: avm, sync: ctx.sync);
2068	if (ret) {
2069	pr_err("Failed to update page directories\n");
2070	goto out_unreserve;
2071	}
2072
2073	entry->is_mapped = true;
2074	mem->mapped_to_gpu_memory++;
2075	pr_debug("\t INC mapping count %d\n",
2076	mem->mapped_to_gpu_memory);
2077	}
2078
2079	ret = unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2080
2081	goto out;
2082
2083	out_unreserve:
2084	unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2085	out:
2086	mutex_unlock(lock: &mem->process_info->lock);
2087	mutex_unlock(lock: &mem->lock);
2088	return ret;
2089	}
2090
2091	int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv)
2092	{
2093	struct kfd_mem_attachment *entry;
2094	struct amdgpu_vm *vm;
2095	int ret;
2096
2097	vm = drm_priv_to_vm(drm_priv);
2098
2099	mutex_lock(&mem->lock);
2100
2101	ret = amdgpu_bo_reserve(bo: mem->bo, no_intr: true);
2102	if (ret)
2103	goto out;
2104
2105	list_for_each_entry(entry, &mem->attachments, list) {
2106	if (entry->bo_va->base.vm != vm)
2107	continue;
2108	if (entry->bo_va->base.bo->tbo.ttm &&
2109	!entry->bo_va->base.bo->tbo.ttm->sg)
2110	continue;
2111
2112	kfd_mem_dmaunmap_attachment(mem, attachment: entry);
2113	}
2114
2115	amdgpu_bo_unreserve(bo: mem->bo);
2116	out:
2117	mutex_unlock(lock: &mem->lock);
2118
2119	return ret;
2120	}
2121
2122	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
2123	struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
2124	{
2125	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2126	unsigned long bo_size = mem->bo->tbo.base.size;
2127	struct kfd_mem_attachment *entry;
2128	struct bo_vm_reservation_context ctx;
2129	int ret;
2130
2131	mutex_lock(&mem->lock);
2132
2133	ret = reserve_bo_and_cond_vms(mem, vm: avm, map_type: BO_VM_MAPPED, ctx: &ctx);
2134	if (unlikely(ret))
2135	goto out;
2136	/* If no VMs were reserved, it means the BO wasn't actually mapped */
2137	if (ctx.n_vms == 0) {
2138	ret = -EINVAL;
2139	goto unreserve_out;
2140	}
2141
2142	ret = vm_validate_pt_pd_bos(vm: avm, NULL);
2143	if (unlikely(ret))
2144	goto unreserve_out;
2145
2146	pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
2147	mem->va,
2148	mem->va + bo_size * (1 + mem->aql_queue),
2149	avm);
2150
2151	list_for_each_entry(entry, &mem->attachments, list) {
2152	if (entry->bo_va->base.vm != avm || !entry->is_mapped)
2153	continue;
2154
2155	pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
2156	entry->va, entry->va + bo_size, entry);
2157
2158	unmap_bo_from_gpuvm(mem, entry, sync: ctx.sync);
2159	entry->is_mapped = false;
2160
2161	mem->mapped_to_gpu_memory--;
2162	pr_debug("\t DEC mapping count %d\n",
2163	mem->mapped_to_gpu_memory);
2164	}
2165
2166	unreserve_out:
2167	unreserve_bo_and_vms(ctx: &ctx, wait: false, intr: false);
2168	out:
2169	mutex_unlock(lock: &mem->lock);
2170	return ret;
2171	}
2172
2173	int amdgpu_amdkfd_gpuvm_sync_memory(
2174	struct amdgpu_device *adev, struct kgd_mem *mem, bool intr)
2175	{
2176	struct amdgpu_sync sync;
2177	int ret;
2178
2179	amdgpu_sync_create(sync: &sync);
2180
2181	mutex_lock(&mem->lock);
2182	amdgpu_sync_clone(source: &mem->sync, clone: &sync);
2183	mutex_unlock(lock: &mem->lock);
2184
2185	ret = amdgpu_sync_wait(sync: &sync, intr);
2186	amdgpu_sync_free(sync: &sync);
2187	return ret;
2188	}
2189
2190	/**
2191	* amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count
2192	* @bo: Buffer object to be mapped
2193	*
2194	* Before return, bo reference count is incremented. To release the reference and unpin/
2195	* unmap the BO, call amdgpu_amdkfd_free_gtt_mem.
2196	*/
2197	int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo)
2198	{
2199	int ret;
2200
2201	ret = amdgpu_bo_reserve(bo, no_intr: true);
2202	if (ret) {
2203	pr_err("Failed to reserve bo. ret %d\n", ret);
2204	goto err_reserve_bo_failed;
2205	}
2206
2207	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2208	if (ret) {
2209	pr_err("Failed to pin bo. ret %d\n", ret);
2210	goto err_pin_bo_failed;
2211	}
2212
2213	ret = amdgpu_ttm_alloc_gart(bo: &bo->tbo);
2214	if (ret) {
2215	pr_err("Failed to bind bo to GART. ret %d\n", ret);
2216	goto err_map_bo_gart_failed;
2217	}
2218
2219	amdgpu_amdkfd_remove_eviction_fence(
2220	bo, ef: bo->vm_bo->vm->process_info->eviction_fence);
2221
2222	amdgpu_bo_unreserve(bo);
2223
2224	bo = amdgpu_bo_ref(bo);
2225
2226	return 0;
2227
2228	err_map_bo_gart_failed:
2229	amdgpu_bo_unpin(bo);
2230	err_pin_bo_failed:
2231	amdgpu_bo_unreserve(bo);
2232	err_reserve_bo_failed:
2233
2234	return ret;
2235	}
2236
2237	/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
2238	*
2239	* @mem: Buffer object to be mapped for CPU access
2240	* @kptr[out]: pointer in kernel CPU address space
2241	* @size[out]: size of the buffer
2242	*
2243	* Pins the BO and maps it for kernel CPU access. The eviction fence is removed
2244	* from the BO, since pinned BOs cannot be evicted. The bo must remain on the
2245	* validate_list, so the GPU mapping can be restored after a page table was
2246	* evicted.
2247	*
2248	* Return: 0 on success, error code on failure
2249	*/
2250	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
2251	void **kptr, uint64_t *size)
2252	{
2253	int ret;
2254	struct amdgpu_bo *bo = mem->bo;
2255
2256	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm)) {
2257	pr_err("userptr can't be mapped to kernel\n");
2258	return -EINVAL;
2259	}
2260
2261	mutex_lock(&mem->process_info->lock);
2262
2263	ret = amdgpu_bo_reserve(bo, no_intr: true);
2264	if (ret) {
2265	pr_err("Failed to reserve bo. ret %d\n", ret);
2266	goto bo_reserve_failed;
2267	}
2268
2269	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
2270	if (ret) {
2271	pr_err("Failed to pin bo. ret %d\n", ret);
2272	goto pin_failed;
2273	}
2274
2275	ret = amdgpu_bo_kmap(bo, ptr: kptr);
2276	if (ret) {
2277	pr_err("Failed to map bo to kernel. ret %d\n", ret);
2278	goto kmap_failed;
2279	}
2280
2281	amdgpu_amdkfd_remove_eviction_fence(
2282	bo, ef: mem->process_info->eviction_fence);
2283
2284	if (size)
2285	*size = amdgpu_bo_size(bo);
2286
2287	amdgpu_bo_unreserve(bo);
2288
2289	mutex_unlock(lock: &mem->process_info->lock);
2290	return 0;
2291
2292	kmap_failed:
2293	amdgpu_bo_unpin(bo);
2294	pin_failed:
2295	amdgpu_bo_unreserve(bo);
2296	bo_reserve_failed:
2297	mutex_unlock(lock: &mem->process_info->lock);
2298
2299	return ret;
2300	}
2301
2302	/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
2303	*
2304	* @mem: Buffer object to be unmapped for CPU access
2305	*
2306	* Removes the kernel CPU mapping and unpins the BO. It does not restore the
2307	* eviction fence, so this function should only be used for cleanup before the
2308	* BO is destroyed.
2309	*/
2310	void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
2311	{
2312	struct amdgpu_bo *bo = mem->bo;
2313
2314	amdgpu_bo_reserve(bo, no_intr: true);
2315	amdgpu_bo_kunmap(bo);
2316	amdgpu_bo_unpin(bo);
2317	amdgpu_bo_unreserve(bo);
2318	}
2319
2320	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
2321	struct kfd_vm_fault_info *mem)
2322	{
2323	if (atomic_read(v: &adev->gmc.vm_fault_info_updated) == 1) {
2324	*mem = *adev->gmc.vm_fault_info;
2325	mb(); /* make sure read happened */
2326	atomic_set(v: &adev->gmc.vm_fault_info_updated, i: 0);
2327	}
2328	return 0;
2329	}
2330
2331	static int import_obj_create(struct amdgpu_device *adev,
2332	struct dma_buf *dma_buf,
2333	struct drm_gem_object *obj,
2334	uint64_t va, void *drm_priv,
2335	struct kgd_mem **mem, uint64_t *size,
2336	uint64_t *mmap_offset)
2337	{
2338	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
2339	struct amdgpu_bo *bo;
2340	int ret;
2341
2342	bo = gem_to_amdgpu_bo(obj);
2343	if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
2344	AMDGPU_GEM_DOMAIN_GTT)))
2345	/* Only VRAM and GTT BOs are supported */
2346	return -EINVAL;
2347
2348	*mem = kzalloc(size: sizeof(struct kgd_mem), GFP_KERNEL);
2349	if (!*mem)
2350	return -ENOMEM;
2351
2352	ret = drm_vma_node_allow(node: &obj->vma_node, tag: drm_priv);
2353	if (ret)
2354	goto err_free_mem;
2355
2356	if (size)
2357	*size = amdgpu_bo_size(bo);
2358
2359	if (mmap_offset)
2360	*mmap_offset = amdgpu_bo_mmap_offset(bo);
2361
2362	INIT_LIST_HEAD(list: &(*mem)->attachments);
2363	mutex_init(&(*mem)->lock);
2364
2365	(*mem)->alloc_flags =
2366	((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
2367	KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT)
2368	| KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE
2369	| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
2370
2371	get_dma_buf(dmabuf: dma_buf);
2372	(*mem)->dmabuf = dma_buf;
2373	(*mem)->bo = bo;
2374	(*mem)->va = va;
2375	(*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ?
2376	AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
2377
2378	(*mem)->mapped_to_gpu_memory = 0;
2379	(*mem)->process_info = avm->process_info;
2380	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info: avm->process_info, userptr: false);
2381	amdgpu_sync_create(sync: &(*mem)->sync);
2382	(*mem)->is_imported = true;
2383
2384	mutex_lock(&avm->process_info->lock);
2385	if (avm->process_info->eviction_fence &&
2386	!dma_fence_is_signaled(fence: &avm->process_info->eviction_fence->base))
2387	ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain: (*mem)->domain,
2388	fence: &avm->process_info->eviction_fence->base);
2389	mutex_unlock(lock: &avm->process_info->lock);
2390	if (ret)
2391	goto err_remove_mem;
2392
2393	return 0;
2394
2395	err_remove_mem:
2396	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info: avm->process_info);
2397	drm_vma_node_revoke(node: &obj->vma_node, tag: drm_priv);
2398	err_free_mem:
2399	kfree(objp: *mem);
2400	return ret;
2401	}
2402
2403	int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd,
2404	uint64_t va, void *drm_priv,
2405	struct kgd_mem **mem, uint64_t *size,
2406	uint64_t *mmap_offset)
2407	{
2408	struct drm_gem_object *obj;
2409	uint32_t handle;
2410	int ret;
2411
2412	ret = drm_gem_prime_fd_to_handle(dev: &adev->ddev, file_priv: adev->kfd.client.file, prime_fd: fd,
2413	handle: &handle);
2414	if (ret)
2415	return ret;
2416	obj = drm_gem_object_lookup(filp: adev->kfd.client.file, handle);
2417	if (!obj) {
2418	ret = -EINVAL;
2419	goto err_release_handle;
2420	}
2421
2422	ret = import_obj_create(adev, dma_buf: obj->dma_buf, obj, va, drm_priv, mem, size,
2423	mmap_offset);
2424	if (ret)
2425	goto err_put_obj;
2426
2427	(*mem)->gem_handle = handle;
2428
2429	return 0;
2430
2431	err_put_obj:
2432	drm_gem_object_put(obj);
2433	err_release_handle:
2434	drm_gem_handle_delete(filp: adev->kfd.client.file, handle);
2435	return ret;
2436	}
2437
2438	int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem,
2439	struct dma_buf **dma_buf)
2440	{
2441	int ret;
2442
2443	mutex_lock(&mem->lock);
2444	ret = kfd_mem_export_dmabuf(mem);
2445	if (ret)
2446	goto out;
2447
2448	get_dma_buf(dmabuf: mem->dmabuf);
2449	*dma_buf = mem->dmabuf;
2450	out:
2451	mutex_unlock(lock: &mem->lock);
2452	return ret;
2453	}
2454
2455	/* Evict a userptr BO by stopping the queues if necessary
2456	*
2457	* Runs in MMU notifier, may be in RECLAIM_FS context. This means it
2458	* cannot do any memory allocations, and cannot take any locks that
2459	* are held elsewhere while allocating memory.
2460	*
2461	* It doesn't do anything to the BO itself. The real work happens in
2462	* restore, where we get updated page addresses. This function only
2463	* ensures that GPU access to the BO is stopped.
2464	*/
2465	int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,
2466	unsigned long cur_seq, struct kgd_mem *mem)
2467	{
2468	struct amdkfd_process_info *process_info = mem->process_info;
2469	int r = 0;
2470
2471	/* Do not process MMU notifications during CRIU restore until
2472	* KFD_CRIU_OP_RESUME IOCTL is received
2473	*/
2474	if (READ_ONCE(process_info->block_mmu_notifications))
2475	return 0;
2476
2477	mutex_lock(&process_info->notifier_lock);
2478	mmu_interval_set_seq(interval_sub: mni, cur_seq);
2479
2480	mem->invalid++;
2481	if (++process_info->evicted_bos == 1) {
2482	/* First eviction, stop the queues */
2483	r = kgd2kfd_quiesce_mm(mm: mni->mm,
2484	trigger: KFD_QUEUE_EVICTION_TRIGGER_USERPTR);
2485	if (r)
2486	pr_err("Failed to quiesce KFD\n");
2487	queue_delayed_work(wq: system_freezable_wq,
2488	dwork: &process_info->restore_userptr_work,
2489	delay: msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2490	}
2491	mutex_unlock(lock: &process_info->notifier_lock);
2492
2493	return r;
2494	}
2495
2496	/* Update invalid userptr BOs
2497	*
2498	* Moves invalidated (evicted) userptr BOs from userptr_valid_list to
2499	* userptr_inval_list and updates user pages for all BOs that have
2500	* been invalidated since their last update.
2501	*/
2502	static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
2503	struct mm_struct *mm)
2504	{
2505	struct kgd_mem *mem, *tmp_mem;
2506	struct amdgpu_bo *bo;
2507	struct ttm_operation_ctx ctx = { false, false };
2508	uint32_t invalid;
2509	int ret = 0;
2510
2511	mutex_lock(&process_info->notifier_lock);
2512
2513	/* Move all invalidated BOs to the userptr_inval_list */
2514	list_for_each_entry_safe(mem, tmp_mem,
2515	&process_info->userptr_valid_list,
2516	validate_list)
2517	if (mem->invalid)
2518	list_move_tail(list: &mem->validate_list,
2519	head: &process_info->userptr_inval_list);
2520
2521	/* Go through userptr_inval_list and update any invalid user_pages */
2522	list_for_each_entry(mem, &process_info->userptr_inval_list,
2523	validate_list) {
2524	invalid = mem->invalid;
2525	if (!invalid)
2526	/* BO hasn't been invalidated since the last
2527	* revalidation attempt. Keep its page list.
2528	*/
2529	continue;
2530
2531	bo = mem->bo;
2532
2533	amdgpu_ttm_tt_discard_user_pages(ttm: bo->tbo.ttm, range: mem->range);
2534	mem->range = NULL;
2535
2536	/* BO reservations and getting user pages (hmm_range_fault)
2537	* must happen outside the notifier lock
2538	*/
2539	mutex_unlock(lock: &process_info->notifier_lock);
2540
2541	/* Move the BO to system (CPU) domain if necessary to unmap
2542	* and free the SG table
2543	*/
2544	if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) {
2545	if (amdgpu_bo_reserve(bo, no_intr: true))
2546	return -EAGAIN;
2547	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
2548	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
2549	amdgpu_bo_unreserve(bo);
2550	if (ret) {
2551	pr_err("%s: Failed to invalidate userptr BO\n",
2552	__func__);
2553	return -EAGAIN;
2554	}
2555	}
2556
2557	/* Get updated user pages */
2558	ret = amdgpu_ttm_tt_get_user_pages(bo, pages: bo->tbo.ttm->pages,
2559	range: &mem->range);
2560	if (ret) {
2561	pr_debug("Failed %d to get user pages\n", ret);
2562
2563	/* Return -EFAULT bad address error as success. It will
2564	* fail later with a VM fault if the GPU tries to access
2565	* it. Better than hanging indefinitely with stalled
2566	* user mode queues.
2567	*
2568	* Return other error -EBUSY or -ENOMEM to retry restore
2569	*/
2570	if (ret != -EFAULT)
2571	return ret;
2572
2573	ret = 0;
2574	}
2575
2576	mutex_lock(&process_info->notifier_lock);
2577
2578	/* Mark the BO as valid unless it was invalidated
2579	* again concurrently.
2580	*/
2581	if (mem->invalid != invalid) {
2582	ret = -EAGAIN;
2583	goto unlock_out;
2584	}
2585	/* set mem valid if mem has hmm range associated */
2586	if (mem->range)
2587	mem->invalid = 0;
2588	}
2589
2590	unlock_out:
2591	mutex_unlock(lock: &process_info->notifier_lock);
2592
2593	return ret;
2594	}
2595
2596	/* Validate invalid userptr BOs
2597	*
2598	* Validates BOs on the userptr_inval_list. Also updates GPUVM page tables
2599	* with new page addresses and waits for the page table updates to complete.
2600	*/
2601	static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
2602	{
2603	struct ttm_operation_ctx ctx = { false, false };
2604	struct amdgpu_sync sync;
2605	struct drm_exec exec;
2606
2607	struct amdgpu_vm *peer_vm;
2608	struct kgd_mem *mem, *tmp_mem;
2609	struct amdgpu_bo *bo;
2610	int ret;
2611
2612	amdgpu_sync_create(sync: &sync);
2613
2614	drm_exec_init(exec: &exec, flags: 0, nr: 0);
2615	/* Reserve all BOs and page tables for validation */
2616	drm_exec_until_all_locked(&exec) {
2617	/* Reserve all the page directories */
2618	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2619	vm_list_node) {
2620	ret = amdgpu_vm_lock_pd(vm: peer_vm, exec: &exec, num_fences: 2);
2621	drm_exec_retry_on_contention(&exec);
2622	if (unlikely(ret))
2623	goto unreserve_out;
2624	}
2625
2626	/* Reserve the userptr_inval_list entries to resv_list */
2627	list_for_each_entry(mem, &process_info->userptr_inval_list,
2628	validate_list) {
2629	struct drm_gem_object *gobj;
2630
2631	gobj = &mem->bo->tbo.base;
2632	ret = drm_exec_prepare_obj(exec: &exec, obj: gobj, num_fences: 1);
2633	drm_exec_retry_on_contention(&exec);
2634	if (unlikely(ret))
2635	goto unreserve_out;
2636	}
2637	}
2638
2639	ret = process_validate_vms(process_info, NULL);
2640	if (ret)
2641	goto unreserve_out;
2642
2643	/* Validate BOs and update GPUVM page tables */
2644	list_for_each_entry_safe(mem, tmp_mem,
2645	&process_info->userptr_inval_list,
2646	validate_list) {
2647	struct kfd_mem_attachment *attachment;
2648
2649	bo = mem->bo;
2650
2651	/* Validate the BO if we got user pages */
2652	if (bo->tbo.ttm->pages[0]) {
2653	amdgpu_bo_placement_from_domain(abo: bo, domain: mem->domain);
2654	ret = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
2655	if (ret) {
2656	pr_err("%s: failed to validate BO\n", __func__);
2657	goto unreserve_out;
2658	}
2659	}
2660
2661	/* Update mapping. If the BO was not validated
2662	* (because we couldn't get user pages), this will
2663	* clear the page table entries, which will result in
2664	* VM faults if the GPU tries to access the invalid
2665	* memory.
2666	*/
2667	list_for_each_entry(attachment, &mem->attachments, list) {
2668	if (!attachment->is_mapped)
2669	continue;
2670
2671	kfd_mem_dmaunmap_attachment(mem, attachment);
2672	ret = update_gpuvm_pte(mem, entry: attachment, sync: &sync);
2673	if (ret) {
2674	pr_err("%s: update PTE failed\n", __func__);
2675	/* make sure this gets validated again */
2676	mutex_lock(&process_info->notifier_lock);
2677	mem->invalid++;
2678	mutex_unlock(lock: &process_info->notifier_lock);
2679	goto unreserve_out;
2680	}
2681	}
2682	}
2683
2684	/* Update page directories */
2685	ret = process_update_pds(process_info, sync: &sync);
2686
2687	unreserve_out:
2688	drm_exec_fini(exec: &exec);
2689	amdgpu_sync_wait(sync: &sync, intr: false);
2690	amdgpu_sync_free(sync: &sync);
2691
2692	return ret;
2693	}
2694
2695	/* Confirm that all user pages are valid while holding the notifier lock
2696	*
2697	* Moves valid BOs from the userptr_inval_list back to userptr_val_list.
2698	*/
2699	static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info)
2700	{
2701	struct kgd_mem *mem, *tmp_mem;
2702	int ret = 0;
2703
2704	list_for_each_entry_safe(mem, tmp_mem,
2705	&process_info->userptr_inval_list,
2706	validate_list) {
2707	bool valid;
2708
2709	/* keep mem without hmm range at userptr_inval_list */
2710	if (!mem->range)
2711	continue;
2712
2713	/* Only check mem with hmm range associated */
2714	valid = amdgpu_ttm_tt_get_user_pages_done(
2715	ttm: mem->bo->tbo.ttm, range: mem->range);
2716
2717	mem->range = NULL;
2718	if (!valid) {
2719	WARN(!mem->invalid, "Invalid BO not marked invalid");
2720	ret = -EAGAIN;
2721	continue;
2722	}
2723
2724	if (mem->invalid) {
2725	WARN(1, "Valid BO is marked invalid");
2726	ret = -EAGAIN;
2727	continue;
2728	}
2729
2730	list_move_tail(list: &mem->validate_list,
2731	head: &process_info->userptr_valid_list);
2732	}
2733
2734	return ret;
2735	}
2736
2737	/* Worker callback to restore evicted userptr BOs
2738	*
2739	* Tries to update and validate all userptr BOs. If successful and no
2740	* concurrent evictions happened, the queues are restarted. Otherwise,
2741	* reschedule for another attempt later.
2742	*/
2743	static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
2744	{
2745	struct delayed_work *dwork = to_delayed_work(work);
2746	struct amdkfd_process_info *process_info =
2747	container_of(dwork, struct amdkfd_process_info,
2748	restore_userptr_work);
2749	struct task_struct *usertask;
2750	struct mm_struct *mm;
2751	uint32_t evicted_bos;
2752
2753	mutex_lock(&process_info->notifier_lock);
2754	evicted_bos = process_info->evicted_bos;
2755	mutex_unlock(lock: &process_info->notifier_lock);
2756	if (!evicted_bos)
2757	return;
2758
2759	/* Reference task and mm in case of concurrent process termination */
2760	usertask = get_pid_task(pid: process_info->pid, PIDTYPE_PID);
2761	if (!usertask)
2762	return;
2763	mm = get_task_mm(task: usertask);
2764	if (!mm) {
2765	put_task_struct(t: usertask);
2766	return;
2767	}
2768
2769	mutex_lock(&process_info->lock);
2770
2771	if (update_invalid_user_pages(process_info, mm))
2772	goto unlock_out;
2773	/* userptr_inval_list can be empty if all evicted userptr BOs
2774	* have been freed. In that case there is nothing to validate
2775	* and we can just restart the queues.
2776	*/
2777	if (!list_empty(head: &process_info->userptr_inval_list)) {
2778	if (validate_invalid_user_pages(process_info))
2779	goto unlock_out;
2780	}
2781	/* Final check for concurrent evicton and atomic update. If
2782	* another eviction happens after successful update, it will
2783	* be a first eviction that calls quiesce_mm. The eviction
2784	* reference counting inside KFD will handle this case.
2785	*/
2786	mutex_lock(&process_info->notifier_lock);
2787	if (process_info->evicted_bos != evicted_bos)
2788	goto unlock_notifier_out;
2789
2790	if (confirm_valid_user_pages_locked(process_info)) {
2791	WARN(1, "User pages unexpectedly invalid");
2792	goto unlock_notifier_out;
2793	}
2794
2795	process_info->evicted_bos = evicted_bos = 0;
2796
2797	if (kgd2kfd_resume_mm(mm)) {
2798	pr_err("%s: Failed to resume KFD\n", __func__);
2799	/* No recovery from this failure. Probably the CP is
2800	* hanging. No point trying again.
2801	*/
2802	}
2803
2804	unlock_notifier_out:
2805	mutex_unlock(lock: &process_info->notifier_lock);
2806	unlock_out:
2807	mutex_unlock(lock: &process_info->lock);
2808
2809	/* If validation failed, reschedule another attempt */
2810	if (evicted_bos) {
2811	queue_delayed_work(wq: system_freezable_wq,
2812	dwork: &process_info->restore_userptr_work,
2813	delay: msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
2814
2815	kfd_smi_event_queue_restore_rescheduled(mm);
2816	}
2817	mmput(mm);
2818	put_task_struct(t: usertask);
2819	}
2820
2821	static void replace_eviction_fence(struct dma_fence __rcu **ef,
2822	struct dma_fence *new_ef)
2823	{
2824	struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true
2825	/* protected by process_info->lock */);
2826
2827	/* If we're replacing an unsignaled eviction fence, that fence will
2828	* never be signaled, and if anyone is still waiting on that fence,
2829	* they will hang forever. This should never happen. We should only
2830	* replace the fence in restore_work that only gets scheduled after
2831	* eviction work signaled the fence.
2832	*/
2833	WARN_ONCE(!dma_fence_is_signaled(old_ef),
2834	"Replacing unsignaled eviction fence");
2835	dma_fence_put(fence: old_ef);
2836	}
2837
2838	/** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given
2839	* KFD process identified by process_info
2840	*
2841	* @process_info: amdkfd_process_info of the KFD process
2842	*
2843	* After memory eviction, restore thread calls this function. The function
2844	* should be called when the Process is still valid. BO restore involves -
2845	*
2846	* 1. Release old eviction fence and create new one
2847	* 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list.
2848	* 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of
2849	* BOs that need to be reserved.
2850	* 4. Reserve all the BOs
2851	* 5. Validate of PD and PT BOs.
2852	* 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence
2853	* 7. Add fence to all PD and PT BOs.
2854	* 8. Unreserve all BOs
2855	*/
2856	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef)
2857	{
2858	struct amdkfd_process_info *process_info = info;
2859	struct amdgpu_vm *peer_vm;
2860	struct kgd_mem *mem;
2861	struct list_head duplicate_save;
2862	struct amdgpu_sync sync_obj;
2863	unsigned long failed_size = 0;
2864	unsigned long total_size = 0;
2865	struct drm_exec exec;
2866	int ret;
2867
2868	INIT_LIST_HEAD(list: &duplicate_save);
2869
2870	mutex_lock(&process_info->lock);
2871
2872	drm_exec_init(exec: &exec, DRM_EXEC_IGNORE_DUPLICATES, nr: 0);
2873	drm_exec_until_all_locked(&exec) {
2874	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2875	vm_list_node) {
2876	ret = amdgpu_vm_lock_pd(vm: peer_vm, exec: &exec, num_fences: 2);
2877	drm_exec_retry_on_contention(&exec);
2878	if (unlikely(ret)) {
2879	pr_err("Locking VM PD failed, ret: %d\n", ret);
2880	goto ttm_reserve_fail;
2881	}
2882	}
2883
2884	/* Reserve all BOs and page tables/directory. Add all BOs from
2885	* kfd_bo_list to ctx.list
2886	*/
2887	list_for_each_entry(mem, &process_info->kfd_bo_list,
2888	validate_list) {
2889	struct drm_gem_object *gobj;
2890
2891	gobj = &mem->bo->tbo.base;
2892	ret = drm_exec_prepare_obj(exec: &exec, obj: gobj, num_fences: 1);
2893	drm_exec_retry_on_contention(&exec);
2894	if (unlikely(ret)) {
2895	pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret);
2896	goto ttm_reserve_fail;
2897	}
2898	}
2899	}
2900
2901	amdgpu_sync_create(sync: &sync_obj);
2902
2903	/* Validate BOs and map them to GPUVM (update VM page tables). */
2904	list_for_each_entry(mem, &process_info->kfd_bo_list,
2905	validate_list) {
2906
2907	struct amdgpu_bo *bo = mem->bo;
2908	uint32_t domain = mem->domain;
2909	struct kfd_mem_attachment *attachment;
2910	struct dma_resv_iter cursor;
2911	struct dma_fence *fence;
2912
2913	total_size += amdgpu_bo_size(bo);
2914
2915	ret = amdgpu_amdkfd_bo_validate(bo, domain, wait: false);
2916	if (ret) {
2917	pr_debug("Memory eviction: Validate BOs failed\n");
2918	failed_size += amdgpu_bo_size(bo);
2919	ret = amdgpu_amdkfd_bo_validate(bo,
2920	AMDGPU_GEM_DOMAIN_GTT, wait: false);
2921	if (ret) {
2922	pr_debug("Memory eviction: Try again\n");
2923	goto validate_map_fail;
2924	}
2925	}
2926	dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
2927	DMA_RESV_USAGE_KERNEL, fence) {
2928	ret = amdgpu_sync_fence(sync: &sync_obj, f: fence);
2929	if (ret) {
2930	pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2931	goto validate_map_fail;
2932	}
2933	}
2934	list_for_each_entry(attachment, &mem->attachments, list) {
2935	if (!attachment->is_mapped)
2936	continue;
2937
2938	if (attachment->bo_va->base.bo->tbo.pin_count)
2939	continue;
2940
2941	kfd_mem_dmaunmap_attachment(mem, attachment);
2942	ret = update_gpuvm_pte(mem, entry: attachment, sync: &sync_obj);
2943	if (ret) {
2944	pr_debug("Memory eviction: update PTE failed. Try again\n");
2945	goto validate_map_fail;
2946	}
2947	}
2948	}
2949
2950	if (failed_size)
2951	pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
2952
2953	/* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO
2954	* validations above would invalidate DMABuf imports again.
2955	*/
2956	ret = process_validate_vms(process_info, ticket: &exec.ticket);
2957	if (ret) {
2958	pr_debug("Validating VMs failed, ret: %d\n", ret);
2959	goto validate_map_fail;
2960	}
2961
2962	/* Update mappings not managed by KFD */
2963	list_for_each_entry(peer_vm, &process_info->vm_list_head,
2964	vm_list_node) {
2965	struct amdgpu_device *adev = amdgpu_ttm_adev(
2966	bdev: peer_vm->root.bo->tbo.bdev);
2967
2968	ret = amdgpu_vm_handle_moved(adev, vm: peer_vm, ticket: &exec.ticket);
2969	if (ret) {
2970	pr_debug("Memory eviction: handle moved failed. Try again\n");
2971	goto validate_map_fail;
2972	}
2973	}
2974
2975	/* Update page directories */
2976	ret = process_update_pds(process_info, sync: &sync_obj);
2977	if (ret) {
2978	pr_debug("Memory eviction: update PDs failed. Try again\n");
2979	goto validate_map_fail;
2980	}
2981
2982	/* Sync with fences on all the page tables. They implicitly depend on any
2983	* move fences from amdgpu_vm_handle_moved above.
2984	*/
2985	ret = process_sync_pds_resv(process_info, sync: &sync_obj);
2986	if (ret) {
2987	pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2988	goto validate_map_fail;
2989	}
2990
2991	/* Wait for validate and PT updates to finish */
2992	amdgpu_sync_wait(sync: &sync_obj, intr: false);
2993
2994	/* The old eviction fence may be unsignaled if restore happens
2995	* after a GPU reset or suspend/resume. Keep the old fence in that
2996	* case. Otherwise release the old eviction fence and create new
2997	* one, because fence only goes from unsignaled to signaled once
2998	* and cannot be reused. Use context and mm from the old fence.
2999	*
3000	* If an old eviction fence signals after this check, that's OK.
3001	* Anyone signaling an eviction fence must stop the queues first
3002	* and schedule another restore worker.
3003	*/
3004	if (dma_fence_is_signaled(fence: &process_info->eviction_fence->base)) {
3005	struct amdgpu_amdkfd_fence *new_fence =
3006	amdgpu_amdkfd_fence_create(
3007	context: process_info->eviction_fence->base.context,
3008	mm: process_info->eviction_fence->mm,
3009	NULL);
3010
3011	if (!new_fence) {
3012	pr_err("Failed to create eviction fence\n");
3013	ret = -ENOMEM;
3014	goto validate_map_fail;
3015	}
3016	dma_fence_put(fence: &process_info->eviction_fence->base);
3017	process_info->eviction_fence = new_fence;
3018	replace_eviction_fence(ef, new_ef: dma_fence_get(fence: &new_fence->base));
3019	} else {
3020	WARN_ONCE(*ef != &process_info->eviction_fence->base,
3021	"KFD eviction fence doesn't match KGD process_info");
3022	}
3023
3024	/* Attach new eviction fence to all BOs except pinned ones */
3025	list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) {
3026	if (mem->bo->tbo.pin_count)
3027	continue;
3028
3029	dma_resv_add_fence(obj: mem->bo->tbo.base.resv,
3030	fence: &process_info->eviction_fence->base,
3031	usage: DMA_RESV_USAGE_BOOKKEEP);
3032	}
3033	/* Attach eviction fence to PD / PT BOs and DMABuf imports */
3034	list_for_each_entry(peer_vm, &process_info->vm_list_head,
3035	vm_list_node) {
3036	struct amdgpu_bo *bo = peer_vm->root.bo;
3037
3038	dma_resv_add_fence(obj: bo->tbo.base.resv,
3039	fence: &process_info->eviction_fence->base,
3040	usage: DMA_RESV_USAGE_BOOKKEEP);
3041	}
3042
3043	validate_map_fail:
3044	amdgpu_sync_free(sync: &sync_obj);
3045	ttm_reserve_fail:
3046	drm_exec_fini(exec: &exec);
3047	mutex_unlock(lock: &process_info->lock);
3048	return ret;
3049	}
3050
3051	int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
3052	{
3053	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3054	struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
3055	int ret;
3056
3057	if (!info || !gws)
3058	return -EINVAL;
3059
3060	*mem = kzalloc(size: sizeof(struct kgd_mem), GFP_KERNEL);
3061	if (!*mem)
3062	return -ENOMEM;
3063
3064	mutex_init(&(*mem)->lock);
3065	INIT_LIST_HEAD(list: &(*mem)->attachments);
3066	(*mem)->bo = amdgpu_bo_ref(bo: gws_bo);
3067	(*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
3068	(*mem)->process_info = process_info;
3069	add_kgd_mem_to_kfd_bo_list(mem: *mem, process_info, userptr: false);
3070	amdgpu_sync_create(sync: &(*mem)->sync);
3071
3072
3073	/* Validate gws bo the first time it is added to process */
3074	mutex_lock(&(*mem)->process_info->lock);
3075	ret = amdgpu_bo_reserve(bo: gws_bo, no_intr: false);
3076	if (unlikely(ret)) {
3077	pr_err("Reserve gws bo failed %d\n", ret);
3078	goto bo_reservation_failure;
3079	}
3080
3081	ret = amdgpu_amdkfd_bo_validate(bo: gws_bo, AMDGPU_GEM_DOMAIN_GWS, wait: true);
3082	if (ret) {
3083	pr_err("GWS BO validate failed %d\n", ret);
3084	goto bo_validation_failure;
3085	}
3086	/* GWS resource is shared b/t amdgpu and amdkfd
3087	* Add process eviction fence to bo so they can
3088	* evict each other.
3089	*/
3090	ret = dma_resv_reserve_fences(obj: gws_bo->tbo.base.resv, num_fences: 1);
3091	if (ret)
3092	goto reserve_shared_fail;
3093	dma_resv_add_fence(obj: gws_bo->tbo.base.resv,
3094	fence: &process_info->eviction_fence->base,
3095	usage: DMA_RESV_USAGE_BOOKKEEP);
3096	amdgpu_bo_unreserve(bo: gws_bo);
3097	mutex_unlock(lock: &(*mem)->process_info->lock);
3098
3099	return ret;
3100
3101	reserve_shared_fail:
3102	bo_validation_failure:
3103	amdgpu_bo_unreserve(bo: gws_bo);
3104	bo_reservation_failure:
3105	mutex_unlock(lock: &(*mem)->process_info->lock);
3106	amdgpu_sync_free(sync: &(*mem)->sync);
3107	remove_kgd_mem_from_kfd_bo_list(mem: *mem, process_info);
3108	amdgpu_bo_unref(bo: &gws_bo);
3109	mutex_destroy(lock: &(*mem)->lock);
3110	kfree(objp: *mem);
3111	*mem = NULL;
3112	return ret;
3113	}
3114
3115	int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
3116	{
3117	int ret;
3118	struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
3119	struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
3120	struct amdgpu_bo *gws_bo = kgd_mem->bo;
3121
3122	/* Remove BO from process's validate list so restore worker won't touch
3123	* it anymore
3124	*/
3125	remove_kgd_mem_from_kfd_bo_list(mem: kgd_mem, process_info);
3126
3127	ret = amdgpu_bo_reserve(bo: gws_bo, no_intr: false);
3128	if (unlikely(ret)) {
3129	pr_err("Reserve gws bo failed %d\n", ret);
3130	//TODO add BO back to validate_list?
3131	return ret;
3132	}
3133	amdgpu_amdkfd_remove_eviction_fence(bo: gws_bo,
3134	ef: process_info->eviction_fence);
3135	amdgpu_bo_unreserve(bo: gws_bo);
3136	amdgpu_sync_free(sync: &kgd_mem->sync);
3137	amdgpu_bo_unref(bo: &gws_bo);
3138	mutex_destroy(lock: &kgd_mem->lock);
3139	kfree(objp: mem);
3140	return 0;
3141	}
3142
3143	/* Returns GPU-specific tiling mode information */
3144	int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
3145	struct tile_config *config)
3146	{
3147	config->gb_addr_config = adev->gfx.config.gb_addr_config;
3148	config->tile_config_ptr = adev->gfx.config.tile_mode_array;
3149	config->num_tile_configs =
3150	ARRAY_SIZE(adev->gfx.config.tile_mode_array);
3151	config->macro_tile_config_ptr =
3152	adev->gfx.config.macrotile_mode_array;
3153	config->num_macro_tile_configs =
3154	ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
3155
3156	/* Those values are not set from GFX9 onwards */
3157	config->num_banks = adev->gfx.config.num_banks;
3158	config->num_ranks = adev->gfx.config.num_ranks;
3159
3160	return 0;
3161	}
3162
3163	bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem)
3164	{
3165	struct kfd_mem_attachment *entry;
3166
3167	list_for_each_entry(entry, &mem->attachments, list) {
3168	if (entry->is_mapped && entry->adev == adev)
3169	return true;
3170	}
3171	return false;
3172	}
3173
3174	#if defined(CONFIG_DEBUG_FS)
3175
3176	int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data)
3177	{
3178
3179	spin_lock(lock: &kfd_mem_limit.mem_limit_lock);
3180	seq_printf(m, fmt: "System mem used %lldM out of %lluM\n",
3181	(kfd_mem_limit.system_mem_used >> 20),
3182	(kfd_mem_limit.max_system_mem_limit >> 20));
3183	seq_printf(m, fmt: "TTM mem used %lldM out of %lluM\n",
3184	(kfd_mem_limit.ttm_mem_used >> 20),
3185	(kfd_mem_limit.max_ttm_mem_limit >> 20));
3186	spin_unlock(lock: &kfd_mem_limit.mem_limit_lock);
3187
3188	return 0;
3189	}
3190
3191	#endif
3192