1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Copyright 2020-2021 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23
24	#include <linux/types.h>
25	#include <linux/sched/task.h>
26	#include <linux/dynamic_debug.h>
27	#include <drm/ttm/ttm_tt.h>
28	#include <drm/drm_exec.h>
29
30	#include "amdgpu_sync.h"
31	#include "amdgpu_object.h"
32	#include "amdgpu_vm.h"
33	#include "amdgpu_hmm.h"
34	#include "amdgpu.h"
35	#include "amdgpu_xgmi.h"
36	#include "kfd_priv.h"
37	#include "kfd_svm.h"
38	#include "kfd_migrate.h"
39	#include "kfd_smi_events.h"
40
41	#ifdef dev_fmt
42	#undef dev_fmt
43	#endif
44	#define dev_fmt(fmt) "kfd_svm: %s: " fmt, __func__
45
46	#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1
47
48	/* Long enough to ensure no retry fault comes after svm range is restored and
49	* page table is updated.
50	*/
51	#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING (2UL * NSEC_PER_MSEC)
52	#if IS_ENABLED(CONFIG_DYNAMIC_DEBUG)
53	#define dynamic_svm_range_dump(svms) \
54	_dynamic_func_call_no_desc("svm_range_dump", svm_range_debug_dump, svms)
55	#else
56	#define dynamic_svm_range_dump(svms) \
57	do { if (0) svm_range_debug_dump(svms); } while (0)
58	#endif
59
60	/* Giant svm range split into smaller ranges based on this, it is decided using
61	* minimum of all dGPU/APU 1/32 VRAM size, between 2MB to 1GB and alignment to
62	* power of 2MB.
63	*/
64	static uint64_t max_svm_range_pages;
65
66	struct criu_svm_metadata {
67	struct list_head list;
68	struct kfd_criu_svm_range_priv_data data;
69	};
70
71	static void svm_range_evict_svm_bo_worker(struct work_struct *work);
72	static bool
73	svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
74	const struct mmu_notifier_range *range,
75	unsigned long cur_seq);
76	static int
77	svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last,
78	uint64_t *bo_s, uint64_t *bo_l);
79	static const struct mmu_interval_notifier_ops svm_range_mn_ops = {
80	.invalidate = svm_range_cpu_invalidate_pagetables,
81	};
82
83	/**
84	* svm_range_unlink - unlink svm_range from lists and interval tree
85	* @prange: svm range structure to be removed
86	*
87	* Remove the svm_range from the svms and svm_bo lists and the svms
88	* interval tree.
89	*
90	* Context: The caller must hold svms->lock
91	*/
92	static void svm_range_unlink(struct svm_range *prange)
93	{
94	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
95	prange, prange->start, prange->last);
96
97	if (prange->svm_bo) {
98	spin_lock(lock: &prange->svm_bo->list_lock);
99	list_del(entry: &prange->svm_bo_list);
100	spin_unlock(lock: &prange->svm_bo->list_lock);
101	}
102
103	list_del(entry: &prange->list);
104	if (prange->it_node.start != 0 && prange->it_node.last != 0)
105	interval_tree_remove(node: &prange->it_node, root: &prange->svms->objects);
106	}
107
108	static void
109	svm_range_add_notifier_locked(struct mm_struct *mm, struct svm_range *prange)
110	{
111	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
112	prange, prange->start, prange->last);
113
114	mmu_interval_notifier_insert_locked(interval_sub: &prange->notifier, mm,
115	start: prange->start << PAGE_SHIFT,
116	length: prange->npages << PAGE_SHIFT,
117	ops: &svm_range_mn_ops);
118	}
119
120	/**
121	* svm_range_add_to_svms - add svm range to svms
122	* @prange: svm range structure to be added
123	*
124	* Add the svm range to svms interval tree and link list
125	*
126	* Context: The caller must hold svms->lock
127	*/
128	static void svm_range_add_to_svms(struct svm_range *prange)
129	{
130	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
131	prange, prange->start, prange->last);
132
133	list_move_tail(list: &prange->list, head: &prange->svms->list);
134	prange->it_node.start = prange->start;
135	prange->it_node.last = prange->last;
136	interval_tree_insert(node: &prange->it_node, root: &prange->svms->objects);
137	}
138
139	static void svm_range_remove_notifier(struct svm_range *prange)
140	{
141	pr_debug("remove notifier svms 0x%p prange 0x%p [0x%lx 0x%lx]\n",
142	prange->svms, prange,
143	prange->notifier.interval_tree.start >> PAGE_SHIFT,
144	prange->notifier.interval_tree.last >> PAGE_SHIFT);
145
146	if (prange->notifier.interval_tree.start != 0 &&
147	prange->notifier.interval_tree.last != 0)
148	mmu_interval_notifier_remove(interval_sub: &prange->notifier);
149	}
150
151	static bool
152	svm_is_valid_dma_mapping_addr(struct device *dev, dma_addr_t dma_addr)
153	{
154	return dma_addr && !dma_mapping_error(dev, dma_addr) &&
155	!(dma_addr & SVM_RANGE_VRAM_DOMAIN);
156	}
157
158	static int
159	svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange,
160	unsigned long offset, unsigned long npages,
161	unsigned long *hmm_pfns, uint32_t gpuidx)
162	{
163	enum dma_data_direction dir = DMA_BIDIRECTIONAL;
164	dma_addr_t *addr = prange->dma_addr[gpuidx];
165	struct device *dev = adev->dev;
166	struct page *page;
167	int i, r;
168
169	if (!addr) {
170	addr = kvcalloc(prange->npages, sizeof(*addr), GFP_KERNEL);
171	if (!addr)
172	return -ENOMEM;
173	prange->dma_addr[gpuidx] = addr;
174	}
175
176	addr += offset;
177	for (i = 0; i < npages; i++) {
178	if (svm_is_valid_dma_mapping_addr(dev, dma_addr: addr[i]))
179	dma_unmap_page(dev, addr[i], PAGE_SIZE, dir);
180
181	page = hmm_pfn_to_page(hmm_pfn: hmm_pfns[i]);
182	if (is_zone_device_page(page)) {
183	struct amdgpu_device *bo_adev = prange->svm_bo->node->adev;
184
185	addr[i] = (hmm_pfns[i] << PAGE_SHIFT) +
186	bo_adev->vm_manager.vram_base_offset -
187	bo_adev->kfd.pgmap.range.start;
188	addr[i] |= SVM_RANGE_VRAM_DOMAIN;
189	pr_debug_ratelimited("vram address: 0x%llx\n", addr[i]);
190	continue;
191	}
192	addr[i] = dma_map_page(dev, page, 0, PAGE_SIZE, dir);
193	r = dma_mapping_error(dev, dma_addr: addr[i]);
194	if (r) {
195	dev_err(dev, "failed %d dma_map_page\n", r);
196	return r;
197	}
198	pr_debug_ratelimited("dma mapping 0x%llx for page addr 0x%lx\n",
199	addr[i] >> PAGE_SHIFT, page_to_pfn(page));
200	}
201
202	return 0;
203	}
204
205	static int
206	svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap,
207	unsigned long offset, unsigned long npages,
208	unsigned long *hmm_pfns)
209	{
210	struct kfd_process *p;
211	uint32_t gpuidx;
212	int r;
213
214	p = container_of(prange->svms, struct kfd_process, svms);
215
216	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
217	struct kfd_process_device *pdd;
218
219	pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
220	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
221	if (!pdd) {
222	pr_debug("failed to find device idx %d\n", gpuidx);
223	return -EINVAL;
224	}
225
226	r = svm_range_dma_map_dev(adev: pdd->dev->adev, prange, offset, npages,
227	hmm_pfns, gpuidx);
228	if (r)
229	break;
230	}
231
232	return r;
233	}
234
235	void svm_range_dma_unmap_dev(struct device *dev, dma_addr_t *dma_addr,
236	unsigned long offset, unsigned long npages)
237	{
238	enum dma_data_direction dir = DMA_BIDIRECTIONAL;
239	int i;
240
241	if (!dma_addr)
242	return;
243
244	for (i = offset; i < offset + npages; i++) {
245	if (!svm_is_valid_dma_mapping_addr(dev, dma_addr: dma_addr[i]))
246	continue;
247	pr_debug_ratelimited("unmap 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
248	dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
249	dma_addr[i] = 0;
250	}
251	}
252
253	void svm_range_dma_unmap(struct svm_range *prange)
254	{
255	struct kfd_process_device *pdd;
256	dma_addr_t *dma_addr;
257	struct device *dev;
258	struct kfd_process *p;
259	uint32_t gpuidx;
260
261	p = container_of(prange->svms, struct kfd_process, svms);
262
263	for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {
264	dma_addr = prange->dma_addr[gpuidx];
265	if (!dma_addr)
266	continue;
267
268	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
269	if (!pdd) {
270	pr_debug("failed to find device idx %d\n", gpuidx);
271	continue;
272	}
273	dev = &pdd->dev->adev->pdev->dev;
274
275	svm_range_dma_unmap_dev(dev, dma_addr, offset: 0, npages: prange->npages);
276	}
277	}
278
279	static void svm_range_free(struct svm_range *prange, bool do_unmap)
280	{
281	uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT;
282	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
283	uint32_t gpuidx;
284
285	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,
286	prange->start, prange->last);
287
288	svm_range_vram_node_free(prange);
289	if (do_unmap)
290	svm_range_dma_unmap(prange);
291
292	if (do_unmap && !p->xnack_enabled) {
293	pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size);
294	amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
295	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0);
296	}
297
298	/* free dma_addr array for each gpu */
299	for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {
300	if (prange->dma_addr[gpuidx]) {
301	kvfree(addr: prange->dma_addr[gpuidx]);
302	prange->dma_addr[gpuidx] = NULL;
303	}
304	}
305
306	mutex_destroy(lock: &prange->lock);
307	mutex_destroy(lock: &prange->migrate_mutex);
308	kfree(objp: prange);
309	}
310
311	static void
312	svm_range_set_default_attributes(struct svm_range_list *svms, int32_t *location,
313	int32_t *prefetch_loc, uint8_t *granularity,
314	uint32_t *flags)
315	{
316	*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
317	*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
318	*granularity = svms->default_granularity;
319	*flags =
320	KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT;
321	}
322
323	static struct
324	svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
325	uint64_t last, bool update_mem_usage)
326	{
327	uint64_t size = last - start + 1;
328	struct svm_range *prange;
329	struct kfd_process *p;
330
331	prange = kzalloc(sizeof(*prange), GFP_KERNEL);
332	if (!prange)
333	return NULL;
334
335	p = container_of(svms, struct kfd_process, svms);
336	if (!p->xnack_enabled && update_mem_usage &&
337	amdgpu_amdkfd_reserve_mem_limit(NULL, size: size << PAGE_SHIFT,
338	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0)) {
339	pr_info("SVM mapping failed, exceeds resident system memory limit\n");
340	kfree(objp: prange);
341	return NULL;
342	}
343	prange->npages = size;
344	prange->svms = svms;
345	prange->start = start;
346	prange->last = last;
347	INIT_LIST_HEAD(list: &prange->list);
348	INIT_LIST_HEAD(list: &prange->update_list);
349	INIT_LIST_HEAD(list: &prange->svm_bo_list);
350	INIT_LIST_HEAD(list: &prange->deferred_list);
351	INIT_LIST_HEAD(list: &prange->child_list);
352	atomic_set(v: &prange->invalid, i: 0);
353	prange->validate_timestamp = 0;
354	prange->vram_pages = 0;
355	mutex_init(&prange->migrate_mutex);
356	mutex_init(&prange->lock);
357
358	if (p->xnack_enabled)
359	bitmap_copy(dst: prange->bitmap_access, src: svms->bitmap_supported,
360	MAX_GPU_INSTANCE);
361
362	svm_range_set_default_attributes(svms, location: &prange->preferred_loc,
363	prefetch_loc: &prange->prefetch_loc,
364	granularity: &prange->granularity, flags: &prange->flags);
365
366	pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
367
368	return prange;
369	}
370
371	static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)
372	{
373	if (!svm_bo || !kref_get_unless_zero(kref: &svm_bo->kref))
374	return false;
375
376	return true;
377	}
378
379	static void svm_range_bo_release(struct kref *kref)
380	{
381	struct svm_range_bo *svm_bo;
382
383	svm_bo = container_of(kref, struct svm_range_bo, kref);
384	pr_debug("svm_bo 0x%p\n", svm_bo);
385
386	spin_lock(lock: &svm_bo->list_lock);
387	while (!list_empty(head: &svm_bo->range_list)) {
388	struct svm_range *prange =
389	list_first_entry(&svm_bo->range_list,
390	struct svm_range, svm_bo_list);
391	/* list_del_init tells a concurrent svm_range_vram_node_new when
392	* it's safe to reuse the svm_bo pointer and svm_bo_list head.
393	*/
394	list_del_init(entry: &prange->svm_bo_list);
395	spin_unlock(lock: &svm_bo->list_lock);
396
397	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
398	prange->start, prange->last);
399	mutex_lock(&prange->lock);
400	prange->svm_bo = NULL;
401	/* prange should not hold vram page now */
402	WARN_ONCE(prange->actual_loc, "prange should not hold vram page");
403	mutex_unlock(lock: &prange->lock);
404
405	spin_lock(lock: &svm_bo->list_lock);
406	}
407	spin_unlock(lock: &svm_bo->list_lock);
408
409	if (mmget_not_zero(mm: svm_bo->eviction_fence->mm)) {
410	struct kfd_process_device *pdd;
411	struct kfd_process *p;
412	struct mm_struct *mm;
413
414	mm = svm_bo->eviction_fence->mm;
415	/*
416	* The forked child process takes svm_bo device pages ref, svm_bo could be
417	* released after parent process is gone.
418	*/
419	p = kfd_lookup_process_by_mm(mm);
420	if (p) {
421	pdd = kfd_get_process_device_data(dev: svm_bo->node, p);
422	if (pdd)
423	atomic64_sub(i: amdgpu_bo_size(bo: svm_bo->bo), v: &pdd->vram_usage);
424	kfd_unref_process(p);
425	}
426	mmput(mm);
427	}
428
429	if (!dma_fence_is_signaled(fence: &svm_bo->eviction_fence->base))
430	/* We're not in the eviction worker. Signal the fence. */
431	dma_fence_signal(fence: &svm_bo->eviction_fence->base);
432	dma_fence_put(fence: &svm_bo->eviction_fence->base);
433	amdgpu_bo_unref(bo: &svm_bo->bo);
434	kfree(objp: svm_bo);
435	}
436
437	static void svm_range_bo_wq_release(struct work_struct *work)
438	{
439	struct svm_range_bo *svm_bo;
440
441	svm_bo = container_of(work, struct svm_range_bo, release_work);
442	svm_range_bo_release(kref: &svm_bo->kref);
443	}
444
445	static void svm_range_bo_release_async(struct kref *kref)
446	{
447	struct svm_range_bo *svm_bo;
448
449	svm_bo = container_of(kref, struct svm_range_bo, kref);
450	pr_debug("svm_bo 0x%p\n", svm_bo);
451	INIT_WORK(&svm_bo->release_work, svm_range_bo_wq_release);
452	schedule_work(work: &svm_bo->release_work);
453	}
454
455	void svm_range_bo_unref_async(struct svm_range_bo *svm_bo)
456	{
457	kref_put(kref: &svm_bo->kref, release: svm_range_bo_release_async);
458	}
459
460	static void svm_range_bo_unref(struct svm_range_bo *svm_bo)
461	{
462	if (svm_bo)
463	kref_put(kref: &svm_bo->kref, release: svm_range_bo_release);
464	}
465
466	static bool
467	svm_range_validate_svm_bo(struct kfd_node *node, struct svm_range *prange)
468	{
469	mutex_lock(&prange->lock);
470	if (!prange->svm_bo) {
471	mutex_unlock(lock: &prange->lock);
472	return false;
473	}
474	if (prange->ttm_res) {
475	/* We still have a reference, all is well */
476	mutex_unlock(lock: &prange->lock);
477	return true;
478	}
479	if (svm_bo_ref_unless_zero(svm_bo: prange->svm_bo)) {
480	/*
481	* Migrate from GPU to GPU, remove range from source svm_bo->node
482	* range list, and return false to allocate svm_bo from destination
483	* node.
484	*/
485	if (prange->svm_bo->node != node) {
486	mutex_unlock(lock: &prange->lock);
487
488	spin_lock(lock: &prange->svm_bo->list_lock);
489	list_del_init(entry: &prange->svm_bo_list);
490	spin_unlock(lock: &prange->svm_bo->list_lock);
491
492	svm_range_bo_unref(svm_bo: prange->svm_bo);
493	return false;
494	}
495	if (READ_ONCE(prange->svm_bo->evicting)) {
496	struct dma_fence *f;
497	struct svm_range_bo *svm_bo;
498	/* The BO is getting evicted,
499	* we need to get a new one
500	*/
501	mutex_unlock(lock: &prange->lock);
502	svm_bo = prange->svm_bo;
503	f = dma_fence_get(fence: &svm_bo->eviction_fence->base);
504	svm_range_bo_unref(svm_bo: prange->svm_bo);
505	/* wait for the fence to avoid long spin-loop
506	* at list_empty_careful
507	*/
508	dma_fence_wait(fence: f, intr: false);
509	dma_fence_put(fence: f);
510	} else {
511	/* The BO was still around and we got
512	* a new reference to it
513	*/
514	mutex_unlock(lock: &prange->lock);
515	pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n",
516	prange->svms, prange->start, prange->last);
517
518	prange->ttm_res = prange->svm_bo->bo->tbo.resource;
519	return true;
520	}
521
522	} else {
523	mutex_unlock(lock: &prange->lock);
524	}
525
526	/* We need a new svm_bo. Spin-loop to wait for concurrent
527	* svm_range_bo_release to finish removing this range from
528	* its range list and set prange->svm_bo to null. After this,
529	* it is safe to reuse the svm_bo pointer and svm_bo_list head.
530	*/
531	while (!list_empty_careful(head: &prange->svm_bo_list) || prange->svm_bo)
532	cond_resched();
533
534	return false;
535	}
536
537	static struct svm_range_bo *svm_range_bo_new(void)
538	{
539	struct svm_range_bo *svm_bo;
540
541	svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL);
542	if (!svm_bo)
543	return NULL;
544
545	kref_init(kref: &svm_bo->kref);
546	INIT_LIST_HEAD(list: &svm_bo->range_list);
547	spin_lock_init(&svm_bo->list_lock);
548
549	return svm_bo;
550	}
551
552	int
553	svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
554	bool clear)
555	{
556	struct kfd_process_device *pdd;
557	struct amdgpu_bo_param bp;
558	struct svm_range_bo *svm_bo;
559	struct amdgpu_bo_user *ubo;
560	struct amdgpu_bo *bo;
561	struct kfd_process *p;
562	struct mm_struct *mm;
563	int r;
564
565	p = container_of(prange->svms, struct kfd_process, svms);
566	pr_debug("process pid: %d svms 0x%p [0x%lx 0x%lx]\n",
567	p->lead_thread->pid, prange->svms,
568	prange->start, prange->last);
569
570	if (svm_range_validate_svm_bo(node, prange))
571	return 0;
572
573	svm_bo = svm_range_bo_new();
574	if (!svm_bo) {
575	pr_debug("failed to alloc svm bo\n");
576	return -ENOMEM;
577	}
578	mm = get_task_mm(task: p->lead_thread);
579	if (!mm) {
580	pr_debug("failed to get mm\n");
581	kfree(objp: svm_bo);
582	return -ESRCH;
583	}
584	svm_bo->node = node;
585	svm_bo->eviction_fence =
586	amdgpu_amdkfd_fence_create(context: dma_fence_context_alloc(num: 1),
587	mm,
588	svm_bo);
589	mmput(mm);
590	INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker);
591	svm_bo->evicting = 0;
592	memset(&bp, 0, sizeof(bp));
593	bp.size = prange->npages * PAGE_SIZE;
594	bp.byte_align = PAGE_SIZE;
595	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
596	bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
597	bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
598	bp.flags |= AMDGPU_GEM_CREATE_DISCARDABLE;
599	bp.type = ttm_bo_type_device;
600	bp.resv = NULL;
601	if (node->xcp)
602	bp.xcp_id_plus1 = node->xcp->id + 1;
603
604	r = amdgpu_bo_create_user(adev: node->adev, bp: &bp, ubo_ptr: &ubo);
605	if (r) {
606	pr_debug("failed %d to create bo\n", r);
607	goto create_bo_failed;
608	}
609	bo = &ubo->bo;
610
611	pr_debug("alloc bo at offset 0x%lx size 0x%lx on partition %d\n",
612	bo->tbo.resource->start << PAGE_SHIFT, bp.size,
613	bp.xcp_id_plus1 - 1);
614
615	r = amdgpu_bo_reserve(bo, no_intr: true);
616	if (r) {
617	pr_debug("failed %d to reserve bo\n", r);
618	goto reserve_bo_failed;
619	}
620
621	if (clear) {
622	r = amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, intr: false);
623	if (r) {
624	pr_debug("failed %d to sync bo\n", r);
625	amdgpu_bo_unreserve(bo);
626	goto reserve_bo_failed;
627	}
628	}
629
630	r = dma_resv_reserve_fences(obj: bo->tbo.base.resv, num_fences: 1);
631	if (r) {
632	pr_debug("failed %d to reserve bo\n", r);
633	amdgpu_bo_unreserve(bo);
634	goto reserve_bo_failed;
635	}
636	amdgpu_bo_fence(bo, fence: &svm_bo->eviction_fence->base, shared: true);
637
638	amdgpu_bo_unreserve(bo);
639
640	svm_bo->bo = bo;
641	prange->svm_bo = svm_bo;
642	prange->ttm_res = bo->tbo.resource;
643	prange->offset = 0;
644
645	spin_lock(lock: &svm_bo->list_lock);
646	list_add(new: &prange->svm_bo_list, head: &svm_bo->range_list);
647	spin_unlock(lock: &svm_bo->list_lock);
648
649	pdd = svm_range_get_pdd_by_node(prange, node);
650	if (pdd)
651	atomic64_add(i: amdgpu_bo_size(bo), v: &pdd->vram_usage);
652
653	return 0;
654
655	reserve_bo_failed:
656	amdgpu_bo_unref(bo: &bo);
657	create_bo_failed:
658	dma_fence_put(fence: &svm_bo->eviction_fence->base);
659	kfree(objp: svm_bo);
660	prange->ttm_res = NULL;
661
662	return r;
663	}
664
665	void svm_range_vram_node_free(struct svm_range *prange)
666	{
667	/* serialize prange->svm_bo unref */
668	mutex_lock(&prange->lock);
669	/* prange->svm_bo has not been unref */
670	if (prange->ttm_res) {
671	prange->ttm_res = NULL;
672	mutex_unlock(lock: &prange->lock);
673	svm_range_bo_unref(svm_bo: prange->svm_bo);
674	} else
675	mutex_unlock(lock: &prange->lock);
676	}
677
678	struct kfd_node *
679	svm_range_get_node_by_id(struct svm_range *prange, uint32_t gpu_id)
680	{
681	struct kfd_process *p;
682	struct kfd_process_device *pdd;
683
684	p = container_of(prange->svms, struct kfd_process, svms);
685	pdd = kfd_process_device_data_by_id(process: p, gpu_id);
686	if (!pdd) {
687	pr_debug("failed to get kfd process device by id 0x%x\n", gpu_id);
688	return NULL;
689	}
690
691	return pdd->dev;
692	}
693
694	struct kfd_process_device *
695	svm_range_get_pdd_by_node(struct svm_range *prange, struct kfd_node *node)
696	{
697	struct kfd_process *p;
698
699	p = container_of(prange->svms, struct kfd_process, svms);
700
701	return kfd_get_process_device_data(dev: node, p);
702	}
703
704	static int svm_range_bo_validate(void *param, struct amdgpu_bo *bo)
705	{
706	struct ttm_operation_ctx ctx = { false, false };
707
708	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_VRAM);
709
710	return ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
711	}
712
713	static int
714	svm_range_check_attr(struct kfd_process *p,
715	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
716	{
717	uint32_t i;
718
719	for (i = 0; i < nattr; i++) {
720	uint32_t val = attrs[i].value;
721	int gpuidx = MAX_GPU_INSTANCE;
722
723	switch (attrs[i].type) {
724	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
725	if (val != KFD_IOCTL_SVM_LOCATION_SYSMEM &&
726	val != KFD_IOCTL_SVM_LOCATION_UNDEFINED)
727	gpuidx = kfd_process_gpuidx_from_gpuid(p, gpu_id: val);
728	break;
729	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
730	if (val != KFD_IOCTL_SVM_LOCATION_SYSMEM)
731	gpuidx = kfd_process_gpuidx_from_gpuid(p, gpu_id: val);
732	break;
733	case KFD_IOCTL_SVM_ATTR_ACCESS:
734	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
735	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
736	gpuidx = kfd_process_gpuidx_from_gpuid(p, gpu_id: val);
737	break;
738	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
739	break;
740	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
741	break;
742	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
743	break;
744	default:
745	pr_debug("unknown attr type 0x%x\n", attrs[i].type);
746	return -EINVAL;
747	}
748
749	if (gpuidx < 0) {
750	pr_debug("no GPU 0x%x found\n", val);
751	return -EINVAL;
752	} else if (gpuidx < MAX_GPU_INSTANCE &&
753	!test_bit(gpuidx, p->svms.bitmap_supported)) {
754	pr_debug("GPU 0x%x not supported\n", val);
755	return -EINVAL;
756	}
757	}
758
759	return 0;
760	}
761
762	static void
763	svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange,
764	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
765	bool *update_mapping)
766	{
767	uint32_t i;
768	int gpuidx;
769
770	for (i = 0; i < nattr; i++) {
771	switch (attrs[i].type) {
772	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
773	prange->preferred_loc = attrs[i].value;
774	break;
775	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
776	prange->prefetch_loc = attrs[i].value;
777	break;
778	case KFD_IOCTL_SVM_ATTR_ACCESS:
779	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
780	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
781	if (!p->xnack_enabled)
782	*update_mapping = true;
783
784	gpuidx = kfd_process_gpuidx_from_gpuid(p,
785	gpu_id: attrs[i].value);
786	if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {
787	bitmap_clear(map: prange->bitmap_access, start: gpuidx, nbits: 1);
788	bitmap_clear(map: prange->bitmap_aip, start: gpuidx, nbits: 1);
789	} else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {
790	bitmap_set(map: prange->bitmap_access, start: gpuidx, nbits: 1);
791	bitmap_clear(map: prange->bitmap_aip, start: gpuidx, nbits: 1);
792	} else {
793	bitmap_clear(map: prange->bitmap_access, start: gpuidx, nbits: 1);
794	bitmap_set(map: prange->bitmap_aip, start: gpuidx, nbits: 1);
795	}
796	break;
797	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
798	*update_mapping = true;
799	prange->flags |= attrs[i].value;
800	break;
801	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
802	*update_mapping = true;
803	prange->flags &= ~attrs[i].value;
804	break;
805	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
806	prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F);
807	break;
808	default:
809	WARN_ONCE(1, "svm_range_check_attrs wasn't called?");
810	}
811	}
812	}
813
814	static bool
815	svm_range_is_same_attrs(struct kfd_process *p, struct svm_range *prange,
816	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
817	{
818	uint32_t i;
819	int gpuidx;
820
821	for (i = 0; i < nattr; i++) {
822	switch (attrs[i].type) {
823	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
824	if (prange->preferred_loc != attrs[i].value)
825	return false;
826	break;
827	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
828	/* Prefetch should always trigger a migration even
829	* if the value of the attribute didn't change.
830	*/
831	return false;
832	case KFD_IOCTL_SVM_ATTR_ACCESS:
833	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
834	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
835	gpuidx = kfd_process_gpuidx_from_gpuid(p,
836	gpu_id: attrs[i].value);
837	if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {
838	if (test_bit(gpuidx, prange->bitmap_access) ||
839	test_bit(gpuidx, prange->bitmap_aip))
840	return false;
841	} else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {
842	if (!test_bit(gpuidx, prange->bitmap_access))
843	return false;
844	} else {
845	if (!test_bit(gpuidx, prange->bitmap_aip))
846	return false;
847	}
848	break;
849	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
850	if ((prange->flags & attrs[i].value) != attrs[i].value)
851	return false;
852	break;
853	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
854	if ((prange->flags & attrs[i].value) != 0)
855	return false;
856	break;
857	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
858	if (prange->granularity != attrs[i].value)
859	return false;
860	break;
861	default:
862	WARN_ONCE(1, "svm_range_check_attrs wasn't called?");
863	}
864	}
865
866	return true;
867	}
868
869	/**
870	* svm_range_debug_dump - print all range information from svms
871	* @svms: svm range list header
872	*
873	* debug output svm range start, end, prefetch location from svms
874	* interval tree and link list
875	*
876	* Context: The caller must hold svms->lock
877	*/
878	static void svm_range_debug_dump(struct svm_range_list *svms)
879	{
880	struct interval_tree_node *node;
881	struct svm_range *prange;
882
883	pr_debug("dump svms 0x%p list\n", svms);
884	pr_debug("range\tstart\tpage\tend\t\tlocation\n");
885
886	list_for_each_entry(prange, &svms->list, list) {
887	pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",
888	prange, prange->start, prange->npages,
889	prange->start + prange->npages - 1,
890	prange->actual_loc);
891	}
892
893	pr_debug("dump svms 0x%p interval tree\n", svms);
894	pr_debug("range\tstart\tpage\tend\t\tlocation\n");
895	node = interval_tree_iter_first(root: &svms->objects, start: 0, last: ~0ULL);
896	while (node) {
897	prange = container_of(node, struct svm_range, it_node);
898	pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",
899	prange, prange->start, prange->npages,
900	prange->start + prange->npages - 1,
901	prange->actual_loc);
902	node = interval_tree_iter_next(node, start: 0, last: ~0ULL);
903	}
904	}
905
906	static void *
907	svm_range_copy_array(void *psrc, size_t size, uint64_t num_elements,
908	uint64_t offset, uint64_t *vram_pages)
909	{
910	unsigned char *src = (unsigned char *)psrc + offset;
911	unsigned char *dst;
912	uint64_t i;
913
914	dst = kvmalloc_array(num_elements, size, GFP_KERNEL);
915	if (!dst)
916	return NULL;
917
918	if (!vram_pages) {
919	memcpy(dst, src, num_elements * size);
920	return (void *)dst;
921	}
922
923	*vram_pages = 0;
924	for (i = 0; i < num_elements; i++) {
925	dma_addr_t *temp;
926	temp = (dma_addr_t *)dst + i;
927	*temp = *((dma_addr_t *)src + i);
928	if (*temp&SVM_RANGE_VRAM_DOMAIN)
929	(*vram_pages)++;
930	}
931
932	return (void *)dst;
933	}
934
935	static int
936	svm_range_copy_dma_addrs(struct svm_range *dst, struct svm_range *src)
937	{
938	int i;
939
940	for (i = 0; i < MAX_GPU_INSTANCE; i++) {
941	if (!src->dma_addr[i])
942	continue;
943	dst->dma_addr[i] = svm_range_copy_array(psrc: src->dma_addr[i],
944	size: sizeof(*src->dma_addr[i]), num_elements: src->npages, offset: 0, NULL);
945	if (!dst->dma_addr[i])
946	return -ENOMEM;
947	}
948
949	return 0;
950	}
951
952	static int
953	svm_range_split_array(void *ppnew, void *ppold, size_t size,
954	uint64_t old_start, uint64_t old_n,
955	uint64_t new_start, uint64_t new_n, uint64_t *new_vram_pages)
956	{
957	unsigned char *new, *old, *pold;
958	uint64_t d;
959
960	if (!ppold)
961	return 0;
962	pold = *(unsigned char **)ppold;
963	if (!pold)
964	return 0;
965
966	d = (new_start - old_start) * size;
967	/* get dma addr array for new range and calculte its vram page number */
968	new = svm_range_copy_array(psrc: pold, size, num_elements: new_n, offset: d, vram_pages: new_vram_pages);
969	if (!new)
970	return -ENOMEM;
971	d = (new_start == old_start) ? new_n * size : 0;
972	old = svm_range_copy_array(psrc: pold, size, num_elements: old_n, offset: d, NULL);
973	if (!old) {
974	kvfree(addr: new);
975	return -ENOMEM;
976	}
977	kvfree(addr: pold);
978	*(void **)ppold = old;
979	*(void **)ppnew = new;
980
981	return 0;
982	}
983
984	static int
985	svm_range_split_pages(struct svm_range *new, struct svm_range *old,
986	uint64_t start, uint64_t last)
987	{
988	uint64_t npages = last - start + 1;
989	int i, r;
990
991	for (i = 0; i < MAX_GPU_INSTANCE; i++) {
992	r = svm_range_split_array(ppnew: &new->dma_addr[i], ppold: &old->dma_addr[i],
993	size: sizeof(*old->dma_addr[i]), old_start: old->start,
994	old_n: npages, new_start: new->start, new_n: new->npages,
995	new_vram_pages: old->actual_loc ? &new->vram_pages : NULL);
996	if (r)
997	return r;
998	}
999	if (old->actual_loc)
1000	old->vram_pages -= new->vram_pages;
1001
1002	return 0;
1003	}
1004
1005	static int
1006	svm_range_split_nodes(struct svm_range *new, struct svm_range *old,
1007	uint64_t start, uint64_t last)
1008	{
1009	uint64_t npages = last - start + 1;
1010
1011	pr_debug("svms 0x%p new prange 0x%p start 0x%lx [0x%llx 0x%llx]\n",
1012	new->svms, new, new->start, start, last);
1013
1014	if (new->start == old->start) {
1015	new->offset = old->offset;
1016	old->offset += new->npages;
1017	} else {
1018	new->offset = old->offset + npages;
1019	}
1020
1021	new->svm_bo = svm_range_bo_ref(svm_bo: old->svm_bo);
1022	new->ttm_res = old->ttm_res;
1023
1024	spin_lock(lock: &new->svm_bo->list_lock);
1025	list_add(new: &new->svm_bo_list, head: &new->svm_bo->range_list);
1026	spin_unlock(lock: &new->svm_bo->list_lock);
1027
1028	return 0;
1029	}
1030
1031	/**
1032	* svm_range_split_adjust - split range and adjust
1033	*
1034	* @new: new range
1035	* @old: the old range
1036	* @start: the old range adjust to start address in pages
1037	* @last: the old range adjust to last address in pages
1038	*
1039	* Copy system memory dma_addr or vram ttm_res in old range to new
1040	* range from new_start up to size new->npages, the remaining old range is from
1041	* start to last
1042	*
1043	* Return:
1044	* 0 - OK, -ENOMEM - out of memory
1045	*/
1046	static int
1047	svm_range_split_adjust(struct svm_range *new, struct svm_range *old,
1048	uint64_t start, uint64_t last)
1049	{
1050	int r;
1051
1052	pr_debug("svms 0x%p new 0x%lx old [0x%lx 0x%lx] => [0x%llx 0x%llx]\n",
1053	new->svms, new->start, old->start, old->last, start, last);
1054
1055	if (new->start < old->start ||
1056	new->last > old->last) {
1057	WARN_ONCE(1, "invalid new range start or last\n");
1058	return -EINVAL;
1059	}
1060
1061	r = svm_range_split_pages(new, old, start, last);
1062	if (r)
1063	return r;
1064
1065	if (old->actual_loc && old->ttm_res) {
1066	r = svm_range_split_nodes(new, old, start, last);
1067	if (r)
1068	return r;
1069	}
1070
1071	old->npages = last - start + 1;
1072	old->start = start;
1073	old->last = last;
1074	new->flags = old->flags;
1075	new->preferred_loc = old->preferred_loc;
1076	new->prefetch_loc = old->prefetch_loc;
1077	new->actual_loc = old->actual_loc;
1078	new->granularity = old->granularity;
1079	new->mapped_to_gpu = old->mapped_to_gpu;
1080	bitmap_copy(dst: new->bitmap_access, src: old->bitmap_access, MAX_GPU_INSTANCE);
1081	bitmap_copy(dst: new->bitmap_aip, src: old->bitmap_aip, MAX_GPU_INSTANCE);
1082	atomic_set(v: &new->queue_refcount, i: atomic_read(v: &old->queue_refcount));
1083
1084	return 0;
1085	}
1086
1087	/**
1088	* svm_range_split - split a range in 2 ranges
1089	*
1090	* @prange: the svm range to split
1091	* @start: the remaining range start address in pages
1092	* @last: the remaining range last address in pages
1093	* @new: the result new range generated
1094	*
1095	* Two cases only:
1096	* case 1: if start == prange->start
1097	* prange ==> prange[start, last]
1098	* new range [last + 1, prange->last]
1099	*
1100	* case 2: if last == prange->last
1101	* prange ==> prange[start, last]
1102	* new range [prange->start, start - 1]
1103	*
1104	* Return:
1105	* 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last
1106	*/
1107	static int
1108	svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,
1109	struct svm_range **new)
1110	{
1111	uint64_t old_start = prange->start;
1112	uint64_t old_last = prange->last;
1113	struct svm_range_list *svms;
1114	int r = 0;
1115
1116	pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms,
1117	old_start, old_last, start, last);
1118
1119	if (old_start != start && old_last != last)
1120	return -EINVAL;
1121	if (start < old_start || last > old_last)
1122	return -EINVAL;
1123
1124	svms = prange->svms;
1125	if (old_start == start)
1126	*new = svm_range_new(svms, start: last + 1, last: old_last, update_mem_usage: false);
1127	else
1128	*new = svm_range_new(svms, start: old_start, last: start - 1, update_mem_usage: false);
1129	if (!*new)
1130	return -ENOMEM;
1131
1132	r = svm_range_split_adjust(new: *new, old: prange, start, last);
1133	if (r) {
1134	pr_debug("failed %d split [0x%llx 0x%llx] to [0x%llx 0x%llx]\n",
1135	r, old_start, old_last, start, last);
1136	svm_range_free(prange: *new, do_unmap: false);
1137	*new = NULL;
1138	}
1139
1140	return r;
1141	}
1142
1143	static int
1144	svm_range_split_tail(struct svm_range *prange, uint64_t new_last,
1145	struct list_head *insert_list, struct list_head *remap_list)
1146	{
1147	unsigned long last_align_down = ALIGN_DOWN(prange->last, 512);
1148	unsigned long start_align = ALIGN(prange->start, 512);
1149	bool huge_page_mapping = last_align_down > start_align;
1150	struct svm_range *tail = NULL;
1151	int r;
1152
1153	r = svm_range_split(prange, start: prange->start, last: new_last, new: &tail);
1154
1155	if (r)
1156	return r;
1157
1158	list_add(new: &tail->list, head: insert_list);
1159
1160	if (huge_page_mapping && tail->start > start_align &&
1161	tail->start < last_align_down && (!IS_ALIGNED(tail->start, 512)))
1162	list_add(new: &tail->update_list, head: remap_list);
1163
1164	return 0;
1165	}
1166
1167	static int
1168	svm_range_split_head(struct svm_range *prange, uint64_t new_start,
1169	struct list_head *insert_list, struct list_head *remap_list)
1170	{
1171	unsigned long last_align_down = ALIGN_DOWN(prange->last, 512);
1172	unsigned long start_align = ALIGN(prange->start, 512);
1173	bool huge_page_mapping = last_align_down > start_align;
1174	struct svm_range *head = NULL;
1175	int r;
1176
1177	r = svm_range_split(prange, start: new_start, last: prange->last, new: &head);
1178
1179	if (r)
1180	return r;
1181
1182	list_add(new: &head->list, head: insert_list);
1183
1184	if (huge_page_mapping && head->last + 1 > start_align &&
1185	head->last + 1 < last_align_down && (!IS_ALIGNED(head->last, 512)))
1186	list_add(new: &head->update_list, head: remap_list);
1187
1188	return 0;
1189	}
1190
1191	static void
1192	svm_range_add_child(struct svm_range *prange, struct svm_range *pchild, enum svm_work_list_ops op)
1193	{
1194	pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n",
1195	pchild, pchild->start, pchild->last, prange, op);
1196
1197	pchild->work_item.mm = NULL;
1198	pchild->work_item.op = op;
1199	list_add_tail(new: &pchild->child_list, head: &prange->child_list);
1200	}
1201
1202	static bool
1203	svm_nodes_in_same_hive(struct kfd_node *node_a, struct kfd_node *node_b)
1204	{
1205	return (node_a->adev == node_b->adev ||
1206	amdgpu_xgmi_same_hive(adev: node_a->adev, bo_adev: node_b->adev));
1207	}
1208
1209	static uint64_t
1210	svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
1211	struct svm_range *prange, int domain)
1212	{
1213	struct kfd_node *bo_node;
1214	uint32_t flags = prange->flags;
1215	uint32_t mapping_flags = 0;
1216	uint32_t gc_ip_version = KFD_GC_VERSION(node);
1217	uint64_t pte_flags;
1218	bool snoop = (domain != SVM_RANGE_VRAM_DOMAIN);
1219	bool coherent = flags & (KFD_IOCTL_SVM_FLAG_COHERENT | KFD_IOCTL_SVM_FLAG_EXT_COHERENT);
1220	bool ext_coherent = flags & KFD_IOCTL_SVM_FLAG_EXT_COHERENT;
1221	unsigned int mtype_local;
1222
1223	if (domain == SVM_RANGE_VRAM_DOMAIN)
1224	bo_node = prange->svm_bo->node;
1225
1226	switch (gc_ip_version) {
1227	case IP_VERSION(9, 4, 1):
1228	if (domain == SVM_RANGE_VRAM_DOMAIN) {
1229	if (bo_node == node) {
1230	mapping_flags |= coherent ?
1231	AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
1232	} else {
1233	mapping_flags |= coherent ?
1234	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1235	if (svm_nodes_in_same_hive(node_a: node, node_b: bo_node))
1236	snoop = true;
1237	}
1238	} else {
1239	mapping_flags |= coherent ?
1240	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1241	}
1242	break;
1243	case IP_VERSION(9, 4, 2):
1244	if (domain == SVM_RANGE_VRAM_DOMAIN) {
1245	if (bo_node == node) {
1246	mapping_flags |= coherent ?
1247	AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
1248	if (node->adev->gmc.xgmi.connected_to_cpu)
1249	snoop = true;
1250	} else {
1251	mapping_flags |= coherent ?
1252	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1253	if (svm_nodes_in_same_hive(node_a: node, node_b: bo_node))
1254	snoop = true;
1255	}
1256	} else {
1257	mapping_flags |= coherent ?
1258	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1259	}
1260	break;
1261	case IP_VERSION(9, 4, 3):
1262	case IP_VERSION(9, 4, 4):
1263	case IP_VERSION(9, 5, 0):
1264	if (ext_coherent)
1265	mtype_local = AMDGPU_VM_MTYPE_CC;
1266	else
1267	mtype_local = amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
1268	amdgpu_mtype_local == 2 ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
1269	snoop = true;
1270	if (domain == SVM_RANGE_VRAM_DOMAIN) {
1271	/* local HBM region close to partition */
1272	if (bo_node->adev == node->adev &&
1273	(!bo_node->xcp || !node->xcp || bo_node->xcp->mem_id == node->xcp->mem_id))
1274	mapping_flags |= mtype_local;
1275	/* local HBM region far from partition or remote XGMI GPU
1276	* with regular system scope coherence
1277	*/
1278	else if (svm_nodes_in_same_hive(node_a: bo_node, node_b: node) && !ext_coherent)
1279	mapping_flags |= AMDGPU_VM_MTYPE_NC;
1280	/* PCIe P2P on GPUs pre-9.5.0 */
1281	else if (gc_ip_version < IP_VERSION(9, 5, 0) &&
1282	!svm_nodes_in_same_hive(node_a: bo_node, node_b: node))
1283	mapping_flags |= AMDGPU_VM_MTYPE_UC;
1284	/* Other remote memory */
1285	else
1286	mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1287	/* system memory accessed by the APU */
1288	} else if (node->adev->flags & AMD_IS_APU) {
1289	/* On NUMA systems, locality is determined per-page
1290	* in amdgpu_gmc_override_vm_pte_flags
1291	*/
1292	if (num_possible_nodes() <= 1)
1293	mapping_flags |= mtype_local;
1294	else
1295	mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1296	/* system memory accessed by the dGPU */
1297	} else {
1298	if (gc_ip_version < IP_VERSION(9, 5, 0) || ext_coherent)
1299	mapping_flags |= AMDGPU_VM_MTYPE_UC;
1300	else
1301	mapping_flags |= AMDGPU_VM_MTYPE_NC;
1302	}
1303	break;
1304	case IP_VERSION(12, 0, 0):
1305	case IP_VERSION(12, 0, 1):
1306	mapping_flags |= AMDGPU_VM_MTYPE_NC;
1307	break;
1308	default:
1309	mapping_flags |= coherent ?
1310	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
1311	}
1312
1313	if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC)
1314	mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
1315
1316	pte_flags = AMDGPU_PTE_VALID;
1317	pte_flags |= (domain == SVM_RANGE_VRAM_DOMAIN) ? 0 : AMDGPU_PTE_SYSTEM;
1318	pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1319	if (gc_ip_version >= IP_VERSION(12, 0, 0))
1320	pte_flags |= AMDGPU_PTE_IS_PTE;
1321
1322	amdgpu_gmc_get_vm_pte(node->adev, vm, NULL, mapping_flags, &pte_flags);
1323	pte_flags |= AMDGPU_PTE_READABLE;
1324	if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
1325	pte_flags |= AMDGPU_PTE_WRITEABLE;
1326	return pte_flags;
1327	}
1328
1329	static int
1330	svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1331	uint64_t start, uint64_t last,
1332	struct dma_fence **fence)
1333	{
1334	uint64_t init_pte_value = 0;
1335
1336	pr_debug("[0x%llx 0x%llx]\n", start, last);
1337
1338	return amdgpu_vm_update_range(adev, vm, immediate: false, unlocked: true, flush_tlb: true, allow_override: false, NULL, start,
1339	last, flags: init_pte_value, offset: 0, vram_base: 0, NULL, NULL,
1340	fence);
1341	}
1342
1343	static int
1344	svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
1345	unsigned long last, uint32_t trigger)
1346	{
1347	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
1348	struct kfd_process_device *pdd;
1349	struct dma_fence *fence = NULL;
1350	struct kfd_process *p;
1351	uint32_t gpuidx;
1352	int r = 0;
1353
1354	if (!prange->mapped_to_gpu) {
1355	pr_debug("prange 0x%p [0x%lx 0x%lx] not mapped to GPU\n",
1356	prange, prange->start, prange->last);
1357	return 0;
1358	}
1359
1360	if (prange->start == start && prange->last == last) {
1361	pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange);
1362	prange->mapped_to_gpu = false;
1363	}
1364
1365	bitmap_or(dst: bitmap, src1: prange->bitmap_access, src2: prange->bitmap_aip,
1366	MAX_GPU_INSTANCE);
1367	p = container_of(prange->svms, struct kfd_process, svms);
1368
1369	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
1370	pr_debug("unmap from gpu idx 0x%x\n", gpuidx);
1371	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
1372	if (!pdd) {
1373	pr_debug("failed to find device idx %d\n", gpuidx);
1374	return -EINVAL;
1375	}
1376
1377	kfd_smi_event_unmap_from_gpu(node: pdd->dev, pid: p->lead_thread->pid,
1378	address: start, last, trigger);
1379
1380	r = svm_range_unmap_from_gpu(adev: pdd->dev->adev,
1381	drm_priv_to_vm(pdd->drm_priv),
1382	start, last, fence: &fence);
1383	if (r)
1384	break;
1385
1386	if (fence) {
1387	r = dma_fence_wait(fence, intr: false);
1388	dma_fence_put(fence);
1389	fence = NULL;
1390	if (r)
1391	break;
1392	}
1393	kfd_flush_tlb(pdd, type: TLB_FLUSH_HEAVYWEIGHT);
1394	}
1395
1396	return r;
1397	}
1398
1399	static int
1400	svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
1401	unsigned long offset, unsigned long npages, bool readonly,
1402	dma_addr_t *dma_addr, struct amdgpu_device *bo_adev,
1403	struct dma_fence **fence, bool flush_tlb)
1404	{
1405	struct amdgpu_device *adev = pdd->dev->adev;
1406	struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
1407	uint64_t pte_flags;
1408	unsigned long last_start;
1409	int last_domain;
1410	int r = 0;
1411	int64_t i, j;
1412
1413	last_start = prange->start + offset;
1414
1415	pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms,
1416	last_start, last_start + npages - 1, readonly);
1417
1418	for (i = offset; i < offset + npages; i++) {
1419	last_domain = dma_addr[i] & SVM_RANGE_VRAM_DOMAIN;
1420	dma_addr[i] &= ~SVM_RANGE_VRAM_DOMAIN;
1421
1422	/* Collect all pages in the same address range and memory domain
1423	* that can be mapped with a single call to update mapping.
1424	*/
1425	if (i < offset + npages - 1 &&
1426	last_domain == (dma_addr[i + 1] & SVM_RANGE_VRAM_DOMAIN))
1427	continue;
1428
1429	pr_debug("Mapping range [0x%lx 0x%llx] on domain: %s\n",
1430	last_start, prange->start + i, last_domain ? "GPU" : "CPU");
1431
1432	pte_flags = svm_range_get_pte_flags(node: pdd->dev, vm, prange, domain: last_domain);
1433	if (readonly)
1434	pte_flags &= ~AMDGPU_PTE_WRITEABLE;
1435
1436	pr_debug("svms 0x%p map [0x%lx 0x%llx] vram %d PTE 0x%llx\n",
1437	prange->svms, last_start, prange->start + i,
1438	(last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0,
1439	pte_flags);
1440
1441	/* For dGPU mode, we use same vm_manager to allocate VRAM for
1442	* different memory partition based on fpfn/lpfn, we should use
1443	* same vm_manager.vram_base_offset regardless memory partition.
1444	*/
1445	r = amdgpu_vm_update_range(adev, vm, immediate: false, unlocked: false, flush_tlb, allow_override: true,
1446	NULL, start: last_start, last: prange->start + i,
1447	flags: pte_flags,
1448	offset: (last_start - prange->start) << PAGE_SHIFT,
1449	vram_base: bo_adev ? bo_adev->vm_manager.vram_base_offset : 0,
1450	NULL, pages_addr: dma_addr, fence: &vm->last_update);
1451
1452	for (j = last_start - prange->start; j <= i; j++)
1453	dma_addr[j] |= last_domain;
1454
1455	if (r) {
1456	pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);
1457	goto out;
1458	}
1459	last_start = prange->start + i + 1;
1460	}
1461
1462	r = amdgpu_vm_update_pdes(adev, vm, immediate: false);
1463	if (r) {
1464	pr_debug("failed %d to update directories 0x%lx\n", r,
1465	prange->start);
1466	goto out;
1467	}
1468
1469	if (fence)
1470	*fence = dma_fence_get(fence: vm->last_update);
1471
1472	out:
1473	return r;
1474	}
1475
1476	static int
1477	svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset,
1478	unsigned long npages, bool readonly,
1479	unsigned long *bitmap, bool wait, bool flush_tlb)
1480	{
1481	struct kfd_process_device *pdd;
1482	struct amdgpu_device *bo_adev = NULL;
1483	struct kfd_process *p;
1484	struct dma_fence *fence = NULL;
1485	uint32_t gpuidx;
1486	int r = 0;
1487
1488	if (prange->svm_bo && prange->ttm_res)
1489	bo_adev = prange->svm_bo->node->adev;
1490
1491	p = container_of(prange->svms, struct kfd_process, svms);
1492	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
1493	pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
1494	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
1495	if (!pdd) {
1496	pr_debug("failed to find device idx %d\n", gpuidx);
1497	return -EINVAL;
1498	}
1499
1500	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
1501	if (IS_ERR(ptr: pdd))
1502	return -EINVAL;
1503
1504	if (bo_adev && pdd->dev->adev != bo_adev &&
1505	!amdgpu_xgmi_same_hive(adev: pdd->dev->adev, bo_adev)) {
1506	pr_debug("cannot map to device idx %d\n", gpuidx);
1507	continue;
1508	}
1509
1510	r = svm_range_map_to_gpu(pdd, prange, offset, npages, readonly,
1511	dma_addr: prange->dma_addr[gpuidx],
1512	bo_adev, fence: wait ? &fence : NULL,
1513	flush_tlb);
1514	if (r)
1515	break;
1516
1517	if (fence) {
1518	r = dma_fence_wait(fence, intr: false);
1519	dma_fence_put(fence);
1520	fence = NULL;
1521	if (r) {
1522	pr_debug("failed %d to dma fence wait\n", r);
1523	break;
1524	}
1525	}
1526
1527	kfd_flush_tlb(pdd, type: TLB_FLUSH_LEGACY);
1528	}
1529
1530	return r;
1531	}
1532
1533	struct svm_validate_context {
1534	struct kfd_process *process;
1535	struct svm_range *prange;
1536	bool intr;
1537	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
1538	struct drm_exec exec;
1539	};
1540
1541	static int svm_range_reserve_bos(struct svm_validate_context *ctx, bool intr)
1542	{
1543	struct kfd_process_device *pdd;
1544	struct amdgpu_vm *vm;
1545	uint32_t gpuidx;
1546	int r;
1547
1548	drm_exec_init(exec: &ctx->exec, flags: intr ? DRM_EXEC_INTERRUPTIBLE_WAIT: 0, nr: 0);
1549	drm_exec_until_all_locked(&ctx->exec) {
1550	for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {
1551	pdd = kfd_process_device_from_gpuidx(p: ctx->process, gpuidx);
1552	if (!pdd) {
1553	pr_debug("failed to find device idx %d\n", gpuidx);
1554	r = -EINVAL;
1555	goto unreserve_out;
1556	}
1557	vm = drm_priv_to_vm(pdd->drm_priv);
1558
1559	r = amdgpu_vm_lock_pd(vm, exec: &ctx->exec, num_fences: 2);
1560	drm_exec_retry_on_contention(&ctx->exec);
1561	if (unlikely(r)) {
1562	pr_debug("failed %d to reserve bo\n", r);
1563	goto unreserve_out;
1564	}
1565	}
1566	}
1567
1568	for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {
1569	pdd = kfd_process_device_from_gpuidx(p: ctx->process, gpuidx);
1570	if (!pdd) {
1571	pr_debug("failed to find device idx %d\n", gpuidx);
1572	r = -EINVAL;
1573	goto unreserve_out;
1574	}
1575
1576	r = amdgpu_vm_validate(adev: pdd->dev->adev,
1577	drm_priv_to_vm(pdd->drm_priv), NULL,
1578	callback: svm_range_bo_validate, NULL);
1579	if (r) {
1580	pr_debug("failed %d validate pt bos\n", r);
1581	goto unreserve_out;
1582	}
1583	}
1584
1585	return 0;
1586
1587	unreserve_out:
1588	drm_exec_fini(exec: &ctx->exec);
1589	return r;
1590	}
1591
1592	static void svm_range_unreserve_bos(struct svm_validate_context *ctx)
1593	{
1594	drm_exec_fini(exec: &ctx->exec);
1595	}
1596
1597	static void *kfd_svm_page_owner(struct kfd_process *p, int32_t gpuidx)
1598	{
1599	struct kfd_process_device *pdd;
1600
1601	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
1602	if (!pdd)
1603	return NULL;
1604
1605	return SVM_ADEV_PGMAP_OWNER(pdd->dev->adev);
1606	}
1607
1608	/*
1609	* Validation+GPU mapping with concurrent invalidation (MMU notifiers)
1610	*
1611	* To prevent concurrent destruction or change of range attributes, the
1612	* svm_read_lock must be held. The caller must not hold the svm_write_lock
1613	* because that would block concurrent evictions and lead to deadlocks. To
1614	* serialize concurrent migrations or validations of the same range, the
1615	* prange->migrate_mutex must be held.
1616	*
1617	* For VRAM ranges, the SVM BO must be allocated and valid (protected by its
1618	* eviction fence.
1619	*
1620	* The following sequence ensures race-free validation and GPU mapping:
1621	*
1622	* 1. Reserve page table (and SVM BO if range is in VRAM)
1623	* 2. hmm_range_fault to get page addresses (if system memory)
1624	* 3. DMA-map pages (if system memory)
1625	* 4-a. Take notifier lock
1626	* 4-b. Check that pages still valid (mmu_interval_read_retry)
1627	* 4-c. Check that the range was not split or otherwise invalidated
1628	* 4-d. Update GPU page table
1629	* 4.e. Release notifier lock
1630	* 5. Release page table (and SVM BO) reservation
1631	*/
1632	static int svm_range_validate_and_map(struct mm_struct *mm,
1633	unsigned long map_start, unsigned long map_last,
1634	struct svm_range *prange, int32_t gpuidx,
1635	bool intr, bool wait, bool flush_tlb)
1636	{
1637	struct svm_validate_context *ctx;
1638	unsigned long start, end, addr;
1639	struct kfd_process *p;
1640	void *owner;
1641	int32_t idx;
1642	int r = 0;
1643
1644	ctx = kzalloc(sizeof(struct svm_validate_context), GFP_KERNEL);
1645	if (!ctx)
1646	return -ENOMEM;
1647	ctx->process = container_of(prange->svms, struct kfd_process, svms);
1648	ctx->prange = prange;
1649	ctx->intr = intr;
1650
1651	if (gpuidx < MAX_GPU_INSTANCE) {
1652	bitmap_zero(dst: ctx->bitmap, MAX_GPU_INSTANCE);
1653	bitmap_set(map: ctx->bitmap, start: gpuidx, nbits: 1);
1654	} else if (ctx->process->xnack_enabled) {
1655	bitmap_copy(dst: ctx->bitmap, src: prange->bitmap_aip, MAX_GPU_INSTANCE);
1656
1657	/* If prefetch range to GPU, or GPU retry fault migrate range to
1658	* GPU, which has ACCESS attribute to the range, create mapping
1659	* on that GPU.
1660	*/
1661	if (prange->actual_loc) {
1662	gpuidx = kfd_process_gpuidx_from_gpuid(p: ctx->process,
1663	gpu_id: prange->actual_loc);
1664	if (gpuidx < 0) {
1665	WARN_ONCE(1, "failed get device by id 0x%x\n",
1666	prange->actual_loc);
1667	r = -EINVAL;
1668	goto free_ctx;
1669	}
1670	if (test_bit(gpuidx, prange->bitmap_access))
1671	bitmap_set(map: ctx->bitmap, start: gpuidx, nbits: 1);
1672	}
1673
1674	/*
1675	* If prange is already mapped or with always mapped flag,
1676	* update mapping on GPUs with ACCESS attribute
1677	*/
1678	if (bitmap_empty(src: ctx->bitmap, MAX_GPU_INSTANCE)) {
1679	if (prange->mapped_to_gpu ||
1680	prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)
1681	bitmap_copy(dst: ctx->bitmap, src: prange->bitmap_access, MAX_GPU_INSTANCE);
1682	}
1683	} else {
1684	bitmap_or(dst: ctx->bitmap, src1: prange->bitmap_access,
1685	src2: prange->bitmap_aip, MAX_GPU_INSTANCE);
1686	}
1687
1688	if (bitmap_empty(src: ctx->bitmap, MAX_GPU_INSTANCE)) {
1689	r = 0;
1690	goto free_ctx;
1691	}
1692
1693	if (prange->actual_loc && !prange->ttm_res) {
1694	/* This should never happen. actual_loc gets set by
1695	* svm_migrate_ram_to_vram after allocating a BO.
1696	*/
1697	WARN_ONCE(1, "VRAM BO missing during validation\n");
1698	r = -EINVAL;
1699	goto free_ctx;
1700	}
1701
1702	r = svm_range_reserve_bos(ctx, intr);
1703	if (r)
1704	goto free_ctx;
1705
1706	p = container_of(prange->svms, struct kfd_process, svms);
1707	owner = kfd_svm_page_owner(p, gpuidx: find_first_bit(addr: ctx->bitmap,
1708	MAX_GPU_INSTANCE));
1709	for_each_set_bit(idx, ctx->bitmap, MAX_GPU_INSTANCE) {
1710	if (kfd_svm_page_owner(p, gpuidx: idx) != owner) {
1711	owner = NULL;
1712	break;
1713	}
1714	}
1715
1716	start = map_start << PAGE_SHIFT;
1717	end = (map_last + 1) << PAGE_SHIFT;
1718	for (addr = start; !r && addr < end; ) {
1719	struct amdgpu_hmm_range *range = NULL;
1720	unsigned long map_start_vma;
1721	unsigned long map_last_vma;
1722	struct vm_area_struct *vma;
1723	unsigned long next = 0;
1724	unsigned long offset;
1725	unsigned long npages;
1726	bool readonly;
1727
1728	vma = vma_lookup(mm, addr);
1729	if (vma) {
1730	readonly = !(vma->vm_flags & VM_WRITE);
1731
1732	next = min(vma->vm_end, end);
1733	npages = (next - addr) >> PAGE_SHIFT;
1734	/* HMM requires at least READ permissions. If provided with PROT_NONE,
1735	* unmap the memory. If it's not already mapped, this is a no-op
1736	* If PROT_WRITE is provided without READ, warn first then unmap
1737	*/
1738	if (!(vma->vm_flags & VM_READ)) {
1739	unsigned long e, s;
1740
1741	svm_range_lock(prange);
1742	if (vma->vm_flags & VM_WRITE)
1743	pr_debug("VM_WRITE without VM_READ is not supported");
1744	s = max(start, prange->start);
1745	e = min(end, prange->last);
1746	if (e >= s)
1747	r = svm_range_unmap_from_gpus(prange, start: s, last: e,
1748	trigger: KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU);
1749	svm_range_unlock(prange);
1750	/* If unmap returns non-zero, we'll bail on the next for loop
1751	* iteration, so just leave r and continue
1752	*/
1753	addr = next;
1754	continue;
1755	}
1756
1757	WRITE_ONCE(p->svms.faulting_task, current);
1758	range = amdgpu_hmm_range_alloc(NULL);
1759	if (likely(range))
1760	r = amdgpu_hmm_range_get_pages(notifier: &prange->notifier, start: addr, npages,
1761	readonly, owner, range);
1762	else
1763	r = -ENOMEM;
1764	WRITE_ONCE(p->svms.faulting_task, NULL);
1765	if (r)
1766	pr_debug("failed %d to get svm range pages\n", r);
1767	} else {
1768	r = -EFAULT;
1769	}
1770
1771	if (!r) {
1772	offset = (addr >> PAGE_SHIFT) - prange->start;
1773	r = svm_range_dma_map(prange, bitmap: ctx->bitmap, offset, npages,
1774	hmm_pfns: range->hmm_range.hmm_pfns);
1775	if (r)
1776	pr_debug("failed %d to dma map range\n", r);
1777	}
1778
1779	svm_range_lock(prange);
1780
1781	/* Free backing memory of hmm_range if it was initialized
1782	* Override return value to TRY AGAIN only if prior returns
1783	* were successful
1784	*/
1785	if (range && !amdgpu_hmm_range_valid(range) && !r) {
1786	pr_debug("hmm update the range, need validate again\n");
1787	r = -EAGAIN;
1788	}
1789
1790	/* Free the hmm range */
1791	amdgpu_hmm_range_free(range);
1792
1793	if (!r && !list_empty(head: &prange->child_list)) {
1794	pr_debug("range split by unmap in parallel, validate again\n");
1795	r = -EAGAIN;
1796	}
1797
1798	if (!r) {
1799	map_start_vma = max(map_start, prange->start + offset);
1800	map_last_vma = min(map_last, prange->start + offset + npages - 1);
1801	if (map_start_vma <= map_last_vma) {
1802	offset = map_start_vma - prange->start;
1803	npages = map_last_vma - map_start_vma + 1;
1804	r = svm_range_map_to_gpus(prange, offset, npages, readonly,
1805	bitmap: ctx->bitmap, wait, flush_tlb);
1806	}
1807	}
1808
1809	if (!r && next == end)
1810	prange->mapped_to_gpu = true;
1811
1812	svm_range_unlock(prange);
1813
1814	addr = next;
1815	}
1816
1817	svm_range_unreserve_bos(ctx);
1818	if (!r)
1819	prange->validate_timestamp = ktime_get_boottime();
1820
1821	free_ctx:
1822	kfree(objp: ctx);
1823
1824	return r;
1825	}
1826
1827	/**
1828	* svm_range_list_lock_and_flush_work - flush pending deferred work
1829	*
1830	* @svms: the svm range list
1831	* @mm: the mm structure
1832	*
1833	* Context: Returns with mmap write lock held, pending deferred work flushed
1834	*
1835	*/
1836	void
1837	svm_range_list_lock_and_flush_work(struct svm_range_list *svms,
1838	struct mm_struct *mm)
1839	{
1840	retry_flush_work:
1841	flush_work(work: &svms->deferred_list_work);
1842	mmap_write_lock(mm);
1843
1844	if (list_empty(head: &svms->deferred_range_list))
1845	return;
1846	mmap_write_unlock(mm);
1847	pr_debug("retry flush\n");
1848	goto retry_flush_work;
1849	}
1850
1851	static void svm_range_restore_work(struct work_struct *work)
1852	{
1853	struct delayed_work *dwork = to_delayed_work(work);
1854	struct amdkfd_process_info *process_info;
1855	struct svm_range_list *svms;
1856	struct svm_range *prange;
1857	struct kfd_process *p;
1858	struct mm_struct *mm;
1859	int evicted_ranges;
1860	int invalid;
1861	int r;
1862
1863	svms = container_of(dwork, struct svm_range_list, restore_work);
1864	evicted_ranges = atomic_read(v: &svms->evicted_ranges);
1865	if (!evicted_ranges)
1866	return;
1867
1868	pr_debug("restore svm ranges\n");
1869
1870	p = container_of(svms, struct kfd_process, svms);
1871	process_info = p->kgd_process_info;
1872
1873	/* Keep mm reference when svm_range_validate_and_map ranges */
1874	mm = get_task_mm(task: p->lead_thread);
1875	if (!mm) {
1876	pr_debug("svms 0x%p process mm gone\n", svms);
1877	return;
1878	}
1879
1880	mutex_lock(&process_info->lock);
1881	svm_range_list_lock_and_flush_work(svms, mm);
1882	mutex_lock(&svms->lock);
1883
1884	evicted_ranges = atomic_read(v: &svms->evicted_ranges);
1885
1886	list_for_each_entry(prange, &svms->list, list) {
1887	invalid = atomic_read(v: &prange->invalid);
1888	if (!invalid)
1889	continue;
1890
1891	pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n",
1892	prange->svms, prange, prange->start, prange->last,
1893	invalid);
1894
1895	/*
1896	* If range is migrating, wait for migration is done.
1897	*/
1898	mutex_lock(&prange->migrate_mutex);
1899
1900	r = svm_range_validate_and_map(mm, map_start: prange->start, map_last: prange->last, prange,
1901	MAX_GPU_INSTANCE, intr: false, wait: true, flush_tlb: false);
1902	if (r)
1903	pr_debug("failed %d to map 0x%lx to gpus\n", r,
1904	prange->start);
1905
1906	mutex_unlock(lock: &prange->migrate_mutex);
1907	if (r)
1908	goto out_reschedule;
1909
1910	if (atomic_cmpxchg(v: &prange->invalid, old: invalid, new: 0) != invalid)
1911	goto out_reschedule;
1912	}
1913
1914	if (atomic_cmpxchg(v: &svms->evicted_ranges, old: evicted_ranges, new: 0) !=
1915	evicted_ranges)
1916	goto out_reschedule;
1917
1918	evicted_ranges = 0;
1919
1920	r = kgd2kfd_resume_mm(mm);
1921	if (r) {
1922	/* No recovery from this failure. Probably the CP is
1923	* hanging. No point trying again.
1924	*/
1925	pr_debug("failed %d to resume KFD\n", r);
1926	}
1927
1928	pr_debug("restore svm ranges successfully\n");
1929
1930	out_reschedule:
1931	mutex_unlock(lock: &svms->lock);
1932	mmap_write_unlock(mm);
1933	mutex_unlock(lock: &process_info->lock);
1934
1935	/* If validation failed, reschedule another attempt */
1936	if (evicted_ranges) {
1937	pr_debug("reschedule to restore svm range\n");
1938	queue_delayed_work(wq: system_freezable_wq, dwork: &svms->restore_work,
1939	delay: msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
1940
1941	kfd_smi_event_queue_restore_rescheduled(mm);
1942	}
1943	mmput(mm);
1944	}
1945
1946	/**
1947	* svm_range_evict - evict svm range
1948	* @prange: svm range structure
1949	* @mm: current process mm_struct
1950	* @start: starting process queue number
1951	* @last: last process queue number
1952	* @event: mmu notifier event when range is evicted or migrated
1953	*
1954	* Stop all queues of the process to ensure GPU doesn't access the memory, then
1955	* return to let CPU evict the buffer and proceed CPU pagetable update.
1956	*
1957	* Don't need use lock to sync cpu pagetable invalidation with GPU execution.
1958	* If invalidation happens while restore work is running, restore work will
1959	* restart to ensure to get the latest CPU pages mapping to GPU, then start
1960	* the queues.
1961	*/
1962	static int
1963	svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
1964	unsigned long start, unsigned long last,
1965	enum mmu_notifier_event event)
1966	{
1967	struct svm_range_list *svms = prange->svms;
1968	struct svm_range *pchild;
1969	struct kfd_process *p;
1970	int r = 0;
1971
1972	p = container_of(svms, struct kfd_process, svms);
1973
1974	pr_debug("invalidate svms 0x%p prange [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
1975	svms, prange->start, prange->last, start, last);
1976
1977	if (!p->xnack_enabled ||
1978	(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) {
1979	int evicted_ranges;
1980	bool mapped = prange->mapped_to_gpu;
1981
1982	list_for_each_entry(pchild, &prange->child_list, child_list) {
1983	if (!pchild->mapped_to_gpu)
1984	continue;
1985	mapped = true;
1986	mutex_lock_nested(lock: &pchild->lock, subclass: 1);
1987	if (pchild->start <= last && pchild->last >= start) {
1988	pr_debug("increment pchild invalid [0x%lx 0x%lx]\n",
1989	pchild->start, pchild->last);
1990	atomic_inc(v: &pchild->invalid);
1991	}
1992	mutex_unlock(lock: &pchild->lock);
1993	}
1994
1995	if (!mapped)
1996	return r;
1997
1998	if (prange->start <= last && prange->last >= start)
1999	atomic_inc(v: &prange->invalid);
2000
2001	evicted_ranges = atomic_inc_return(v: &svms->evicted_ranges);
2002	if (evicted_ranges != 1)
2003	return r;
2004
2005	pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",
2006	prange->svms, prange->start, prange->last);
2007
2008	/* First eviction, stop the queues */
2009	r = kgd2kfd_quiesce_mm(mm, trigger: KFD_QUEUE_EVICTION_TRIGGER_SVM);
2010	if (r)
2011	pr_debug("failed to quiesce KFD\n");
2012
2013	pr_debug("schedule to restore svm %p ranges\n", svms);
2014	queue_delayed_work(wq: system_freezable_wq, dwork: &svms->restore_work,
2015	delay: msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
2016	} else {
2017	unsigned long s, l;
2018	uint32_t trigger;
2019
2020	if (event == MMU_NOTIFY_MIGRATE)
2021	trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE;
2022	else
2023	trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY;
2024
2025	pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
2026	prange->svms, start, last);
2027	list_for_each_entry(pchild, &prange->child_list, child_list) {
2028	mutex_lock_nested(lock: &pchild->lock, subclass: 1);
2029	s = max(start, pchild->start);
2030	l = min(last, pchild->last);
2031	if (l >= s)
2032	svm_range_unmap_from_gpus(prange: pchild, start: s, last: l, trigger);
2033	mutex_unlock(lock: &pchild->lock);
2034	}
2035	s = max(start, prange->start);
2036	l = min(last, prange->last);
2037	if (l >= s)
2038	svm_range_unmap_from_gpus(prange, start: s, last: l, trigger);
2039	}
2040
2041	return r;
2042	}
2043
2044	static struct svm_range *svm_range_clone(struct svm_range *old)
2045	{
2046	struct svm_range *new;
2047
2048	new = svm_range_new(svms: old->svms, start: old->start, last: old->last, update_mem_usage: false);
2049	if (!new)
2050	return NULL;
2051	if (svm_range_copy_dma_addrs(dst: new, src: old)) {
2052	svm_range_free(prange: new, do_unmap: false);
2053	return NULL;
2054	}
2055	if (old->svm_bo) {
2056	new->ttm_res = old->ttm_res;
2057	new->offset = old->offset;
2058	new->svm_bo = svm_range_bo_ref(svm_bo: old->svm_bo);
2059	spin_lock(lock: &new->svm_bo->list_lock);
2060	list_add(new: &new->svm_bo_list, head: &new->svm_bo->range_list);
2061	spin_unlock(lock: &new->svm_bo->list_lock);
2062	}
2063	new->flags = old->flags;
2064	new->preferred_loc = old->preferred_loc;
2065	new->prefetch_loc = old->prefetch_loc;
2066	new->actual_loc = old->actual_loc;
2067	new->granularity = old->granularity;
2068	new->mapped_to_gpu = old->mapped_to_gpu;
2069	new->vram_pages = old->vram_pages;
2070	bitmap_copy(dst: new->bitmap_access, src: old->bitmap_access, MAX_GPU_INSTANCE);
2071	bitmap_copy(dst: new->bitmap_aip, src: old->bitmap_aip, MAX_GPU_INSTANCE);
2072	atomic_set(v: &new->queue_refcount, i: atomic_read(v: &old->queue_refcount));
2073
2074	return new;
2075	}
2076
2077	void svm_range_set_max_pages(struct amdgpu_device *adev)
2078	{
2079	uint64_t max_pages;
2080	uint64_t pages, _pages;
2081	uint64_t min_pages = 0;
2082	int i, id;
2083
2084	for (i = 0; i < adev->kfd.dev->num_nodes; i++) {
2085	if (adev->kfd.dev->nodes[i]->xcp)
2086	id = adev->kfd.dev->nodes[i]->xcp->id;
2087	else
2088	id = -1;
2089	pages = KFD_XCP_MEMORY_SIZE(adev, id) >> 17;
2090	pages = clamp(pages, 1ULL << 9, 1ULL << 18);
2091	pages = rounddown_pow_of_two(pages);
2092	min_pages = min_not_zero(min_pages, pages);
2093	}
2094
2095	do {
2096	max_pages = READ_ONCE(max_svm_range_pages);
2097	_pages = min_not_zero(max_pages, min_pages);
2098	} while (cmpxchg(&max_svm_range_pages, max_pages, _pages) != max_pages);
2099	}
2100
2101	static int
2102	svm_range_split_new(struct svm_range_list *svms, uint64_t start, uint64_t last,
2103	uint64_t max_pages, struct list_head *insert_list,
2104	struct list_head *update_list)
2105	{
2106	struct svm_range *prange;
2107	uint64_t l;
2108
2109	pr_debug("max_svm_range_pages 0x%llx adding [0x%llx 0x%llx]\n",
2110	max_pages, start, last);
2111
2112	while (last >= start) {
2113	l = min(last, ALIGN_DOWN(start + max_pages, max_pages) - 1);
2114
2115	prange = svm_range_new(svms, start, last: l, update_mem_usage: true);
2116	if (!prange)
2117	return -ENOMEM;
2118	list_add(new: &prange->list, head: insert_list);
2119	list_add(new: &prange->update_list, head: update_list);
2120
2121	start = l + 1;
2122	}
2123	return 0;
2124	}
2125
2126	/**
2127	* svm_range_add - add svm range and handle overlap
2128	* @p: the range add to this process svms
2129	* @start: page size aligned
2130	* @size: page size aligned
2131	* @nattr: number of attributes
2132	* @attrs: array of attributes
2133	* @update_list: output, the ranges need validate and update GPU mapping
2134	* @insert_list: output, the ranges need insert to svms
2135	* @remove_list: output, the ranges are replaced and need remove from svms
2136	* @remap_list: output, remap unaligned svm ranges
2137	*
2138	* Check if the virtual address range has overlap with any existing ranges,
2139	* split partly overlapping ranges and add new ranges in the gaps. All changes
2140	* should be applied to the range_list and interval tree transactionally. If
2141	* any range split or allocation fails, the entire update fails. Therefore any
2142	* existing overlapping svm_ranges are cloned and the original svm_ranges left
2143	* unchanged.
2144	*
2145	* If the transaction succeeds, the caller can update and insert clones and
2146	* new ranges, then free the originals.
2147	*
2148	* Otherwise the caller can free the clones and new ranges, while the old
2149	* svm_ranges remain unchanged.
2150	*
2151	* Context: Process context, caller must hold svms->lock
2152	*
2153	* Return:
2154	* 0 - OK, otherwise error code
2155	*/
2156	static int
2157	svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
2158	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
2159	struct list_head *update_list, struct list_head *insert_list,
2160	struct list_head *remove_list, struct list_head *remap_list)
2161	{
2162	unsigned long last = start + size - 1UL;
2163	struct svm_range_list *svms = &p->svms;
2164	struct interval_tree_node *node;
2165	struct svm_range *prange;
2166	struct svm_range *tmp;
2167	struct list_head new_list;
2168	int r = 0;
2169
2170	pr_debug("svms 0x%p [0x%llx 0x%lx]\n", &p->svms, start, last);
2171
2172	INIT_LIST_HEAD(list: update_list);
2173	INIT_LIST_HEAD(list: insert_list);
2174	INIT_LIST_HEAD(list: remove_list);
2175	INIT_LIST_HEAD(list: &new_list);
2176	INIT_LIST_HEAD(list: remap_list);
2177
2178	node = interval_tree_iter_first(root: &svms->objects, start, last);
2179	while (node) {
2180	struct interval_tree_node *next;
2181	unsigned long next_start;
2182
2183	pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start,
2184	node->last);
2185
2186	prange = container_of(node, struct svm_range, it_node);
2187	next = interval_tree_iter_next(node, start, last);
2188	next_start = min(node->last, last) + 1;
2189
2190	if (svm_range_is_same_attrs(p, prange, nattr, attrs) &&
2191	prange->mapped_to_gpu) {
2192	/* nothing to do */
2193	} else if (node->start < start || node->last > last) {
2194	/* node intersects the update range and its attributes
2195	* will change. Clone and split it, apply updates only
2196	* to the overlapping part
2197	*/
2198	struct svm_range *old = prange;
2199
2200	prange = svm_range_clone(old);
2201	if (!prange) {
2202	r = -ENOMEM;
2203	goto out;
2204	}
2205
2206	list_add(new: &old->update_list, head: remove_list);
2207	list_add(new: &prange->list, head: insert_list);
2208	list_add(new: &prange->update_list, head: update_list);
2209
2210	if (node->start < start) {
2211	pr_debug("change old range start\n");
2212	r = svm_range_split_head(prange, new_start: start,
2213	insert_list, remap_list);
2214	if (r)
2215	goto out;
2216	}
2217	if (node->last > last) {
2218	pr_debug("change old range last\n");
2219	r = svm_range_split_tail(prange, new_last: last,
2220	insert_list, remap_list);
2221	if (r)
2222	goto out;
2223	}
2224	} else {
2225	/* The node is contained within start..last,
2226	* just update it
2227	*/
2228	list_add(new: &prange->update_list, head: update_list);
2229	}
2230
2231	/* insert a new node if needed */
2232	if (node->start > start) {
2233	r = svm_range_split_new(svms, start, last: node->start - 1,
2234	READ_ONCE(max_svm_range_pages),
2235	insert_list: &new_list, update_list);
2236	if (r)
2237	goto out;
2238	}
2239
2240	node = next;
2241	start = next_start;
2242	}
2243
2244	/* add a final range at the end if needed */
2245	if (start <= last)
2246	r = svm_range_split_new(svms, start, last,
2247	READ_ONCE(max_svm_range_pages),
2248	insert_list: &new_list, update_list);
2249
2250	out:
2251	if (r) {
2252	list_for_each_entry_safe(prange, tmp, insert_list, list)
2253	svm_range_free(prange, do_unmap: false);
2254	list_for_each_entry_safe(prange, tmp, &new_list, list)
2255	svm_range_free(prange, do_unmap: true);
2256	} else {
2257	list_splice(list: &new_list, head: insert_list);
2258	}
2259
2260	return r;
2261	}
2262
2263	static void
2264	svm_range_update_notifier_and_interval_tree(struct mm_struct *mm,
2265	struct svm_range *prange)
2266	{
2267	unsigned long start;
2268	unsigned long last;
2269
2270	start = prange->notifier.interval_tree.start >> PAGE_SHIFT;
2271	last = prange->notifier.interval_tree.last >> PAGE_SHIFT;
2272
2273	if (prange->start == start && prange->last == last)
2274	return;
2275
2276	pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
2277	prange->svms, prange, start, last, prange->start,
2278	prange->last);
2279
2280	if (start != 0 && last != 0) {
2281	interval_tree_remove(node: &prange->it_node, root: &prange->svms->objects);
2282	svm_range_remove_notifier(prange);
2283	}
2284	prange->it_node.start = prange->start;
2285	prange->it_node.last = prange->last;
2286
2287	interval_tree_insert(node: &prange->it_node, root: &prange->svms->objects);
2288	svm_range_add_notifier_locked(mm, prange);
2289	}
2290
2291	static void
2292	svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange,
2293	struct mm_struct *mm)
2294	{
2295	switch (prange->work_item.op) {
2296	case SVM_OP_NULL:
2297	pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2298	svms, prange, prange->start, prange->last);
2299	break;
2300	case SVM_OP_UNMAP_RANGE:
2301	pr_debug("remove 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2302	svms, prange, prange->start, prange->last);
2303	svm_range_unlink(prange);
2304	svm_range_remove_notifier(prange);
2305	svm_range_free(prange, do_unmap: true);
2306	break;
2307	case SVM_OP_UPDATE_RANGE_NOTIFIER:
2308	pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2309	svms, prange, prange->start, prange->last);
2310	svm_range_update_notifier_and_interval_tree(mm, prange);
2311	break;
2312	case SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP:
2313	pr_debug("update and map 0x%p prange 0x%p [0x%lx 0x%lx]\n",
2314	svms, prange, prange->start, prange->last);
2315	svm_range_update_notifier_and_interval_tree(mm, prange);
2316	/* TODO: implement deferred validation and mapping */
2317	break;
2318	case SVM_OP_ADD_RANGE:
2319	pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange,
2320	prange->start, prange->last);
2321	svm_range_add_to_svms(prange);
2322	svm_range_add_notifier_locked(mm, prange);
2323	break;
2324	case SVM_OP_ADD_RANGE_AND_MAP:
2325	pr_debug("add and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms,
2326	prange, prange->start, prange->last);
2327	svm_range_add_to_svms(prange);
2328	svm_range_add_notifier_locked(mm, prange);
2329	/* TODO: implement deferred validation and mapping */
2330	break;
2331	default:
2332	WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange,
2333	prange->work_item.op);
2334	}
2335	}
2336
2337	static void svm_range_drain_retry_fault(struct svm_range_list *svms)
2338	{
2339	struct kfd_process_device *pdd;
2340	struct kfd_process *p;
2341	uint32_t i;
2342
2343	p = container_of(svms, struct kfd_process, svms);
2344
2345	for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) {
2346	pdd = p->pdds[i];
2347	if (!pdd)
2348	continue;
2349
2350	pr_debug("drain retry fault gpu %d svms %p\n", i, svms);
2351
2352	amdgpu_ih_wait_on_checkpoint_process_ts(adev: pdd->dev->adev,
2353	ih: pdd->dev->adev->irq.retry_cam_enabled ?
2354	&pdd->dev->adev->irq.ih :
2355	&pdd->dev->adev->irq.ih1);
2356
2357	if (pdd->dev->adev->irq.retry_cam_enabled)
2358	amdgpu_ih_wait_on_checkpoint_process_ts(adev: pdd->dev->adev,
2359	ih: &pdd->dev->adev->irq.ih_soft);
2360
2361
2362	pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);
2363	}
2364	}
2365
2366	static void svm_range_deferred_list_work(struct work_struct *work)
2367	{
2368	struct svm_range_list *svms;
2369	struct svm_range *prange;
2370	struct mm_struct *mm;
2371
2372	svms = container_of(work, struct svm_range_list, deferred_list_work);
2373	pr_debug("enter svms 0x%p\n", svms);
2374
2375	spin_lock(lock: &svms->deferred_list_lock);
2376	while (!list_empty(head: &svms->deferred_range_list)) {
2377	prange = list_first_entry(&svms->deferred_range_list,
2378	struct svm_range, deferred_list);
2379	spin_unlock(lock: &svms->deferred_list_lock);
2380
2381	pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,
2382	prange->start, prange->last, prange->work_item.op);
2383
2384	mm = prange->work_item.mm;
2385
2386	mmap_write_lock(mm);
2387
2388	/* Remove from deferred_list must be inside mmap write lock, for
2389	* two race cases:
2390	* 1. unmap_from_cpu may change work_item.op and add the range
2391	* to deferred_list again, cause use after free bug.
2392	* 2. svm_range_list_lock_and_flush_work may hold mmap write
2393	* lock and continue because deferred_list is empty, but
2394	* deferred_list work is actually waiting for mmap lock.
2395	*/
2396	spin_lock(lock: &svms->deferred_list_lock);
2397	list_del_init(entry: &prange->deferred_list);
2398	spin_unlock(lock: &svms->deferred_list_lock);
2399
2400	mutex_lock(&svms->lock);
2401	mutex_lock(&prange->migrate_mutex);
2402	while (!list_empty(head: &prange->child_list)) {
2403	struct svm_range *pchild;
2404
2405	pchild = list_first_entry(&prange->child_list,
2406	struct svm_range, child_list);
2407	pr_debug("child prange 0x%p op %d\n", pchild,
2408	pchild->work_item.op);
2409	list_del_init(entry: &pchild->child_list);
2410	svm_range_handle_list_op(svms, prange: pchild, mm);
2411	}
2412	mutex_unlock(lock: &prange->migrate_mutex);
2413
2414	svm_range_handle_list_op(svms, prange, mm);
2415	mutex_unlock(lock: &svms->lock);
2416	mmap_write_unlock(mm);
2417
2418	/* Pairs with mmget in svm_range_add_list_work. If dropping the
2419	* last mm refcount, schedule release work to avoid circular locking
2420	*/
2421	mmput_async(mm);
2422
2423	spin_lock(lock: &svms->deferred_list_lock);
2424	}
2425	spin_unlock(lock: &svms->deferred_list_lock);
2426	pr_debug("exit svms 0x%p\n", svms);
2427	}
2428
2429	void
2430	svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange,
2431	struct mm_struct *mm, enum svm_work_list_ops op)
2432	{
2433	spin_lock(lock: &svms->deferred_list_lock);
2434	/* if prange is on the deferred list */
2435	if (!list_empty(head: &prange->deferred_list)) {
2436	pr_debug("update exist prange 0x%p work op %d\n", prange, op);
2437	WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n");
2438	if (op != SVM_OP_NULL &&
2439	prange->work_item.op != SVM_OP_UNMAP_RANGE)
2440	prange->work_item.op = op;
2441	} else {
2442	/* Pairs with mmput in deferred_list_work.
2443	* If process is exiting and mm is gone, don't update mmu notifier.
2444	*/
2445	if (mmget_not_zero(mm)) {
2446	prange->work_item.mm = mm;
2447	prange->work_item.op = op;
2448	list_add_tail(new: &prange->deferred_list,
2449	head: &prange->svms->deferred_range_list);
2450	pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n",
2451	prange, prange->start, prange->last, op);
2452	}
2453	}
2454	spin_unlock(lock: &svms->deferred_list_lock);
2455	}
2456
2457	void schedule_deferred_list_work(struct svm_range_list *svms)
2458	{
2459	spin_lock(lock: &svms->deferred_list_lock);
2460	if (!list_empty(head: &svms->deferred_range_list))
2461	schedule_work(work: &svms->deferred_list_work);
2462	spin_unlock(lock: &svms->deferred_list_lock);
2463	}
2464
2465	static void
2466	svm_range_unmap_split(struct svm_range *parent, struct svm_range *prange, unsigned long start,
2467	unsigned long last)
2468	{
2469	struct svm_range *head;
2470	struct svm_range *tail;
2471
2472	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
2473	pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange,
2474	prange->start, prange->last);
2475	return;
2476	}
2477	if (start > prange->last || last < prange->start)
2478	return;
2479
2480	head = tail = prange;
2481	if (start > prange->start)
2482	svm_range_split(prange, start: prange->start, last: start - 1, new: &tail);
2483	if (last < tail->last)
2484	svm_range_split(prange: tail, start: last + 1, last: tail->last, new: &head);
2485
2486	if (head != prange && tail != prange) {
2487	svm_range_add_child(prange: parent, pchild: head, op: SVM_OP_UNMAP_RANGE);
2488	svm_range_add_child(prange: parent, pchild: tail, op: SVM_OP_ADD_RANGE);
2489	} else if (tail != prange) {
2490	svm_range_add_child(prange: parent, pchild: tail, op: SVM_OP_UNMAP_RANGE);
2491	} else if (head != prange) {
2492	svm_range_add_child(prange: parent, pchild: head, op: SVM_OP_UNMAP_RANGE);
2493	} else if (parent != prange) {
2494	prange->work_item.op = SVM_OP_UNMAP_RANGE;
2495	}
2496	}
2497
2498	static void
2499	svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
2500	unsigned long start, unsigned long last)
2501	{
2502	uint32_t trigger = KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU;
2503	struct svm_range_list *svms;
2504	struct svm_range *pchild;
2505	struct kfd_process *p;
2506	unsigned long s, l;
2507	bool unmap_parent;
2508	uint32_t i;
2509
2510	if (atomic_read(v: &prange->queue_refcount)) {
2511	int r;
2512
2513	pr_warn("Freeing queue vital buffer 0x%lx, queue evicted\n",
2514	prange->start << PAGE_SHIFT);
2515	r = kgd2kfd_quiesce_mm(mm, trigger: KFD_QUEUE_EVICTION_TRIGGER_SVM);
2516	if (r)
2517	pr_debug("failed %d to quiesce KFD queues\n", r);
2518	}
2519
2520	p = kfd_lookup_process_by_mm(mm);
2521	if (!p)
2522	return;
2523	svms = &p->svms;
2524
2525	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms,
2526	prange, prange->start, prange->last, start, last);
2527
2528	/* calculate time stamps that are used to decide which page faults need be
2529	* dropped or handled before unmap pages from gpu vm
2530	*/
2531	for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) {
2532	struct kfd_process_device *pdd;
2533	struct amdgpu_device *adev;
2534	struct amdgpu_ih_ring *ih;
2535	uint32_t checkpoint_wptr;
2536
2537	pdd = p->pdds[i];
2538	if (!pdd)
2539	continue;
2540
2541	adev = pdd->dev->adev;
2542
2543	/* Check and drain ih1 ring if cam not available */
2544	if (adev->irq.ih1.ring_size) {
2545	ih = &adev->irq.ih1;
2546	checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);
2547	if (ih->rptr != checkpoint_wptr) {
2548	svms->checkpoint_ts[i] =
2549	amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1);
2550	continue;
2551	}
2552	}
2553
2554	/* check if dev->irq.ih_soft is not empty */
2555	ih = &adev->irq.ih_soft;
2556	checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);
2557	if (ih->rptr != checkpoint_wptr)
2558	svms->checkpoint_ts[i] = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1);
2559	}
2560
2561	unmap_parent = start <= prange->start && last >= prange->last;
2562
2563	list_for_each_entry(pchild, &prange->child_list, child_list) {
2564	mutex_lock_nested(lock: &pchild->lock, subclass: 1);
2565	s = max(start, pchild->start);
2566	l = min(last, pchild->last);
2567	if (l >= s)
2568	svm_range_unmap_from_gpus(prange: pchild, start: s, last: l, trigger);
2569	svm_range_unmap_split(parent: prange, prange: pchild, start, last);
2570	mutex_unlock(lock: &pchild->lock);
2571	}
2572	s = max(start, prange->start);
2573	l = min(last, prange->last);
2574	if (l >= s)
2575	svm_range_unmap_from_gpus(prange, start: s, last: l, trigger);
2576	svm_range_unmap_split(parent: prange, prange, start, last);
2577
2578	if (unmap_parent)
2579	svm_range_add_list_work(svms, prange, mm, op: SVM_OP_UNMAP_RANGE);
2580	else
2581	svm_range_add_list_work(svms, prange, mm,
2582	op: SVM_OP_UPDATE_RANGE_NOTIFIER);
2583	schedule_deferred_list_work(svms);
2584
2585	kfd_unref_process(p);
2586	}
2587
2588	/**
2589	* svm_range_cpu_invalidate_pagetables - interval notifier callback
2590	* @mni: mmu_interval_notifier struct
2591	* @range: mmu_notifier_range struct
2592	* @cur_seq: value to pass to mmu_interval_set_seq()
2593	*
2594	* If event is MMU_NOTIFY_UNMAP, this is from CPU unmap range, otherwise, it
2595	* is from migration, or CPU page invalidation callback.
2596	*
2597	* For unmap event, unmap range from GPUs, remove prange from svms in a delayed
2598	* work thread, and split prange if only part of prange is unmapped.
2599	*
2600	* For invalidation event, if GPU retry fault is not enabled, evict the queues,
2601	* then schedule svm_range_restore_work to update GPU mapping and resume queues.
2602	* If GPU retry fault is enabled, unmap the svm range from GPU, retry fault will
2603	* update GPU mapping to recover.
2604	*
2605	* Context: mmap lock, notifier_invalidate_start lock are held
2606	* for invalidate event, prange lock is held if this is from migration
2607	*/
2608	static bool
2609	svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
2610	const struct mmu_notifier_range *range,
2611	unsigned long cur_seq)
2612	{
2613	struct svm_range *prange;
2614	unsigned long start;
2615	unsigned long last;
2616
2617	if (range->event == MMU_NOTIFY_RELEASE)
2618	return true;
2619
2620	start = mni->interval_tree.start;
2621	last = mni->interval_tree.last;
2622	start = max(start, range->start) >> PAGE_SHIFT;
2623	last = min(last, range->end - 1) >> PAGE_SHIFT;
2624	pr_debug("[0x%lx 0x%lx] range[0x%lx 0x%lx] notifier[0x%lx 0x%lx] %d\n",
2625	start, last, range->start >> PAGE_SHIFT,
2626	(range->end - 1) >> PAGE_SHIFT,
2627	mni->interval_tree.start >> PAGE_SHIFT,
2628	mni->interval_tree.last >> PAGE_SHIFT, range->event);
2629
2630	prange = container_of(mni, struct svm_range, notifier);
2631
2632	svm_range_lock(prange);
2633	mmu_interval_set_seq(interval_sub: mni, cur_seq);
2634
2635	switch (range->event) {
2636	case MMU_NOTIFY_UNMAP:
2637	svm_range_unmap_from_cpu(mm: mni->mm, prange, start, last);
2638	break;
2639	default:
2640	svm_range_evict(prange, mm: mni->mm, start, last, event: range->event);
2641	break;
2642	}
2643
2644	svm_range_unlock(prange);
2645
2646	return true;
2647	}
2648
2649	/**
2650	* svm_range_from_addr - find svm range from fault address
2651	* @svms: svm range list header
2652	* @addr: address to search range interval tree, in pages
2653	* @parent: parent range if range is on child list
2654	*
2655	* Context: The caller must hold svms->lock
2656	*
2657	* Return: the svm_range found or NULL
2658	*/
2659	struct svm_range *
2660	svm_range_from_addr(struct svm_range_list *svms, unsigned long addr,
2661	struct svm_range **parent)
2662	{
2663	struct interval_tree_node *node;
2664	struct svm_range *prange;
2665	struct svm_range *pchild;
2666
2667	node = interval_tree_iter_first(root: &svms->objects, start: addr, last: addr);
2668	if (!node)
2669	return NULL;
2670
2671	prange = container_of(node, struct svm_range, it_node);
2672	pr_debug("address 0x%lx prange [0x%lx 0x%lx] node [0x%lx 0x%lx]\n",
2673	addr, prange->start, prange->last, node->start, node->last);
2674
2675	if (addr >= prange->start && addr <= prange->last) {
2676	if (parent)
2677	*parent = prange;
2678	return prange;
2679	}
2680	list_for_each_entry(pchild, &prange->child_list, child_list)
2681	if (addr >= pchild->start && addr <= pchild->last) {
2682	pr_debug("found address 0x%lx pchild [0x%lx 0x%lx]\n",
2683	addr, pchild->start, pchild->last);
2684	if (parent)
2685	*parent = prange;
2686	return pchild;
2687	}
2688
2689	return NULL;
2690	}
2691
2692	/* svm_range_best_restore_location - decide the best fault restore location
2693	* @prange: svm range structure
2694	* @adev: the GPU on which vm fault happened
2695	*
2696	* This is only called when xnack is on, to decide the best location to restore
2697	* the range mapping after GPU vm fault. Caller uses the best location to do
2698	* migration if actual loc is not best location, then update GPU page table
2699	* mapping to the best location.
2700	*
2701	* If the preferred loc is accessible by faulting GPU, use preferred loc.
2702	* If vm fault gpu idx is on range ACCESSIBLE bitmap, best_loc is vm fault gpu
2703	* If vm fault gpu idx is on range ACCESSIBLE_IN_PLACE bitmap, then
2704	* if range actual loc is cpu, best_loc is cpu
2705	* if vm fault gpu is on xgmi same hive of range actual loc gpu, best_loc is
2706	* range actual loc.
2707	* Otherwise, GPU no access, best_loc is -1.
2708	*
2709	* Return:
2710	* -1 means vm fault GPU no access
2711	* 0 for CPU or GPU id
2712	*/
2713	static int32_t
2714	svm_range_best_restore_location(struct svm_range *prange,
2715	struct kfd_node *node,
2716	int32_t *gpuidx)
2717	{
2718	struct kfd_node *bo_node, *preferred_node;
2719	struct kfd_process *p;
2720	uint32_t gpuid;
2721	int r;
2722
2723	p = container_of(prange->svms, struct kfd_process, svms);
2724
2725	r = kfd_process_gpuid_from_node(p, node, gpuid: &gpuid, gpuidx);
2726	if (r < 0) {
2727	pr_debug("failed to get gpuid from kgd\n");
2728	return -1;
2729	}
2730
2731	if (node->adev->apu_prefer_gtt)
2732	return 0;
2733
2734	if (prange->preferred_loc == gpuid ||
2735	prange->preferred_loc == KFD_IOCTL_SVM_LOCATION_SYSMEM) {
2736	return prange->preferred_loc;
2737	} else if (prange->preferred_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED) {
2738	preferred_node = svm_range_get_node_by_id(prange, gpu_id: prange->preferred_loc);
2739	if (preferred_node && svm_nodes_in_same_hive(node_a: node, node_b: preferred_node))
2740	return prange->preferred_loc;
2741	/* fall through */
2742	}
2743
2744	if (test_bit(*gpuidx, prange->bitmap_access))
2745	return gpuid;
2746
2747	if (test_bit(*gpuidx, prange->bitmap_aip)) {
2748	if (!prange->actual_loc)
2749	return 0;
2750
2751	bo_node = svm_range_get_node_by_id(prange, gpu_id: prange->actual_loc);
2752	if (bo_node && svm_nodes_in_same_hive(node_a: node, node_b: bo_node))
2753	return prange->actual_loc;
2754	else
2755	return 0;
2756	}
2757
2758	return -1;
2759	}
2760
2761	static int
2762	svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr,
2763	unsigned long *start, unsigned long *last,
2764	bool *is_heap_stack)
2765	{
2766	struct vm_area_struct *vma;
2767	struct interval_tree_node *node;
2768	struct rb_node *rb_node;
2769	unsigned long start_limit, end_limit;
2770
2771	vma = vma_lookup(mm: p->mm, addr: addr << PAGE_SHIFT);
2772	if (!vma) {
2773	pr_debug("VMA does not exist in address [0x%llx]\n", addr);
2774	return -EFAULT;
2775	}
2776
2777	*is_heap_stack = vma_is_initial_heap(vma) || vma_is_initial_stack(vma);
2778
2779	start_limit = max(vma->vm_start >> PAGE_SHIFT,
2780	(unsigned long)ALIGN_DOWN(addr, 1UL << p->svms.default_granularity));
2781	end_limit = min(vma->vm_end >> PAGE_SHIFT,
2782	(unsigned long)ALIGN(addr + 1, 1UL << p->svms.default_granularity));
2783
2784	/* First range that starts after the fault address */
2785	node = interval_tree_iter_first(root: &p->svms.objects, start: addr + 1, ULONG_MAX);
2786	if (node) {
2787	end_limit = min(end_limit, node->start);
2788	/* Last range that ends before the fault address */
2789	rb_node = rb_prev(&node->rb);
2790	} else {
2791	/* Last range must end before addr because
2792	* there was no range after addr
2793	*/
2794	rb_node = rb_last(root: &p->svms.objects.rb_root);
2795	}
2796	if (rb_node) {
2797	node = container_of(rb_node, struct interval_tree_node, rb);
2798	if (node->last >= addr) {
2799	WARN(1, "Overlap with prev node and page fault addr\n");
2800	return -EFAULT;
2801	}
2802	start_limit = max(start_limit, node->last + 1);
2803	}
2804
2805	*start = start_limit;
2806	*last = end_limit - 1;
2807
2808	pr_debug("vma [0x%lx 0x%lx] range [0x%lx 0x%lx] is_heap_stack %d\n",
2809	vma->vm_start >> PAGE_SHIFT, vma->vm_end >> PAGE_SHIFT,
2810	*start, *last, *is_heap_stack);
2811
2812	return 0;
2813	}
2814
2815	static int
2816	svm_range_check_vm_userptr(struct kfd_process *p, uint64_t start, uint64_t last,
2817	uint64_t *bo_s, uint64_t *bo_l)
2818	{
2819	struct amdgpu_bo_va_mapping *mapping;
2820	struct interval_tree_node *node;
2821	struct amdgpu_bo *bo = NULL;
2822	unsigned long userptr;
2823	uint32_t i;
2824	int r;
2825
2826	for (i = 0; i < p->n_pdds; i++) {
2827	struct amdgpu_vm *vm;
2828
2829	if (!p->pdds[i]->drm_priv)
2830	continue;
2831
2832	vm = drm_priv_to_vm(p->pdds[i]->drm_priv);
2833	r = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: false);
2834	if (r)
2835	return r;
2836
2837	/* Check userptr by searching entire vm->va interval tree */
2838	node = interval_tree_iter_first(root: &vm->va, start: 0, last: ~0ULL);
2839	while (node) {
2840	mapping = container_of((struct rb_node *)node,
2841	struct amdgpu_bo_va_mapping, rb);
2842	bo = mapping->bo_va->base.bo;
2843
2844	if (!amdgpu_ttm_tt_affect_userptr(ttm: bo->tbo.ttm,
2845	start: start << PAGE_SHIFT,
2846	end: last << PAGE_SHIFT,
2847	userptr: &userptr)) {
2848	node = interval_tree_iter_next(node, start: 0, last: ~0ULL);
2849	continue;
2850	}
2851
2852	pr_debug("[0x%llx 0x%llx] already userptr mapped\n",
2853	start, last);
2854	if (bo_s && bo_l) {
2855	*bo_s = userptr >> PAGE_SHIFT;
2856	*bo_l = *bo_s + bo->tbo.ttm->num_pages - 1;
2857	}
2858	amdgpu_bo_unreserve(bo: vm->root.bo);
2859	return -EADDRINUSE;
2860	}
2861	amdgpu_bo_unreserve(bo: vm->root.bo);
2862	}
2863	return 0;
2864	}
2865
2866	static struct
2867	svm_range *svm_range_create_unregistered_range(struct kfd_node *node,
2868	struct kfd_process *p,
2869	struct mm_struct *mm,
2870	int64_t addr)
2871	{
2872	struct svm_range *prange = NULL;
2873	unsigned long start, last;
2874	uint32_t gpuid, gpuidx;
2875	bool is_heap_stack;
2876	uint64_t bo_s = 0;
2877	uint64_t bo_l = 0;
2878	int r;
2879
2880	if (svm_range_get_range_boundaries(p, addr, start: &start, last: &last,
2881	is_heap_stack: &is_heap_stack))
2882	return NULL;
2883
2884	r = svm_range_check_vm(p, start, last, bo_s: &bo_s, bo_l: &bo_l);
2885	if (r != -EADDRINUSE)
2886	r = svm_range_check_vm_userptr(p, start, last, bo_s: &bo_s, bo_l: &bo_l);
2887
2888	if (r == -EADDRINUSE) {
2889	if (addr >= bo_s && addr <= bo_l)
2890	return NULL;
2891
2892	/* Create one page svm range if 2MB range overlapping */
2893	start = addr;
2894	last = addr;
2895	}
2896
2897	prange = svm_range_new(svms: &p->svms, start, last, update_mem_usage: true);
2898	if (!prange) {
2899	pr_debug("Failed to create prange in address [0x%llx]\n", addr);
2900	return NULL;
2901	}
2902	if (kfd_process_gpuid_from_node(p, node, gpuid: &gpuid, gpuidx: &gpuidx)) {
2903	pr_debug("failed to get gpuid from kgd\n");
2904	svm_range_free(prange, do_unmap: true);
2905	return NULL;
2906	}
2907
2908	if (is_heap_stack)
2909	prange->preferred_loc = KFD_IOCTL_SVM_LOCATION_SYSMEM;
2910
2911	svm_range_add_to_svms(prange);
2912	svm_range_add_notifier_locked(mm, prange);
2913
2914	return prange;
2915	}
2916
2917	/* svm_range_skip_recover - decide if prange can be recovered
2918	* @prange: svm range structure
2919	*
2920	* GPU vm retry fault handle skip recover the range for cases:
2921	* 1. prange is on deferred list to be removed after unmap, it is stale fault,
2922	* deferred list work will drain the stale fault before free the prange.
2923	* 2. prange is on deferred list to add interval notifier after split, or
2924	* 3. prange is child range, it is split from parent prange, recover later
2925	* after interval notifier is added.
2926	*
2927	* Return: true to skip recover, false to recover
2928	*/
2929	static bool svm_range_skip_recover(struct svm_range *prange)
2930	{
2931	struct svm_range_list *svms = prange->svms;
2932
2933	spin_lock(lock: &svms->deferred_list_lock);
2934	if (list_empty(head: &prange->deferred_list) &&
2935	list_empty(head: &prange->child_list)) {
2936	spin_unlock(lock: &svms->deferred_list_lock);
2937	return false;
2938	}
2939	spin_unlock(lock: &svms->deferred_list_lock);
2940
2941	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
2942	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] unmapped\n",
2943	svms, prange, prange->start, prange->last);
2944	return true;
2945	}
2946	if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP ||
2947	prange->work_item.op == SVM_OP_ADD_RANGE) {
2948	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not added yet\n",
2949	svms, prange, prange->start, prange->last);
2950	return true;
2951	}
2952	return false;
2953	}
2954
2955	static void
2956	svm_range_count_fault(struct kfd_node *node, struct kfd_process *p,
2957	int32_t gpuidx)
2958	{
2959	struct kfd_process_device *pdd;
2960
2961	/* fault is on different page of same range
2962	* or fault is skipped to recover later
2963	* or fault is on invalid virtual address
2964	*/
2965	if (gpuidx == MAX_GPU_INSTANCE) {
2966	uint32_t gpuid;
2967	int r;
2968
2969	r = kfd_process_gpuid_from_node(p, node, gpuid: &gpuid, gpuidx: &gpuidx);
2970	if (r < 0)
2971	return;
2972	}
2973
2974	/* fault is recovered
2975	* or fault cannot recover because GPU no access on the range
2976	*/
2977	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
2978	if (pdd)
2979	WRITE_ONCE(pdd->faults, pdd->faults + 1);
2980	}
2981
2982	static bool
2983	svm_fault_allowed(struct vm_area_struct *vma, bool write_fault)
2984	{
2985	unsigned long requested = VM_READ;
2986
2987	if (write_fault)
2988	requested |= VM_WRITE;
2989
2990	pr_debug("requested 0x%lx, vma permission flags 0x%lx\n", requested,
2991	vma->vm_flags);
2992	return (vma->vm_flags & requested) == requested;
2993	}
2994
2995	int
2996	svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
2997	uint32_t vmid, uint32_t node_id,
2998	uint64_t addr, uint64_t ts, bool write_fault)
2999	{
3000	unsigned long start, last, size;
3001	struct mm_struct *mm = NULL;
3002	struct svm_range_list *svms;
3003	struct svm_range *prange;
3004	struct kfd_process *p;
3005	ktime_t timestamp = ktime_get_boottime();
3006	struct kfd_node *node;
3007	int32_t best_loc;
3008	int32_t gpuid, gpuidx = MAX_GPU_INSTANCE;
3009	bool write_locked = false;
3010	struct vm_area_struct *vma;
3011	bool migration = false;
3012	int r = 0;
3013
3014	if (!KFD_IS_SVM_API_SUPPORTED(adev)) {
3015	pr_debug("device does not support SVM\n");
3016	return -EFAULT;
3017	}
3018
3019	p = kfd_lookup_process_by_pasid(pasid, NULL);
3020	if (!p) {
3021	pr_debug("kfd process not founded pasid 0x%x\n", pasid);
3022	return 0;
3023	}
3024	svms = &p->svms;
3025
3026	pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
3027
3028	if (atomic_read(v: &svms->drain_pagefaults)) {
3029	pr_debug("page fault handling disabled, drop fault 0x%llx\n", addr);
3030	r = 0;
3031	goto out;
3032	}
3033
3034	node = kfd_node_by_irq_ids(adev, node_id, vmid);
3035	if (!node) {
3036	pr_debug("kfd node does not exist node_id: %d, vmid: %d\n", node_id,
3037	vmid);
3038	r = -EFAULT;
3039	goto out;
3040	}
3041
3042	if (kfd_process_gpuid_from_node(p, node, gpuid: &gpuid, gpuidx: &gpuidx)) {
3043	pr_debug("failed to get gpuid/gpuidex for node_id: %d\n", node_id);
3044	r = -EFAULT;
3045	goto out;
3046	}
3047
3048	if (!p->xnack_enabled) {
3049	pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
3050	r = -EFAULT;
3051	goto out;
3052	}
3053
3054	/* p->lead_thread is available as kfd_process_wq_release flush the work
3055	* before releasing task ref.
3056	*/
3057	mm = get_task_mm(task: p->lead_thread);
3058	if (!mm) {
3059	pr_debug("svms 0x%p failed to get mm\n", svms);
3060	r = 0;
3061	goto out;
3062	}
3063
3064	mmap_read_lock(mm);
3065	retry_write_locked:
3066	mutex_lock(&svms->lock);
3067
3068	/* check if this page fault time stamp is before svms->checkpoint_ts */
3069	if (svms->checkpoint_ts[gpuidx] != 0) {
3070	if (amdgpu_ih_ts_after_or_equal(ts, svms->checkpoint_ts[gpuidx])) {
3071	pr_debug("draining retry fault, drop fault 0x%llx\n", addr);
3072	if (write_locked)
3073	mmap_write_downgrade(mm);
3074	r = -EAGAIN;
3075	goto out_unlock_svms;
3076	} else {
3077	/* ts is after svms->checkpoint_ts now, reset svms->checkpoint_ts
3078	* to zero to avoid following ts wrap around give wrong comparing
3079	*/
3080	svms->checkpoint_ts[gpuidx] = 0;
3081	}
3082	}
3083
3084	prange = svm_range_from_addr(svms, addr, NULL);
3085	if (!prange) {
3086	pr_debug("failed to find prange svms 0x%p address [0x%llx]\n",
3087	svms, addr);
3088	if (!write_locked) {
3089	/* Need the write lock to create new range with MMU notifier.
3090	* Also flush pending deferred work to make sure the interval
3091	* tree is up to date before we add a new range
3092	*/
3093	mutex_unlock(lock: &svms->lock);
3094	mmap_read_unlock(mm);
3095	mmap_write_lock(mm);
3096	write_locked = true;
3097	goto retry_write_locked;
3098	}
3099	prange = svm_range_create_unregistered_range(node, p, mm, addr);
3100	if (!prange) {
3101	pr_debug("failed to create unregistered range svms 0x%p address [0x%llx]\n",
3102	svms, addr);
3103	mmap_write_downgrade(mm);
3104	r = -EFAULT;
3105	goto out_unlock_svms;
3106	}
3107	}
3108	if (write_locked)
3109	mmap_write_downgrade(mm);
3110
3111	mutex_lock(&prange->migrate_mutex);
3112
3113	if (svm_range_skip_recover(prange)) {
3114	amdgpu_gmc_filter_faults_remove(adev: node->adev, addr, pasid);
3115	r = 0;
3116	goto out_unlock_range;
3117	}
3118
3119	/* skip duplicate vm fault on different pages of same range */
3120	if (ktime_before(cmp1: timestamp, ktime_add_ns(prange->validate_timestamp,
3121	AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING))) {
3122	pr_debug("svms 0x%p [0x%lx %lx] already restored\n",
3123	svms, prange->start, prange->last);
3124	r = 0;
3125	goto out_unlock_range;
3126	}
3127
3128	/* __do_munmap removed VMA, return success as we are handling stale
3129	* retry fault.
3130	*/
3131	vma = vma_lookup(mm, addr: addr << PAGE_SHIFT);
3132	if (!vma) {
3133	pr_debug("address 0x%llx VMA is removed\n", addr);
3134	r = 0;
3135	goto out_unlock_range;
3136	}
3137
3138	if (!svm_fault_allowed(vma, write_fault)) {
3139	pr_debug("fault addr 0x%llx no %s permission\n", addr,
3140	write_fault ? "write" : "read");
3141	r = -EPERM;
3142	goto out_unlock_range;
3143	}
3144
3145	best_loc = svm_range_best_restore_location(prange, node, gpuidx: &gpuidx);
3146	if (best_loc == -1) {
3147	pr_debug("svms %p failed get best restore loc [0x%lx 0x%lx]\n",
3148	svms, prange->start, prange->last);
3149	r = -EACCES;
3150	goto out_unlock_range;
3151	}
3152
3153	pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n",
3154	svms, prange->start, prange->last, best_loc,
3155	prange->actual_loc);
3156
3157	kfd_smi_event_page_fault_start(node, pid: p->lead_thread->pid, address: addr,
3158	write_fault, ts: timestamp);
3159
3160	/* Align migration range start and size to granularity size */
3161	size = 1UL << prange->granularity;
3162	start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start);
3163	last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
3164	if (prange->actual_loc != 0 || best_loc != 0) {
3165	if (best_loc) {
3166	r = svm_migrate_to_vram(prange, best_loc, start, last,
3167	mm, trigger: KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU);
3168	if (r) {
3169	pr_debug("svm_migrate_to_vram failed (%d) at %llx, falling back to system memory\n",
3170	r, addr);
3171	/* Fallback to system memory if migration to
3172	* VRAM failed
3173	*/
3174	if (prange->actual_loc && prange->actual_loc != best_loc)
3175	r = svm_migrate_vram_to_ram(prange, mm, start, last,
3176	trigger: KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, NULL);
3177	else
3178	r = 0;
3179	}
3180	} else {
3181	r = svm_migrate_vram_to_ram(prange, mm, start, last,
3182	trigger: KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, NULL);
3183	}
3184	if (r) {
3185	pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n",
3186	r, svms, start, last);
3187	goto out_migrate_fail;
3188	} else {
3189	migration = true;
3190	}
3191	}
3192
3193	r = svm_range_validate_and_map(mm, map_start: start, map_last: last, prange, gpuidx, intr: false,
3194	wait: false, flush_tlb: false);
3195	if (r)
3196	pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n",
3197	r, svms, start, last);
3198
3199	out_migrate_fail:
3200	kfd_smi_event_page_fault_end(node, pid: p->lead_thread->pid, address: addr,
3201	migration);
3202
3203	out_unlock_range:
3204	mutex_unlock(lock: &prange->migrate_mutex);
3205	out_unlock_svms:
3206	mutex_unlock(lock: &svms->lock);
3207	mmap_read_unlock(mm);
3208
3209	if (r != -EAGAIN)
3210	svm_range_count_fault(node, p, gpuidx);
3211
3212	mmput(mm);
3213	out:
3214	kfd_unref_process(p);
3215
3216	if (r == -EAGAIN) {
3217	pr_debug("recover vm fault later\n");
3218	amdgpu_gmc_filter_faults_remove(adev: node->adev, addr, pasid);
3219	r = 0;
3220	}
3221	return r;
3222	}
3223
3224	int
3225	svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled)
3226	{
3227	struct svm_range *prange, *pchild;
3228	uint64_t reserved_size = 0;
3229	uint64_t size;
3230	int r = 0;
3231
3232	pr_debug("switching xnack from %d to %d\n", p->xnack_enabled, xnack_enabled);
3233
3234	mutex_lock(&p->svms.lock);
3235
3236	list_for_each_entry(prange, &p->svms.list, list) {
3237	svm_range_lock(prange);
3238	list_for_each_entry(pchild, &prange->child_list, child_list) {
3239	size = (pchild->last - pchild->start + 1) << PAGE_SHIFT;
3240	if (xnack_enabled) {
3241	amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
3242	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0);
3243	} else {
3244	r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,
3245	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0);
3246	if (r)
3247	goto out_unlock;
3248	reserved_size += size;
3249	}
3250	}
3251
3252	size = (prange->last - prange->start + 1) << PAGE_SHIFT;
3253	if (xnack_enabled) {
3254	amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
3255	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0);
3256	} else {
3257	r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,
3258	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0);
3259	if (r)
3260	goto out_unlock;
3261	reserved_size += size;
3262	}
3263	out_unlock:
3264	svm_range_unlock(prange);
3265	if (r)
3266	break;
3267	}
3268
3269	if (r)
3270	amdgpu_amdkfd_unreserve_mem_limit(NULL, size: reserved_size,
3271	KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, xcp_id: 0);
3272	else
3273	/* Change xnack mode must be inside svms lock, to avoid race with
3274	* svm_range_deferred_list_work unreserve memory in parallel.
3275	*/
3276	p->xnack_enabled = xnack_enabled;
3277
3278	mutex_unlock(lock: &p->svms.lock);
3279	return r;
3280	}
3281
3282	void svm_range_list_fini(struct kfd_process *p)
3283	{
3284	struct svm_range *prange;
3285	struct svm_range *next;
3286
3287	pr_debug("process pid %d svms 0x%p\n", p->lead_thread->pid,
3288	&p->svms);
3289
3290	cancel_delayed_work_sync(dwork: &p->svms.restore_work);
3291
3292	/* Ensure list work is finished before process is destroyed */
3293	flush_work(work: &p->svms.deferred_list_work);
3294
3295	/*
3296	* Ensure no retry fault comes in afterwards, as page fault handler will
3297	* not find kfd process and take mm lock to recover fault.
3298	* stop kfd page fault handing, then wait pending page faults got drained
3299	*/
3300	atomic_set(v: &p->svms.drain_pagefaults, i: 1);
3301	svm_range_drain_retry_fault(svms: &p->svms);
3302
3303	list_for_each_entry_safe(prange, next, &p->svms.list, list) {
3304	svm_range_unlink(prange);
3305	svm_range_remove_notifier(prange);
3306	svm_range_free(prange, do_unmap: true);
3307	}
3308
3309	mutex_destroy(lock: &p->svms.lock);
3310
3311	pr_debug("process pid %d svms 0x%p done\n",
3312	p->lead_thread->pid, &p->svms);
3313	}
3314
3315	int svm_range_list_init(struct kfd_process *p)
3316	{
3317	struct svm_range_list *svms = &p->svms;
3318	int i;
3319
3320	svms->objects = RB_ROOT_CACHED;
3321	mutex_init(&svms->lock);
3322	INIT_LIST_HEAD(list: &svms->list);
3323	atomic_set(v: &svms->evicted_ranges, i: 0);
3324	atomic_set(v: &svms->drain_pagefaults, i: 0);
3325	INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work);
3326	INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work);
3327	INIT_LIST_HEAD(list: &svms->deferred_range_list);
3328	INIT_LIST_HEAD(list: &svms->criu_svm_metadata_list);
3329	spin_lock_init(&svms->deferred_list_lock);
3330
3331	for (i = 0; i < p->n_pdds; i++)
3332	if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->adev))
3333	bitmap_set(map: svms->bitmap_supported, start: i, nbits: 1);
3334
3335	/* Value of default granularity cannot exceed 0x1B, the
3336	* number of pages supported by a 4-level paging table
3337	*/
3338	svms->default_granularity = min_t(u8, amdgpu_svm_default_granularity, 0x1B);
3339	pr_debug("Default SVM Granularity to use: %d\n", svms->default_granularity);
3340
3341	return 0;
3342	}
3343
3344	/**
3345	* svm_range_check_vm - check if virtual address range mapped already
3346	* @p: current kfd_process
3347	* @start: range start address, in pages
3348	* @last: range last address, in pages
3349	* @bo_s: mapping start address in pages if address range already mapped
3350	* @bo_l: mapping last address in pages if address range already mapped
3351	*
3352	* The purpose is to avoid virtual address ranges already allocated by
3353	* kfd_ioctl_alloc_memory_of_gpu ioctl.
3354	* It looks for each pdd in the kfd_process.
3355	*
3356	* Context: Process context
3357	*
3358	* Return 0 - OK, if the range is not mapped.
3359	* Otherwise error code:
3360	* -EADDRINUSE - if address is mapped already by kfd_ioctl_alloc_memory_of_gpu
3361	* -ERESTARTSYS - A wait for the buffer to become unreserved was interrupted by
3362	* a signal. Release all buffer reservations and return to user-space.
3363	*/
3364	static int
3365	svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last,
3366	uint64_t *bo_s, uint64_t *bo_l)
3367	{
3368	struct amdgpu_bo_va_mapping *mapping;
3369	struct interval_tree_node *node;
3370	uint32_t i;
3371	int r;
3372
3373	for (i = 0; i < p->n_pdds; i++) {
3374	struct amdgpu_vm *vm;
3375
3376	if (!p->pdds[i]->drm_priv)
3377	continue;
3378
3379	vm = drm_priv_to_vm(p->pdds[i]->drm_priv);
3380	r = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: false);
3381	if (r)
3382	return r;
3383
3384	node = interval_tree_iter_first(root: &vm->va, start, last);
3385	if (node) {
3386	pr_debug("range [0x%llx 0x%llx] already TTM mapped\n",
3387	start, last);
3388	mapping = container_of((struct rb_node *)node,
3389	struct amdgpu_bo_va_mapping, rb);
3390	if (bo_s && bo_l) {
3391	*bo_s = mapping->start;
3392	*bo_l = mapping->last;
3393	}
3394	amdgpu_bo_unreserve(bo: vm->root.bo);
3395	return -EADDRINUSE;
3396	}
3397	amdgpu_bo_unreserve(bo: vm->root.bo);
3398	}
3399
3400	return 0;
3401	}
3402
3403	/**
3404	* svm_range_is_valid - check if virtual address range is valid
3405	* @p: current kfd_process
3406	* @start: range start address, in pages
3407	* @size: range size, in pages
3408	*
3409	* Valid virtual address range means it belongs to one or more VMAs
3410	*
3411	* Context: Process context
3412	*
3413	* Return:
3414	* 0 - OK, otherwise error code
3415	*/
3416	static int
3417	svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size)
3418	{
3419	const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP;
3420	struct vm_area_struct *vma;
3421	unsigned long end;
3422	unsigned long start_unchg = start;
3423
3424	start <<= PAGE_SHIFT;
3425	end = start + (size << PAGE_SHIFT);
3426	do {
3427	vma = vma_lookup(mm: p->mm, addr: start);
3428	if (!vma || (vma->vm_flags & device_vma))
3429	return -EFAULT;
3430	start = min(end, vma->vm_end);
3431	} while (start < end);
3432
3433	return svm_range_check_vm(p, start: start_unchg, last: (end - 1) >> PAGE_SHIFT, NULL,
3434	NULL);
3435	}
3436
3437	/**
3438	* svm_range_best_prefetch_location - decide the best prefetch location
3439	* @prange: svm range structure
3440	*
3441	* For xnack off:
3442	* If range map to single GPU, the best prefetch location is prefetch_loc, which
3443	* can be CPU or GPU.
3444	*
3445	* If range is ACCESS or ACCESS_IN_PLACE by mGPUs, only if mGPU connection on
3446	* XGMI same hive, the best prefetch location is prefetch_loc GPU, othervise
3447	* the best prefetch location is always CPU, because GPU can not have coherent
3448	* mapping VRAM of other GPUs even with large-BAR PCIe connection.
3449	*
3450	* For xnack on:
3451	* If range is not ACCESS_IN_PLACE by mGPUs, the best prefetch location is
3452	* prefetch_loc, other GPU access will generate vm fault and trigger migration.
3453	*
3454	* If range is ACCESS_IN_PLACE by mGPUs, only if mGPU connection on XGMI same
3455	* hive, the best prefetch location is prefetch_loc GPU, otherwise the best
3456	* prefetch location is always CPU.
3457	*
3458	* Context: Process context
3459	*
3460	* Return:
3461	* 0 for CPU or GPU id
3462	*/
3463	static uint32_t
3464	svm_range_best_prefetch_location(struct svm_range *prange)
3465	{
3466	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
3467	uint32_t best_loc = prange->prefetch_loc;
3468	struct kfd_process_device *pdd;
3469	struct kfd_node *bo_node;
3470	struct kfd_process *p;
3471	uint32_t gpuidx;
3472
3473	p = container_of(prange->svms, struct kfd_process, svms);
3474
3475	if (!best_loc || best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED)
3476	goto out;
3477
3478	bo_node = svm_range_get_node_by_id(prange, gpu_id: best_loc);
3479	if (!bo_node) {
3480	WARN_ONCE(1, "failed to get valid kfd node at id%x\n", best_loc);
3481	best_loc = 0;
3482	goto out;
3483	}
3484
3485	if (bo_node->adev->apu_prefer_gtt) {
3486	best_loc = 0;
3487	goto out;
3488	}
3489
3490	if (p->xnack_enabled)
3491	bitmap_copy(dst: bitmap, src: prange->bitmap_aip, MAX_GPU_INSTANCE);
3492	else
3493	bitmap_or(dst: bitmap, src1: prange->bitmap_access, src2: prange->bitmap_aip,
3494	MAX_GPU_INSTANCE);
3495
3496	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
3497	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
3498	if (!pdd) {
3499	pr_debug("failed to get device by idx 0x%x\n", gpuidx);
3500	continue;
3501	}
3502
3503	if (pdd->dev->adev == bo_node->adev)
3504	continue;
3505
3506	if (!svm_nodes_in_same_hive(node_a: pdd->dev, node_b: bo_node)) {
3507	best_loc = 0;
3508	break;
3509	}
3510	}
3511
3512	out:
3513	pr_debug("xnack %d svms 0x%p [0x%lx 0x%lx] best loc 0x%x\n",
3514	p->xnack_enabled, &p->svms, prange->start, prange->last,
3515	best_loc);
3516
3517	return best_loc;
3518	}
3519
3520	/* svm_range_trigger_migration - start page migration if prefetch loc changed
3521	* @mm: current process mm_struct
3522	* @prange: svm range structure
3523	* @migrated: output, true if migration is triggered
3524	*
3525	* If range perfetch_loc is GPU, actual loc is cpu 0, then migrate the range
3526	* from ram to vram.
3527	* If range prefetch_loc is cpu 0, actual loc is GPU, then migrate the range
3528	* from vram to ram.
3529	*
3530	* If GPU vm fault retry is not enabled, migration interact with MMU notifier
3531	* and restore work:
3532	* 1. migrate_vma_setup invalidate pages, MMU notifier callback svm_range_evict
3533	* stops all queues, schedule restore work
3534	* 2. svm_range_restore_work wait for migration is done by
3535	* a. svm_range_validate_vram takes prange->migrate_mutex
3536	* b. svm_range_validate_ram HMM get pages wait for CPU fault handle returns
3537	* 3. restore work update mappings of GPU, resume all queues.
3538	*
3539	* Context: Process context
3540	*
3541	* Return:
3542	* 0 - OK, otherwise - error code of migration
3543	*/
3544	static int
3545	svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,
3546	bool *migrated)
3547	{
3548	uint32_t best_loc;
3549	int r = 0;
3550
3551	*migrated = false;
3552	best_loc = svm_range_best_prefetch_location(prange);
3553
3554	/* when best_loc is a gpu node and same as prange->actual_loc
3555	* we still need do migration as prange->actual_loc !=0 does
3556	* not mean all pages in prange are vram. hmm migrate will pick
3557	* up right pages during migration.
3558	*/
3559	if ((best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED) ||
3560	(best_loc == 0 && prange->actual_loc == 0))
3561	return 0;
3562
3563	if (!best_loc) {
3564	r = svm_migrate_vram_to_ram(prange, mm, start: prange->start, last: prange->last,
3565	trigger: KFD_MIGRATE_TRIGGER_PREFETCH, NULL);
3566	*migrated = !r;
3567	return r;
3568	}
3569
3570	r = svm_migrate_to_vram(prange, best_loc, start: prange->start, last: prange->last,
3571	mm, trigger: KFD_MIGRATE_TRIGGER_PREFETCH);
3572	*migrated = !r;
3573
3574	return 0;
3575	}
3576
3577	int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence)
3578	{
3579	/* Dereferencing fence->svm_bo is safe here because the fence hasn't
3580	* signaled yet and we're under the protection of the fence->lock.
3581	* After the fence is signaled in svm_range_bo_release, we cannot get
3582	* here any more.
3583	*
3584	* Reference is dropped in svm_range_evict_svm_bo_worker.
3585	*/
3586	if (svm_bo_ref_unless_zero(svm_bo: fence->svm_bo)) {
3587	WRITE_ONCE(fence->svm_bo->evicting, 1);
3588	schedule_work(work: &fence->svm_bo->eviction_work);
3589	}
3590
3591	return 0;
3592	}
3593
3594	static void svm_range_evict_svm_bo_worker(struct work_struct *work)
3595	{
3596	struct svm_range_bo *svm_bo;
3597	struct mm_struct *mm;
3598	int r = 0;
3599
3600	svm_bo = container_of(work, struct svm_range_bo, eviction_work);
3601
3602	if (mmget_not_zero(mm: svm_bo->eviction_fence->mm)) {
3603	mm = svm_bo->eviction_fence->mm;
3604	} else {
3605	svm_range_bo_unref(svm_bo);
3606	return;
3607	}
3608
3609	mmap_read_lock(mm);
3610	spin_lock(lock: &svm_bo->list_lock);
3611	while (!list_empty(head: &svm_bo->range_list) && !r) {
3612	struct svm_range *prange =
3613	list_first_entry(&svm_bo->range_list,
3614	struct svm_range, svm_bo_list);
3615	int retries = 3;
3616
3617	list_del_init(entry: &prange->svm_bo_list);
3618	spin_unlock(lock: &svm_bo->list_lock);
3619
3620	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
3621	prange->start, prange->last);
3622
3623	mutex_lock(&prange->migrate_mutex);
3624	do {
3625	/* migrate all vram pages in this prange to sys ram
3626	* after that prange->actual_loc should be zero
3627	*/
3628	r = svm_migrate_vram_to_ram(prange, mm,
3629	start: prange->start, last: prange->last,
3630	trigger: KFD_MIGRATE_TRIGGER_TTM_EVICTION, NULL);
3631	} while (!r && prange->actual_loc && --retries);
3632
3633	if (!r && prange->actual_loc)
3634	pr_info_once("Migration failed during eviction");
3635
3636	if (!prange->actual_loc) {
3637	mutex_lock(&prange->lock);
3638	prange->svm_bo = NULL;
3639	mutex_unlock(lock: &prange->lock);
3640	}
3641	mutex_unlock(lock: &prange->migrate_mutex);
3642
3643	spin_lock(lock: &svm_bo->list_lock);
3644	}
3645	spin_unlock(lock: &svm_bo->list_lock);
3646	mmap_read_unlock(mm);
3647	mmput(mm);
3648
3649	dma_fence_signal(fence: &svm_bo->eviction_fence->base);
3650
3651	/* This is the last reference to svm_bo, after svm_range_vram_node_free
3652	* has been called in svm_migrate_vram_to_ram
3653	*/
3654	WARN_ONCE(!r && kref_read(&svm_bo->kref) != 1, "This was not the last reference\n");
3655	svm_range_bo_unref(svm_bo);
3656	}
3657
3658	static int
3659	svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm,
3660	uint64_t start, uint64_t size, uint32_t nattr,
3661	struct kfd_ioctl_svm_attribute *attrs)
3662	{
3663	struct amdkfd_process_info *process_info = p->kgd_process_info;
3664	struct list_head update_list;
3665	struct list_head insert_list;
3666	struct list_head remove_list;
3667	struct list_head remap_list;
3668	struct svm_range_list *svms;
3669	struct svm_range *prange;
3670	struct svm_range *next;
3671	bool update_mapping = false;
3672	bool flush_tlb;
3673	int r, ret = 0;
3674
3675	pr_debug("process pid %d svms 0x%p [0x%llx 0x%llx] pages 0x%llx\n",
3676	p->lead_thread->pid, &p->svms, start, start + size - 1, size);
3677
3678	r = svm_range_check_attr(p, nattr, attrs);
3679	if (r)
3680	return r;
3681
3682	svms = &p->svms;
3683
3684	mutex_lock(&process_info->lock);
3685
3686	svm_range_list_lock_and_flush_work(svms, mm);
3687
3688	r = svm_range_is_valid(p, start, size);
3689	if (r) {
3690	pr_debug("invalid range r=%d\n", r);
3691	mmap_write_unlock(mm);
3692	goto out;
3693	}
3694
3695	mutex_lock(&svms->lock);
3696
3697	/* Add new range and split existing ranges as needed */
3698	r = svm_range_add(p, start, size, nattr, attrs, update_list: &update_list,
3699	insert_list: &insert_list, remove_list: &remove_list, remap_list: &remap_list);
3700	if (r) {
3701	mutex_unlock(lock: &svms->lock);
3702	mmap_write_unlock(mm);
3703	goto out;
3704	}
3705	/* Apply changes as a transaction */
3706	list_for_each_entry_safe(prange, next, &insert_list, list) {
3707	svm_range_add_to_svms(prange);
3708	svm_range_add_notifier_locked(mm, prange);
3709	}
3710	list_for_each_entry(prange, &update_list, update_list) {
3711	svm_range_apply_attrs(p, prange, nattr, attrs, update_mapping: &update_mapping);
3712	/* TODO: unmap ranges from GPU that lost access */
3713	}
3714	update_mapping |= !p->xnack_enabled && !list_empty(head: &remap_list);
3715
3716	list_for_each_entry_safe(prange, next, &remove_list, update_list) {
3717	pr_debug("unlink old 0x%p prange 0x%p [0x%lx 0x%lx]\n",
3718	prange->svms, prange, prange->start,
3719	prange->last);
3720	svm_range_unlink(prange);
3721	svm_range_remove_notifier(prange);
3722	svm_range_free(prange, do_unmap: false);
3723	}
3724
3725	mmap_write_downgrade(mm);
3726	/* Trigger migrations and revalidate and map to GPUs as needed. If
3727	* this fails we may be left with partially completed actions. There
3728	* is no clean way of rolling back to the previous state in such a
3729	* case because the rollback wouldn't be guaranteed to work either.
3730	*/
3731	list_for_each_entry(prange, &update_list, update_list) {
3732	bool migrated;
3733
3734	mutex_lock(&prange->migrate_mutex);
3735
3736	r = svm_range_trigger_migration(mm, prange, migrated: &migrated);
3737	if (r)
3738	goto out_unlock_range;
3739
3740	if (migrated && (!p->xnack_enabled ||
3741	(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) &&
3742	prange->mapped_to_gpu) {
3743	pr_debug("restore_work will update mappings of GPUs\n");
3744	mutex_unlock(lock: &prange->migrate_mutex);
3745	continue;
3746	}
3747
3748	if (!migrated && !update_mapping) {
3749	mutex_unlock(lock: &prange->migrate_mutex);
3750	continue;
3751	}
3752
3753	flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu;
3754
3755	r = svm_range_validate_and_map(mm, map_start: prange->start, map_last: prange->last, prange,
3756	MAX_GPU_INSTANCE, intr: true, wait: true, flush_tlb);
3757	if (r)
3758	pr_debug("failed %d to map svm range\n", r);
3759
3760	out_unlock_range:
3761	mutex_unlock(lock: &prange->migrate_mutex);
3762	if (r)
3763	ret = r;
3764	}
3765
3766	list_for_each_entry(prange, &remap_list, update_list) {
3767	pr_debug("Remapping prange 0x%p [0x%lx 0x%lx]\n",
3768	prange, prange->start, prange->last);
3769	mutex_lock(&prange->migrate_mutex);
3770	r = svm_range_validate_and_map(mm, map_start: prange->start, map_last: prange->last, prange,
3771	MAX_GPU_INSTANCE, intr: true, wait: true, flush_tlb: prange->mapped_to_gpu);
3772	if (r)
3773	pr_debug("failed %d on remap svm range\n", r);
3774	mutex_unlock(lock: &prange->migrate_mutex);
3775	if (r)
3776	ret = r;
3777	}
3778
3779	dynamic_svm_range_dump(svms);
3780
3781	mutex_unlock(lock: &svms->lock);
3782	mmap_read_unlock(mm);
3783	out:
3784	mutex_unlock(lock: &process_info->lock);
3785
3786	pr_debug("process pid %d svms 0x%p [0x%llx 0x%llx] done, r=%d\n",
3787	p->lead_thread->pid, &p->svms, start, start + size - 1, r);
3788
3789	return ret ? ret : r;
3790	}
3791
3792	static int
3793	svm_range_get_attr(struct kfd_process *p, struct mm_struct *mm,
3794	uint64_t start, uint64_t size, uint32_t nattr,
3795	struct kfd_ioctl_svm_attribute *attrs)
3796	{
3797	DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
3798	DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
3799	bool get_preferred_loc = false;
3800	bool get_prefetch_loc = false;
3801	bool get_granularity = false;
3802	bool get_accessible = false;
3803	bool get_flags = false;
3804	uint64_t last = start + size - 1UL;
3805	uint8_t granularity = 0xff;
3806	struct interval_tree_node *node;
3807	struct svm_range_list *svms;
3808	struct svm_range *prange;
3809	uint32_t prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
3810	uint32_t location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
3811	uint32_t flags_and = 0xffffffff;
3812	uint32_t flags_or = 0;
3813	int gpuidx;
3814	uint32_t i;
3815	int r = 0;
3816
3817	pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start,
3818	start + size - 1, nattr);
3819
3820	/* Flush pending deferred work to avoid racing with deferred actions from
3821	* previous memory map changes (e.g. munmap). Concurrent memory map changes
3822	* can still race with get_attr because we don't hold the mmap lock. But that
3823	* would be a race condition in the application anyway, and undefined
3824	* behaviour is acceptable in that case.
3825	*/
3826	flush_work(work: &p->svms.deferred_list_work);
3827
3828	mmap_read_lock(mm);
3829	r = svm_range_is_valid(p, start, size);
3830	mmap_read_unlock(mm);
3831	if (r) {
3832	pr_debug("invalid range r=%d\n", r);
3833	return r;
3834	}
3835
3836	for (i = 0; i < nattr; i++) {
3837	switch (attrs[i].type) {
3838	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
3839	get_preferred_loc = true;
3840	break;
3841	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
3842	get_prefetch_loc = true;
3843	break;
3844	case KFD_IOCTL_SVM_ATTR_ACCESS:
3845	get_accessible = true;
3846	break;
3847	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
3848	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
3849	get_flags = true;
3850	break;
3851	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
3852	get_granularity = true;
3853	break;
3854	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
3855	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
3856	fallthrough;
3857	default:
3858	pr_debug("get invalid attr type 0x%x\n", attrs[i].type);
3859	return -EINVAL;
3860	}
3861	}
3862
3863	svms = &p->svms;
3864
3865	mutex_lock(&svms->lock);
3866
3867	node = interval_tree_iter_first(root: &svms->objects, start, last);
3868	if (!node) {
3869	pr_debug("range attrs not found return default values\n");
3870	svm_range_set_default_attributes(svms, location: &location, prefetch_loc: &prefetch_loc,
3871	granularity: &granularity, flags: &flags_and);
3872	flags_or = flags_and;
3873	if (p->xnack_enabled)
3874	bitmap_copy(dst: bitmap_access, src: svms->bitmap_supported,
3875	MAX_GPU_INSTANCE);
3876	else
3877	bitmap_zero(dst: bitmap_access, MAX_GPU_INSTANCE);
3878	bitmap_zero(dst: bitmap_aip, MAX_GPU_INSTANCE);
3879	goto fill_values;
3880	}
3881	bitmap_copy(dst: bitmap_access, src: svms->bitmap_supported, MAX_GPU_INSTANCE);
3882	bitmap_copy(dst: bitmap_aip, src: svms->bitmap_supported, MAX_GPU_INSTANCE);
3883
3884	while (node) {
3885	struct interval_tree_node *next;
3886
3887	prange = container_of(node, struct svm_range, it_node);
3888	next = interval_tree_iter_next(node, start, last);
3889
3890	if (get_preferred_loc) {
3891	if (prange->preferred_loc ==
3892	KFD_IOCTL_SVM_LOCATION_UNDEFINED ||
3893	(location != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
3894	location != prange->preferred_loc)) {
3895	location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
3896	get_preferred_loc = false;
3897	} else {
3898	location = prange->preferred_loc;
3899	}
3900	}
3901	if (get_prefetch_loc) {
3902	if (prange->prefetch_loc ==
3903	KFD_IOCTL_SVM_LOCATION_UNDEFINED ||
3904	(prefetch_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
3905	prefetch_loc != prange->prefetch_loc)) {
3906	prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
3907	get_prefetch_loc = false;
3908	} else {
3909	prefetch_loc = prange->prefetch_loc;
3910	}
3911	}
3912	if (get_accessible) {
3913	bitmap_and(dst: bitmap_access, src1: bitmap_access,
3914	src2: prange->bitmap_access, MAX_GPU_INSTANCE);
3915	bitmap_and(dst: bitmap_aip, src1: bitmap_aip,
3916	src2: prange->bitmap_aip, MAX_GPU_INSTANCE);
3917	}
3918	if (get_flags) {
3919	flags_and &= prange->flags;
3920	flags_or |= prange->flags;
3921	}
3922
3923	if (get_granularity && prange->granularity < granularity)
3924	granularity = prange->granularity;
3925
3926	node = next;
3927	}
3928	fill_values:
3929	mutex_unlock(lock: &svms->lock);
3930
3931	for (i = 0; i < nattr; i++) {
3932	switch (attrs[i].type) {
3933	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
3934	attrs[i].value = location;
3935	break;
3936	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
3937	attrs[i].value = prefetch_loc;
3938	break;
3939	case KFD_IOCTL_SVM_ATTR_ACCESS:
3940	gpuidx = kfd_process_gpuidx_from_gpuid(p,
3941	gpu_id: attrs[i].value);
3942	if (gpuidx < 0) {
3943	pr_debug("invalid gpuid %x\n", attrs[i].value);
3944	return -EINVAL;
3945	}
3946	if (test_bit(gpuidx, bitmap_access))
3947	attrs[i].type = KFD_IOCTL_SVM_ATTR_ACCESS;
3948	else if (test_bit(gpuidx, bitmap_aip))
3949	attrs[i].type =
3950	KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE;
3951	else
3952	attrs[i].type = KFD_IOCTL_SVM_ATTR_NO_ACCESS;
3953	break;
3954	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
3955	attrs[i].value = flags_and;
3956	break;
3957	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
3958	attrs[i].value = ~flags_or;
3959	break;
3960	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
3961	attrs[i].value = (uint32_t)granularity;
3962	break;
3963	}
3964	}
3965
3966	return 0;
3967	}
3968
3969	int kfd_criu_resume_svm(struct kfd_process *p)
3970	{
3971	struct kfd_ioctl_svm_attribute *set_attr_new, *set_attr = NULL;
3972	int nattr_common = 4, nattr_accessibility = 1;
3973	struct criu_svm_metadata *criu_svm_md = NULL;
3974	struct svm_range_list *svms = &p->svms;
3975	struct criu_svm_metadata *next = NULL;
3976	uint32_t set_flags = 0xffffffff;
3977	int i, j, num_attrs, ret = 0;
3978	uint64_t set_attr_size;
3979	struct mm_struct *mm;
3980
3981	if (list_empty(head: &svms->criu_svm_metadata_list)) {
3982	pr_debug("No SVM data from CRIU restore stage 2\n");
3983	return ret;
3984	}
3985
3986	mm = get_task_mm(task: p->lead_thread);
3987	if (!mm) {
3988	pr_err("failed to get mm for the target process\n");
3989	return -ESRCH;
3990	}
3991
3992	num_attrs = nattr_common + (nattr_accessibility * p->n_pdds);
3993
3994	i = j = 0;
3995	list_for_each_entry(criu_svm_md, &svms->criu_svm_metadata_list, list) {
3996	pr_debug("criu_svm_md[%d]\n\tstart: 0x%llx size: 0x%llx (npages)\n",
3997	i, criu_svm_md->data.start_addr, criu_svm_md->data.size);
3998
3999	for (j = 0; j < num_attrs; j++) {
4000	pr_debug("\ncriu_svm_md[%d]->attrs[%d].type : 0x%x\ncriu_svm_md[%d]->attrs[%d].value : 0x%x\n",
4001	i, j, criu_svm_md->data.attrs[j].type,
4002	i, j, criu_svm_md->data.attrs[j].value);
4003	switch (criu_svm_md->data.attrs[j].type) {
4004	/* During Checkpoint operation, the query for
4005	* KFD_IOCTL_SVM_ATTR_PREFETCH_LOC attribute might
4006	* return KFD_IOCTL_SVM_LOCATION_UNDEFINED if they were
4007	* not used by the range which was checkpointed. Care
4008	* must be taken to not restore with an invalid value
4009	* otherwise the gpuidx value will be invalid and
4010	* set_attr would eventually fail so just replace those
4011	* with another dummy attribute such as
4012	* KFD_IOCTL_SVM_ATTR_SET_FLAGS.
4013	*/
4014	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
4015	if (criu_svm_md->data.attrs[j].value ==
4016	KFD_IOCTL_SVM_LOCATION_UNDEFINED) {
4017	criu_svm_md->data.attrs[j].type =
4018	KFD_IOCTL_SVM_ATTR_SET_FLAGS;
4019	criu_svm_md->data.attrs[j].value = 0;
4020	}
4021	break;
4022	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
4023	set_flags = criu_svm_md->data.attrs[j].value;
4024	break;
4025	default:
4026	break;
4027	}
4028	}
4029
4030	/* CLR_FLAGS is not available via get_attr during checkpoint but
4031	* it needs to be inserted before restoring the ranges so
4032	* allocate extra space for it before calling set_attr
4033	*/
4034	set_attr_size = sizeof(struct kfd_ioctl_svm_attribute) *
4035	(num_attrs + 1);
4036	set_attr_new = krealloc(set_attr, set_attr_size,
4037	GFP_KERNEL);
4038	if (!set_attr_new) {
4039	ret = -ENOMEM;
4040	goto exit;
4041	}
4042	set_attr = set_attr_new;
4043
4044	memcpy(set_attr, criu_svm_md->data.attrs, num_attrs *
4045	sizeof(struct kfd_ioctl_svm_attribute));
4046	set_attr[num_attrs].type = KFD_IOCTL_SVM_ATTR_CLR_FLAGS;
4047	set_attr[num_attrs].value = ~set_flags;
4048
4049	ret = svm_range_set_attr(p, mm, start: criu_svm_md->data.start_addr,
4050	size: criu_svm_md->data.size, nattr: num_attrs + 1,
4051	attrs: set_attr);
4052	if (ret) {
4053	pr_err("CRIU: failed to set range attributes\n");
4054	goto exit;
4055	}
4056
4057	i++;
4058	}
4059	exit:
4060	kfree(objp: set_attr);
4061	list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) {
4062	pr_debug("freeing criu_svm_md[]\n\tstart: 0x%llx\n",
4063	criu_svm_md->data.start_addr);
4064	kfree(objp: criu_svm_md);
4065	}
4066
4067	mmput(mm);
4068	return ret;
4069
4070	}
4071
4072	int kfd_criu_restore_svm(struct kfd_process *p,
4073	uint8_t __user *user_priv_ptr,
4074	uint64_t *priv_data_offset,
4075	uint64_t max_priv_data_size)
4076	{
4077	uint64_t svm_priv_data_size, svm_object_md_size, svm_attrs_size;
4078	int nattr_common = 4, nattr_accessibility = 1;
4079	struct criu_svm_metadata *criu_svm_md = NULL;
4080	struct svm_range_list *svms = &p->svms;
4081	uint32_t num_devices;
4082	int ret = 0;
4083
4084	num_devices = p->n_pdds;
4085	/* Handle one SVM range object at a time, also the number of gpus are
4086	* assumed to be same on the restore node, checking must be done while
4087	* evaluating the topology earlier
4088	*/
4089
4090	svm_attrs_size = sizeof(struct kfd_ioctl_svm_attribute) *
4091	(nattr_common + nattr_accessibility * num_devices);
4092	svm_object_md_size = sizeof(struct criu_svm_metadata) + svm_attrs_size;
4093
4094	svm_priv_data_size = sizeof(struct kfd_criu_svm_range_priv_data) +
4095	svm_attrs_size;
4096
4097	criu_svm_md = kzalloc(svm_object_md_size, GFP_KERNEL);
4098	if (!criu_svm_md) {
4099	pr_err("failed to allocate memory to store svm metadata\n");
4100	return -ENOMEM;
4101	}
4102	if (*priv_data_offset + svm_priv_data_size > max_priv_data_size) {
4103	ret = -EINVAL;
4104	goto exit;
4105	}
4106
4107	ret = copy_from_user(to: &criu_svm_md->data, from: user_priv_ptr + *priv_data_offset,
4108	n: svm_priv_data_size);
4109	if (ret) {
4110	ret = -EFAULT;
4111	goto exit;
4112	}
4113	*priv_data_offset += svm_priv_data_size;
4114
4115	list_add_tail(new: &criu_svm_md->list, head: &svms->criu_svm_metadata_list);
4116
4117	return 0;
4118
4119
4120	exit:
4121	kfree(objp: criu_svm_md);
4122	return ret;
4123	}
4124
4125	void svm_range_get_info(struct kfd_process *p, uint32_t *num_svm_ranges,
4126	uint64_t *svm_priv_data_size)
4127	{
4128	uint64_t total_size, accessibility_size, common_attr_size;
4129	int nattr_common = 4, nattr_accessibility = 1;
4130	int num_devices = p->n_pdds;
4131	struct svm_range_list *svms;
4132	struct svm_range *prange;
4133	uint32_t count = 0;
4134
4135	*svm_priv_data_size = 0;
4136
4137	svms = &p->svms;
4138
4139	mutex_lock(&svms->lock);
4140	list_for_each_entry(prange, &svms->list, list) {
4141	pr_debug("prange: 0x%p start: 0x%lx\t npages: 0x%llx\t end: 0x%llx\n",
4142	prange, prange->start, prange->npages,
4143	prange->start + prange->npages - 1);
4144	count++;
4145	}
4146	mutex_unlock(lock: &svms->lock);
4147
4148	*num_svm_ranges = count;
4149	/* Only the accessbility attributes need to be queried for all the gpus
4150	* individually, remaining ones are spanned across the entire process
4151	* regardless of the various gpu nodes. Of the remaining attributes,
4152	* KFD_IOCTL_SVM_ATTR_CLR_FLAGS need not be saved.
4153	*
4154	* KFD_IOCTL_SVM_ATTR_PREFERRED_LOC
4155	* KFD_IOCTL_SVM_ATTR_PREFETCH_LOC
4156	* KFD_IOCTL_SVM_ATTR_SET_FLAGS
4157	* KFD_IOCTL_SVM_ATTR_GRANULARITY
4158	*
4159	* ** ACCESSBILITY ATTRIBUTES **
4160	* (Considered as one, type is altered during query, value is gpuid)
4161	* KFD_IOCTL_SVM_ATTR_ACCESS
4162	* KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE
4163	* KFD_IOCTL_SVM_ATTR_NO_ACCESS
4164	*/
4165	if (*num_svm_ranges > 0) {
4166	common_attr_size = sizeof(struct kfd_ioctl_svm_attribute) *
4167	nattr_common;
4168	accessibility_size = sizeof(struct kfd_ioctl_svm_attribute) *
4169	nattr_accessibility * num_devices;
4170
4171	total_size = sizeof(struct kfd_criu_svm_range_priv_data) +
4172	common_attr_size + accessibility_size;
4173
4174	*svm_priv_data_size = *num_svm_ranges * total_size;
4175	}
4176
4177	pr_debug("num_svm_ranges %u total_priv_size %llu\n", *num_svm_ranges,
4178	*svm_priv_data_size);
4179	}
4180
4181	int kfd_criu_checkpoint_svm(struct kfd_process *p,
4182	uint8_t __user *user_priv_data,
4183	uint64_t *priv_data_offset)
4184	{
4185	struct kfd_criu_svm_range_priv_data *svm_priv = NULL;
4186	struct kfd_ioctl_svm_attribute *query_attr = NULL;
4187	uint64_t svm_priv_data_size, query_attr_size = 0;
4188	int index, nattr_common = 4, ret = 0;
4189	struct svm_range_list *svms;
4190	int num_devices = p->n_pdds;
4191	struct svm_range *prange;
4192	struct mm_struct *mm;
4193
4194	svms = &p->svms;
4195
4196	mm = get_task_mm(task: p->lead_thread);
4197	if (!mm) {
4198	pr_err("failed to get mm for the target process\n");
4199	return -ESRCH;
4200	}
4201
4202	query_attr_size = sizeof(struct kfd_ioctl_svm_attribute) *
4203	(nattr_common + num_devices);
4204
4205	query_attr = kzalloc(query_attr_size, GFP_KERNEL);
4206	if (!query_attr) {
4207	ret = -ENOMEM;
4208	goto exit;
4209	}
4210
4211	query_attr[0].type = KFD_IOCTL_SVM_ATTR_PREFERRED_LOC;
4212	query_attr[1].type = KFD_IOCTL_SVM_ATTR_PREFETCH_LOC;
4213	query_attr[2].type = KFD_IOCTL_SVM_ATTR_SET_FLAGS;
4214	query_attr[3].type = KFD_IOCTL_SVM_ATTR_GRANULARITY;
4215
4216	for (index = 0; index < num_devices; index++) {
4217	struct kfd_process_device *pdd = p->pdds[index];
4218
4219	query_attr[index + nattr_common].type =
4220	KFD_IOCTL_SVM_ATTR_ACCESS;
4221	query_attr[index + nattr_common].value = pdd->user_gpu_id;
4222	}
4223
4224	svm_priv_data_size = sizeof(*svm_priv) + query_attr_size;
4225
4226	svm_priv = kzalloc(svm_priv_data_size, GFP_KERNEL);
4227	if (!svm_priv) {
4228	ret = -ENOMEM;
4229	goto exit_query;
4230	}
4231
4232	index = 0;
4233	list_for_each_entry(prange, &svms->list, list) {
4234
4235	svm_priv->object_type = KFD_CRIU_OBJECT_TYPE_SVM_RANGE;
4236	svm_priv->start_addr = prange->start;
4237	svm_priv->size = prange->npages;
4238	memcpy(&svm_priv->attrs, query_attr, query_attr_size);
4239	pr_debug("CRIU: prange: 0x%p start: 0x%lx\t npages: 0x%llx end: 0x%llx\t size: 0x%llx\n",
4240	prange, prange->start, prange->npages,
4241	prange->start + prange->npages - 1,
4242	prange->npages * PAGE_SIZE);
4243
4244	ret = svm_range_get_attr(p, mm, start: svm_priv->start_addr,
4245	size: svm_priv->size,
4246	nattr: (nattr_common + num_devices),
4247	attrs: svm_priv->attrs);
4248	if (ret) {
4249	pr_err("CRIU: failed to obtain range attributes\n");
4250	goto exit_priv;
4251	}
4252
4253	if (copy_to_user(to: user_priv_data + *priv_data_offset, from: svm_priv,
4254	n: svm_priv_data_size)) {
4255	pr_err("Failed to copy svm priv to user\n");
4256	ret = -EFAULT;
4257	goto exit_priv;
4258	}
4259
4260	*priv_data_offset += svm_priv_data_size;
4261
4262	}
4263
4264
4265	exit_priv:
4266	kfree(objp: svm_priv);
4267	exit_query:
4268	kfree(objp: query_attr);
4269	exit:
4270	mmput(mm);
4271	return ret;
4272	}
4273
4274	int
4275	svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
4276	uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs)
4277	{
4278	struct mm_struct *mm = current->mm;
4279	int r;
4280
4281	start >>= PAGE_SHIFT;
4282	size >>= PAGE_SHIFT;
4283
4284	switch (op) {
4285	case KFD_IOCTL_SVM_OP_SET_ATTR:
4286	r = svm_range_set_attr(p, mm, start, size, nattr: nattrs, attrs);
4287	break;
4288	case KFD_IOCTL_SVM_OP_GET_ATTR:
4289	r = svm_range_get_attr(p, mm, start, size, nattr: nattrs, attrs);
4290	break;
4291	default:
4292	r = -EINVAL;
4293	break;
4294	}
4295
4296	return r;
4297	}
4298