1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Copyright 2014-2022 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23
24	#include <linux/mutex.h>
25	#include <linux/log2.h>
26	#include <linux/sched.h>
27	#include <linux/sched/mm.h>
28	#include <linux/sched/task.h>
29	#include <linux/mmu_context.h>
30	#include <linux/slab.h>
31	#include <linux/notifier.h>
32	#include <linux/compat.h>
33	#include <linux/mman.h>
34	#include <linux/file.h>
35	#include <linux/pm_runtime.h>
36	#include "amdgpu_amdkfd.h"
37	#include "amdgpu.h"
38
39	struct mm_struct;
40
41	#include "kfd_priv.h"
42	#include "kfd_device_queue_manager.h"
43	#include "kfd_svm.h"
44	#include "kfd_smi_events.h"
45	#include "kfd_debug.h"
46
47	/*
48	* List of struct kfd_process (field kfd_process).
49	* Unique/indexed by mm_struct*
50	*/
51	DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
52	DEFINE_MUTEX(kfd_processes_mutex);
53
54	DEFINE_SRCU(kfd_processes_srcu);
55
56	/* For process termination handling */
57	static struct workqueue_struct *kfd_process_wq;
58
59	/* Ordered, single-threaded workqueue for restoring evicted
60	* processes. Restoring multiple processes concurrently under memory
61	* pressure can lead to processes blocking each other from validating
62	* their BOs and result in a live-lock situation where processes
63	* remain evicted indefinitely.
64	*/
65	static struct workqueue_struct *kfd_restore_wq;
66
67	static struct kfd_process *find_process(const struct task_struct *thread,
68	bool ref);
69	static void kfd_process_ref_release(struct kref *ref);
70	static struct kfd_process *create_process(const struct task_struct *thread);
71
72	static void evict_process_worker(struct work_struct *work);
73	static void restore_process_worker(struct work_struct *work);
74
75	static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
76
77	struct kfd_procfs_tree {
78	struct kobject *kobj;
79	};
80
81	static struct kfd_procfs_tree procfs;
82
83	/*
84	* Structure for SDMA activity tracking
85	*/
86	struct kfd_sdma_activity_handler_workarea {
87	struct work_struct sdma_activity_work;
88	struct kfd_process_device *pdd;
89	uint64_t sdma_activity_counter;
90	};
91
92	struct temp_sdma_queue_list {
93	uint64_t __user *rptr;
94	uint64_t sdma_val;
95	unsigned int queue_id;
96	struct list_head list;
97	};
98
99	static void kfd_sdma_activity_worker(struct work_struct *work)
100	{
101	struct kfd_sdma_activity_handler_workarea *workarea;
102	struct kfd_process_device *pdd;
103	uint64_t val;
104	struct mm_struct *mm;
105	struct queue *q;
106	struct qcm_process_device *qpd;
107	struct device_queue_manager *dqm;
108	int ret = 0;
109	struct temp_sdma_queue_list sdma_q_list;
110	struct temp_sdma_queue_list *sdma_q, *next;
111
112	workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
113	sdma_activity_work);
114
115	pdd = workarea->pdd;
116	if (!pdd)
117	return;
118	dqm = pdd->dev->dqm;
119	qpd = &pdd->qpd;
120	if (!dqm || !qpd)
121	return;
122	/*
123	* Total SDMA activity is current SDMA activity + past SDMA activity
124	* Past SDMA count is stored in pdd.
125	* To get the current activity counters for all active SDMA queues,
126	* we loop over all SDMA queues and get their counts from user-space.
127	*
128	* We cannot call get_user() with dqm_lock held as it can cause
129	* a circular lock dependency situation. To read the SDMA stats,
130	* we need to do the following:
131	*
132	* 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
133	* with dqm_lock/dqm_unlock().
134	* 2. Call get_user() for each node in temporary list without dqm_lock.
135	* Save the SDMA count for each node and also add the count to the total
136	* SDMA count counter.
137	* Its possible, during this step, a few SDMA queue nodes got deleted
138	* from the qpd->queues_list.
139	* 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
140	* If any node got deleted, its SDMA count would be captured in the sdma
141	* past activity counter. So subtract the SDMA counter stored in step 2
142	* for this node from the total SDMA count.
143	*/
144	INIT_LIST_HEAD(list: &sdma_q_list.list);
145
146	/*
147	* Create the temp list of all SDMA queues
148	*/
149	dqm_lock(dqm);
150
151	list_for_each_entry(q, &qpd->queues_list, list) {
152	if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
153	(q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
154	continue;
155
156	sdma_q = kzalloc(size: sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
157	if (!sdma_q) {
158	dqm_unlock(dqm);
159	goto cleanup;
160	}
161
162	INIT_LIST_HEAD(list: &sdma_q->list);
163	sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
164	sdma_q->queue_id = q->properties.queue_id;
165	list_add_tail(new: &sdma_q->list, head: &sdma_q_list.list);
166	}
167
168	/*
169	* If the temp list is empty, then no SDMA queues nodes were found in
170	* qpd->queues_list. Return the past activity count as the total sdma
171	* count
172	*/
173	if (list_empty(head: &sdma_q_list.list)) {
174	workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
175	dqm_unlock(dqm);
176	return;
177	}
178
179	dqm_unlock(dqm);
180
181	/*
182	* Get the usage count for each SDMA queue in temp_list.
183	*/
184	mm = get_task_mm(task: pdd->process->lead_thread);
185	if (!mm)
186	goto cleanup;
187
188	kthread_use_mm(mm);
189
190	list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
191	val = 0;
192	ret = read_sdma_queue_counter(q_rptr: sdma_q->rptr, val: &val);
193	if (ret) {
194	pr_debug("Failed to read SDMA queue active counter for queue id: %d",
195	sdma_q->queue_id);
196	} else {
197	sdma_q->sdma_val = val;
198	workarea->sdma_activity_counter += val;
199	}
200	}
201
202	kthread_unuse_mm(mm);
203	mmput(mm);
204
205	/*
206	* Do a second iteration over qpd_queues_list to check if any SDMA
207	* nodes got deleted while fetching SDMA counter.
208	*/
209	dqm_lock(dqm);
210
211	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
212
213	list_for_each_entry(q, &qpd->queues_list, list) {
214	if (list_empty(head: &sdma_q_list.list))
215	break;
216
217	if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
218	(q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
219	continue;
220
221	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
222	if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
223	(sdma_q->queue_id == q->properties.queue_id)) {
224	list_del(entry: &sdma_q->list);
225	kfree(objp: sdma_q);
226	break;
227	}
228	}
229	}
230
231	dqm_unlock(dqm);
232
233	/*
234	* If temp list is not empty, it implies some queues got deleted
235	* from qpd->queues_list during SDMA usage read. Subtract the SDMA
236	* count for each node from the total SDMA count.
237	*/
238	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
239	workarea->sdma_activity_counter -= sdma_q->sdma_val;
240	list_del(entry: &sdma_q->list);
241	kfree(objp: sdma_q);
242	}
243
244	return;
245
246	cleanup:
247	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
248	list_del(entry: &sdma_q->list);
249	kfree(objp: sdma_q);
250	}
251	}
252
253	/**
254	* kfd_get_cu_occupancy - Collect number of waves in-flight on this device
255	* by current process. Translates acquired wave count into number of compute units
256	* that are occupied.
257	*
258	* @attr: Handle of attribute that allows reporting of wave count. The attribute
259	* handle encapsulates GPU device it is associated with, thereby allowing collection
260	* of waves in flight, etc
261	* @buffer: Handle of user provided buffer updated with wave count
262	*
263	* Return: Number of bytes written to user buffer or an error value
264	*/
265	static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
266	{
267	int cu_cnt;
268	int wave_cnt;
269	int max_waves_per_cu;
270	struct kfd_node *dev = NULL;
271	struct kfd_process *proc = NULL;
272	struct kfd_process_device *pdd = NULL;
273
274	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
275	dev = pdd->dev;
276	if (dev->kfd2kgd->get_cu_occupancy == NULL)
277	return -EINVAL;
278
279	cu_cnt = 0;
280	proc = pdd->process;
281	if (pdd->qpd.queue_count == 0) {
282	pr_debug("Gpu-Id: %d has no active queues for process %d\n",
283	dev->id, proc->pasid);
284	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", cu_cnt);
285	}
286
287	/* Collect wave count from device if it supports */
288	wave_cnt = 0;
289	max_waves_per_cu = 0;
290	dev->kfd2kgd->get_cu_occupancy(dev->adev, proc->pasid, &wave_cnt,
291	&max_waves_per_cu, 0);
292
293	/* Translate wave count to number of compute units */
294	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
295	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", cu_cnt);
296	}
297
298	static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
299	char *buffer)
300	{
301	if (strcmp(attr->name, "pasid") == 0) {
302	struct kfd_process *p = container_of(attr, struct kfd_process,
303	attr_pasid);
304
305	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", p->pasid);
306	} else if (strncmp(attr->name, "vram_", 5) == 0) {
307	struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
308	attr_vram);
309	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%llu\n", READ_ONCE(pdd->vram_usage));
310	} else if (strncmp(attr->name, "sdma_", 5) == 0) {
311	struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
312	attr_sdma);
313	struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
314
315	INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
316	kfd_sdma_activity_worker);
317
318	sdma_activity_work_handler.pdd = pdd;
319	sdma_activity_work_handler.sdma_activity_counter = 0;
320
321	schedule_work(work: &sdma_activity_work_handler.sdma_activity_work);
322
323	flush_work(work: &sdma_activity_work_handler.sdma_activity_work);
324
325	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%llu\n",
326	(sdma_activity_work_handler.sdma_activity_counter)/
327	SDMA_ACTIVITY_DIVISOR);
328	} else {
329	pr_err("Invalid attribute");
330	return -EINVAL;
331	}
332
333	return 0;
334	}
335
336	static void kfd_procfs_kobj_release(struct kobject *kobj)
337	{
338	kfree(objp: kobj);
339	}
340
341	static const struct sysfs_ops kfd_procfs_ops = {
342	.show = kfd_procfs_show,
343	};
344
345	static const struct kobj_type procfs_type = {
346	.release = kfd_procfs_kobj_release,
347	.sysfs_ops = &kfd_procfs_ops,
348	};
349
350	void kfd_procfs_init(void)
351	{
352	int ret = 0;
353
354	procfs.kobj = kfd_alloc_struct(procfs.kobj);
355	if (!procfs.kobj)
356	return;
357
358	ret = kobject_init_and_add(kobj: procfs.kobj, ktype: &procfs_type,
359	parent: &kfd_device->kobj, fmt: "proc");
360	if (ret) {
361	pr_warn("Could not create procfs proc folder");
362	/* If we fail to create the procfs, clean up */
363	kfd_procfs_shutdown();
364	}
365	}
366
367	void kfd_procfs_shutdown(void)
368	{
369	if (procfs.kobj) {
370	kobject_del(kobj: procfs.kobj);
371	kobject_put(kobj: procfs.kobj);
372	procfs.kobj = NULL;
373	}
374	}
375
376	static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
377	struct attribute *attr, char *buffer)
378	{
379	struct queue *q = container_of(kobj, struct queue, kobj);
380
381	if (!strcmp(attr->name, "size"))
382	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%llu",
383	q->properties.queue_size);
384	else if (!strcmp(attr->name, "type"))
385	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d", q->properties.type);
386	else if (!strcmp(attr->name, "gpuid"))
387	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%u", q->device->id);
388	else
389	pr_err("Invalid attribute");
390
391	return 0;
392	}
393
394	static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
395	struct attribute *attr, char *buffer)
396	{
397	if (strcmp(attr->name, "evicted_ms") == 0) {
398	struct kfd_process_device *pdd = container_of(attr,
399	struct kfd_process_device,
400	attr_evict);
401	uint64_t evict_jiffies;
402
403	evict_jiffies = atomic64_read(v: &pdd->evict_duration_counter);
404
405	return snprintf(buf: buffer,
406	PAGE_SIZE,
407	fmt: "%llu\n",
408	jiffies64_to_msecs(j: evict_jiffies));
409
410	/* Sysfs handle that gets CU occupancy is per device */
411	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
412	return kfd_get_cu_occupancy(attr, buffer);
413	} else {
414	pr_err("Invalid attribute");
415	}
416
417	return 0;
418	}
419
420	static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
421	struct attribute *attr, char *buf)
422	{
423	struct kfd_process_device *pdd;
424
425	if (!strcmp(attr->name, "faults")) {
426	pdd = container_of(attr, struct kfd_process_device,
427	attr_faults);
428	return sysfs_emit(buf, fmt: "%llu\n", READ_ONCE(pdd->faults));
429	}
430	if (!strcmp(attr->name, "page_in")) {
431	pdd = container_of(attr, struct kfd_process_device,
432	attr_page_in);
433	return sysfs_emit(buf, fmt: "%llu\n", READ_ONCE(pdd->page_in));
434	}
435	if (!strcmp(attr->name, "page_out")) {
436	pdd = container_of(attr, struct kfd_process_device,
437	attr_page_out);
438	return sysfs_emit(buf, fmt: "%llu\n", READ_ONCE(pdd->page_out));
439	}
440	return 0;
441	}
442
443	static struct attribute attr_queue_size = {
444	.name = "size",
445	.mode = KFD_SYSFS_FILE_MODE
446	};
447
448	static struct attribute attr_queue_type = {
449	.name = "type",
450	.mode = KFD_SYSFS_FILE_MODE
451	};
452
453	static struct attribute attr_queue_gpuid = {
454	.name = "gpuid",
455	.mode = KFD_SYSFS_FILE_MODE
456	};
457
458	static struct attribute *procfs_queue_attrs[] = {
459	&attr_queue_size,
460	&attr_queue_type,
461	&attr_queue_gpuid,
462	NULL
463	};
464	ATTRIBUTE_GROUPS(procfs_queue);
465
466	static const struct sysfs_ops procfs_queue_ops = {
467	.show = kfd_procfs_queue_show,
468	};
469
470	static const struct kobj_type procfs_queue_type = {
471	.sysfs_ops = &procfs_queue_ops,
472	.default_groups = procfs_queue_groups,
473	};
474
475	static const struct sysfs_ops procfs_stats_ops = {
476	.show = kfd_procfs_stats_show,
477	};
478
479	static const struct kobj_type procfs_stats_type = {
480	.sysfs_ops = &procfs_stats_ops,
481	.release = kfd_procfs_kobj_release,
482	};
483
484	static const struct sysfs_ops sysfs_counters_ops = {
485	.show = kfd_sysfs_counters_show,
486	};
487
488	static const struct kobj_type sysfs_counters_type = {
489	.sysfs_ops = &sysfs_counters_ops,
490	.release = kfd_procfs_kobj_release,
491	};
492
493	int kfd_procfs_add_queue(struct queue *q)
494	{
495	struct kfd_process *proc;
496	int ret;
497
498	if (!q || !q->process)
499	return -EINVAL;
500	proc = q->process;
501
502	/* Create proc/<pid>/queues/<queue id> folder */
503	if (!proc->kobj_queues)
504	return -EFAULT;
505	ret = kobject_init_and_add(kobj: &q->kobj, ktype: &procfs_queue_type,
506	parent: proc->kobj_queues, fmt: "%u", q->properties.queue_id);
507	if (ret < 0) {
508	pr_warn("Creating proc/<pid>/queues/%u failed",
509	q->properties.queue_id);
510	kobject_put(kobj: &q->kobj);
511	return ret;
512	}
513
514	return 0;
515	}
516
517	static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
518	char *name)
519	{
520	int ret;
521
522	if (!kobj || !attr || !name)
523	return;
524
525	attr->name = name;
526	attr->mode = KFD_SYSFS_FILE_MODE;
527	sysfs_attr_init(attr);
528
529	ret = sysfs_create_file(kobj, attr);
530	if (ret)
531	pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
532	}
533
534	static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
535	{
536	int ret;
537	int i;
538	char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
539
540	if (!p || !p->kobj)
541	return;
542
543	/*
544	* Create sysfs files for each GPU:
545	* - proc/<pid>/stats_<gpuid>/
546	* - proc/<pid>/stats_<gpuid>/evicted_ms
547	* - proc/<pid>/stats_<gpuid>/cu_occupancy
548	*/
549	for (i = 0; i < p->n_pdds; i++) {
550	struct kfd_process_device *pdd = p->pdds[i];
551
552	snprintf(buf: stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
553	fmt: "stats_%u", pdd->dev->id);
554	pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
555	if (!pdd->kobj_stats)
556	return;
557
558	ret = kobject_init_and_add(kobj: pdd->kobj_stats,
559	ktype: &procfs_stats_type,
560	parent: p->kobj,
561	fmt: stats_dir_filename);
562
563	if (ret) {
564	pr_warn("Creating KFD proc/stats_%s folder failed",
565	stats_dir_filename);
566	kobject_put(kobj: pdd->kobj_stats);
567	pdd->kobj_stats = NULL;
568	return;
569	}
570
571	kfd_sysfs_create_file(kobj: pdd->kobj_stats, attr: &pdd->attr_evict,
572	name: "evicted_ms");
573	/* Add sysfs file to report compute unit occupancy */
574	if (pdd->dev->kfd2kgd->get_cu_occupancy)
575	kfd_sysfs_create_file(kobj: pdd->kobj_stats,
576	attr: &pdd->attr_cu_occupancy,
577	name: "cu_occupancy");
578	}
579	}
580
581	static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
582	{
583	int ret = 0;
584	int i;
585	char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
586
587	if (!p || !p->kobj)
588	return;
589
590	/*
591	* Create sysfs files for each GPU which supports SVM
592	* - proc/<pid>/counters_<gpuid>/
593	* - proc/<pid>/counters_<gpuid>/faults
594	* - proc/<pid>/counters_<gpuid>/page_in
595	* - proc/<pid>/counters_<gpuid>/page_out
596	*/
597	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
598	struct kfd_process_device *pdd = p->pdds[i];
599	struct kobject *kobj_counters;
600
601	snprintf(buf: counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
602	fmt: "counters_%u", pdd->dev->id);
603	kobj_counters = kfd_alloc_struct(kobj_counters);
604	if (!kobj_counters)
605	return;
606
607	ret = kobject_init_and_add(kobj: kobj_counters, ktype: &sysfs_counters_type,
608	parent: p->kobj, fmt: counters_dir_filename);
609	if (ret) {
610	pr_warn("Creating KFD proc/%s folder failed",
611	counters_dir_filename);
612	kobject_put(kobj: kobj_counters);
613	return;
614	}
615
616	pdd->kobj_counters = kobj_counters;
617	kfd_sysfs_create_file(kobj: kobj_counters, attr: &pdd->attr_faults,
618	name: "faults");
619	kfd_sysfs_create_file(kobj: kobj_counters, attr: &pdd->attr_page_in,
620	name: "page_in");
621	kfd_sysfs_create_file(kobj: kobj_counters, attr: &pdd->attr_page_out,
622	name: "page_out");
623	}
624	}
625
626	static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
627	{
628	int i;
629
630	if (!p || !p->kobj)
631	return;
632
633	/*
634	* Create sysfs files for each GPU:
635	* - proc/<pid>/vram_<gpuid>
636	* - proc/<pid>/sdma_<gpuid>
637	*/
638	for (i = 0; i < p->n_pdds; i++) {
639	struct kfd_process_device *pdd = p->pdds[i];
640
641	snprintf(buf: pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, fmt: "vram_%u",
642	pdd->dev->id);
643	kfd_sysfs_create_file(kobj: p->kobj, attr: &pdd->attr_vram,
644	name: pdd->vram_filename);
645
646	snprintf(buf: pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, fmt: "sdma_%u",
647	pdd->dev->id);
648	kfd_sysfs_create_file(kobj: p->kobj, attr: &pdd->attr_sdma,
649	name: pdd->sdma_filename);
650	}
651	}
652
653	void kfd_procfs_del_queue(struct queue *q)
654	{
655	if (!q)
656	return;
657
658	kobject_del(kobj: &q->kobj);
659	kobject_put(kobj: &q->kobj);
660	}
661
662	int kfd_process_create_wq(void)
663	{
664	if (!kfd_process_wq)
665	kfd_process_wq = alloc_workqueue(fmt: "kfd_process_wq", flags: 0, max_active: 0);
666	if (!kfd_restore_wq)
667	kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
668	WQ_FREEZABLE);
669
670	if (!kfd_process_wq || !kfd_restore_wq) {
671	kfd_process_destroy_wq();
672	return -ENOMEM;
673	}
674
675	return 0;
676	}
677
678	void kfd_process_destroy_wq(void)
679	{
680	if (kfd_process_wq) {
681	destroy_workqueue(wq: kfd_process_wq);
682	kfd_process_wq = NULL;
683	}
684	if (kfd_restore_wq) {
685	destroy_workqueue(wq: kfd_restore_wq);
686	kfd_restore_wq = NULL;
687	}
688	}
689
690	static void kfd_process_free_gpuvm(struct kgd_mem *mem,
691	struct kfd_process_device *pdd, void **kptr)
692	{
693	struct kfd_node *dev = pdd->dev;
694
695	if (kptr && *kptr) {
696	amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
697	*kptr = NULL;
698	}
699
700	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(adev: dev->adev, mem, drm_priv: pdd->drm_priv);
701	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: dev->adev, mem, drm_priv: pdd->drm_priv,
702	NULL);
703	}
704
705	/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
706	* This function should be only called right after the process
707	* is created and when kfd_processes_mutex is still being held
708	* to avoid concurrency. Because of that exclusiveness, we do
709	* not need to take p->mutex.
710	*/
711	static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
712	uint64_t gpu_va, uint32_t size,
713	uint32_t flags, struct kgd_mem **mem, void **kptr)
714	{
715	struct kfd_node *kdev = pdd->dev;
716	int err;
717
718	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(adev: kdev->adev, va: gpu_va, size,
719	drm_priv: pdd->drm_priv, mem, NULL,
720	flags, criu_resume: false);
721	if (err)
722	goto err_alloc_mem;
723
724	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(adev: kdev->adev, mem: *mem,
725	drm_priv: pdd->drm_priv);
726	if (err)
727	goto err_map_mem;
728
729	err = amdgpu_amdkfd_gpuvm_sync_memory(adev: kdev->adev, mem: *mem, intr: true);
730	if (err) {
731	pr_debug("Sync memory failed, wait interrupted by user signal\n");
732	goto sync_memory_failed;
733	}
734
735	if (kptr) {
736	err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
737	mem: (struct kgd_mem *)*mem, kptr, NULL);
738	if (err) {
739	pr_debug("Map GTT BO to kernel failed\n");
740	goto sync_memory_failed;
741	}
742	}
743
744	return err;
745
746	sync_memory_failed:
747	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(adev: kdev->adev, mem: *mem, drm_priv: pdd->drm_priv);
748
749	err_map_mem:
750	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: kdev->adev, mem: *mem, drm_priv: pdd->drm_priv,
751	NULL);
752	err_alloc_mem:
753	*mem = NULL;
754	*kptr = NULL;
755	return err;
756	}
757
758	/* kfd_process_device_reserve_ib_mem - Reserve memory inside the
759	* process for IB usage The memory reserved is for KFD to submit
760	* IB to AMDGPU from kernel. If the memory is reserved
761	* successfully, ib_kaddr will have the CPU/kernel
762	* address. Check ib_kaddr before accessing the memory.
763	*/
764	static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
765	{
766	struct qcm_process_device *qpd = &pdd->qpd;
767	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
768	KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
769	KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
770	KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
771	struct kgd_mem *mem;
772	void *kaddr;
773	int ret;
774
775	if (qpd->ib_kaddr || !qpd->ib_base)
776	return 0;
777
778	/* ib_base is only set for dGPU */
779	ret = kfd_process_alloc_gpuvm(pdd, gpu_va: qpd->ib_base, PAGE_SIZE, flags,
780	mem: &mem, kptr: &kaddr);
781	if (ret)
782	return ret;
783
784	qpd->ib_mem = mem;
785	qpd->ib_kaddr = kaddr;
786
787	return 0;
788	}
789
790	static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
791	{
792	struct qcm_process_device *qpd = &pdd->qpd;
793
794	if (!qpd->ib_kaddr || !qpd->ib_base)
795	return;
796
797	kfd_process_free_gpuvm(mem: qpd->ib_mem, pdd, kptr: &qpd->ib_kaddr);
798	}
799
800	struct kfd_process *kfd_create_process(struct task_struct *thread)
801	{
802	struct kfd_process *process;
803	int ret;
804
805	if (!(thread->mm && mmget_not_zero(mm: thread->mm)))
806	return ERR_PTR(error: -EINVAL);
807
808	/* Only the pthreads threading model is supported. */
809	if (thread->group_leader->mm != thread->mm) {
810	mmput(thread->mm);
811	return ERR_PTR(error: -EINVAL);
812	}
813
814	/*
815	* take kfd processes mutex before starting of process creation
816	* so there won't be a case where two threads of the same process
817	* create two kfd_process structures
818	*/
819	mutex_lock(&kfd_processes_mutex);
820
821	if (kfd_is_locked()) {
822	pr_debug("KFD is locked! Cannot create process");
823	process = ERR_PTR(error: -EINVAL);
824	goto out;
825	}
826
827	/* A prior open of /dev/kfd could have already created the process. */
828	process = find_process(thread, ref: false);
829	if (process) {
830	pr_debug("Process already found\n");
831	} else {
832	process = create_process(thread);
833	if (IS_ERR(ptr: process))
834	goto out;
835
836	if (!procfs.kobj)
837	goto out;
838
839	process->kobj = kfd_alloc_struct(process->kobj);
840	if (!process->kobj) {
841	pr_warn("Creating procfs kobject failed");
842	goto out;
843	}
844	ret = kobject_init_and_add(kobj: process->kobj, ktype: &procfs_type,
845	parent: procfs.kobj, fmt: "%d",
846	(int)process->lead_thread->pid);
847	if (ret) {
848	pr_warn("Creating procfs pid directory failed");
849	kobject_put(kobj: process->kobj);
850	goto out;
851	}
852
853	kfd_sysfs_create_file(kobj: process->kobj, attr: &process->attr_pasid,
854	name: "pasid");
855
856	process->kobj_queues = kobject_create_and_add(name: "queues",
857	parent: process->kobj);
858	if (!process->kobj_queues)
859	pr_warn("Creating KFD proc/queues folder failed");
860
861	kfd_procfs_add_sysfs_stats(p: process);
862	kfd_procfs_add_sysfs_files(p: process);
863	kfd_procfs_add_sysfs_counters(p: process);
864
865	init_waitqueue_head(&process->wait_irq_drain);
866	}
867	out:
868	if (!IS_ERR(ptr: process))
869	kref_get(kref: &process->ref);
870	mutex_unlock(lock: &kfd_processes_mutex);
871	mmput(thread->mm);
872
873	return process;
874	}
875
876	struct kfd_process *kfd_get_process(const struct task_struct *thread)
877	{
878	struct kfd_process *process;
879
880	if (!thread->mm)
881	return ERR_PTR(error: -EINVAL);
882
883	/* Only the pthreads threading model is supported. */
884	if (thread->group_leader->mm != thread->mm)
885	return ERR_PTR(error: -EINVAL);
886
887	process = find_process(thread, ref: false);
888	if (!process)
889	return ERR_PTR(error: -EINVAL);
890
891	return process;
892	}
893
894	static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
895	{
896	struct kfd_process *process;
897
898	hash_for_each_possible_rcu(kfd_processes_table, process,
899	kfd_processes, (uintptr_t)mm)
900	if (process->mm == mm)
901	return process;
902
903	return NULL;
904	}
905
906	static struct kfd_process *find_process(const struct task_struct *thread,
907	bool ref)
908	{
909	struct kfd_process *p;
910	int idx;
911
912	idx = srcu_read_lock(ssp: &kfd_processes_srcu);
913	p = find_process_by_mm(mm: thread->mm);
914	if (p && ref)
915	kref_get(kref: &p->ref);
916	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
917
918	return p;
919	}
920
921	void kfd_unref_process(struct kfd_process *p)
922	{
923	kref_put(kref: &p->ref, release: kfd_process_ref_release);
924	}
925
926	/* This increments the process->ref counter. */
927	struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
928	{
929	struct task_struct *task = NULL;
930	struct kfd_process *p = NULL;
931
932	if (!pid) {
933	task = current;
934	get_task_struct(t: task);
935	} else {
936	task = get_pid_task(pid, PIDTYPE_PID);
937	}
938
939	if (task) {
940	p = find_process(thread: task, ref: true);
941	put_task_struct(t: task);
942	}
943
944	return p;
945	}
946
947	static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
948	{
949	struct kfd_process *p = pdd->process;
950	void *mem;
951	int id;
952	int i;
953
954	/*
955	* Remove all handles from idr and release appropriate
956	* local memory object
957	*/
958	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
959
960	for (i = 0; i < p->n_pdds; i++) {
961	struct kfd_process_device *peer_pdd = p->pdds[i];
962
963	if (!peer_pdd->drm_priv)
964	continue;
965	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
966	adev: peer_pdd->dev->adev, mem, drm_priv: peer_pdd->drm_priv);
967	}
968
969	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev, mem,
970	drm_priv: pdd->drm_priv, NULL);
971	kfd_process_device_remove_obj_handle(pdd, handle: id);
972	}
973	}
974
975	/*
976	* Just kunmap and unpin signal BO here. It will be freed in
977	* kfd_process_free_outstanding_kfd_bos()
978	*/
979	static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
980	{
981	struct kfd_process_device *pdd;
982	struct kfd_node *kdev;
983	void *mem;
984
985	kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
986	if (!kdev)
987	return;
988
989	mutex_lock(&p->mutex);
990
991	pdd = kfd_get_process_device_data(dev: kdev, p);
992	if (!pdd)
993	goto out;
994
995	mem = kfd_process_device_translate_handle(
996	p: pdd, GET_IDR_HANDLE(p->signal_handle));
997	if (!mem)
998	goto out;
999
1000	amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1001
1002	out:
1003	mutex_unlock(lock: &p->mutex);
1004	}
1005
1006	static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1007	{
1008	int i;
1009
1010	for (i = 0; i < p->n_pdds; i++)
1011	kfd_process_device_free_bos(pdd: p->pdds[i]);
1012	}
1013
1014	static void kfd_process_destroy_pdds(struct kfd_process *p)
1015	{
1016	int i;
1017
1018	for (i = 0; i < p->n_pdds; i++) {
1019	struct kfd_process_device *pdd = p->pdds[i];
1020
1021	pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
1022	pdd->dev->id, p->pasid);
1023
1024	kfd_process_device_destroy_cwsr_dgpu(pdd);
1025	kfd_process_device_destroy_ib_mem(pdd);
1026
1027	if (pdd->drm_file) {
1028	amdgpu_amdkfd_gpuvm_release_process_vm(
1029	adev: pdd->dev->adev, drm_priv: pdd->drm_priv);
1030	fput(pdd->drm_file);
1031	}
1032
1033	if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1034	free_pages(addr: (unsigned long)pdd->qpd.cwsr_kaddr,
1035	order: get_order(KFD_CWSR_TBA_TMA_SIZE));
1036
1037	idr_destroy(&pdd->alloc_idr);
1038
1039	kfd_free_process_doorbells(kfd: pdd->dev->kfd, pdd);
1040
1041	if (pdd->dev->kfd->shared_resources.enable_mes)
1042	amdgpu_amdkfd_free_gtt_mem(adev: pdd->dev->adev,
1043	mem_obj: pdd->proc_ctx_bo);
1044	/*
1045	* before destroying pdd, make sure to report availability
1046	* for auto suspend
1047	*/
1048	if (pdd->runtime_inuse) {
1049	pm_runtime_mark_last_busy(dev: adev_to_drm(adev: pdd->dev->adev)->dev);
1050	pm_runtime_put_autosuspend(dev: adev_to_drm(adev: pdd->dev->adev)->dev);
1051	pdd->runtime_inuse = false;
1052	}
1053
1054	kfree(objp: pdd);
1055	p->pdds[i] = NULL;
1056	}
1057	p->n_pdds = 0;
1058	}
1059
1060	static void kfd_process_remove_sysfs(struct kfd_process *p)
1061	{
1062	struct kfd_process_device *pdd;
1063	int i;
1064
1065	if (!p->kobj)
1066	return;
1067
1068	sysfs_remove_file(kobj: p->kobj, attr: &p->attr_pasid);
1069	kobject_del(kobj: p->kobj_queues);
1070	kobject_put(kobj: p->kobj_queues);
1071	p->kobj_queues = NULL;
1072
1073	for (i = 0; i < p->n_pdds; i++) {
1074	pdd = p->pdds[i];
1075
1076	sysfs_remove_file(kobj: p->kobj, attr: &pdd->attr_vram);
1077	sysfs_remove_file(kobj: p->kobj, attr: &pdd->attr_sdma);
1078
1079	sysfs_remove_file(kobj: pdd->kobj_stats, attr: &pdd->attr_evict);
1080	if (pdd->dev->kfd2kgd->get_cu_occupancy)
1081	sysfs_remove_file(kobj: pdd->kobj_stats,
1082	attr: &pdd->attr_cu_occupancy);
1083	kobject_del(kobj: pdd->kobj_stats);
1084	kobject_put(kobj: pdd->kobj_stats);
1085	pdd->kobj_stats = NULL;
1086	}
1087
1088	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1089	pdd = p->pdds[i];
1090
1091	sysfs_remove_file(kobj: pdd->kobj_counters, attr: &pdd->attr_faults);
1092	sysfs_remove_file(kobj: pdd->kobj_counters, attr: &pdd->attr_page_in);
1093	sysfs_remove_file(kobj: pdd->kobj_counters, attr: &pdd->attr_page_out);
1094	kobject_del(kobj: pdd->kobj_counters);
1095	kobject_put(kobj: pdd->kobj_counters);
1096	pdd->kobj_counters = NULL;
1097	}
1098
1099	kobject_del(kobj: p->kobj);
1100	kobject_put(kobj: p->kobj);
1101	p->kobj = NULL;
1102	}
1103
1104	/* No process locking is needed in this function, because the process
1105	* is not findable any more. We must assume that no other thread is
1106	* using it any more, otherwise we couldn't safely free the process
1107	* structure in the end.
1108	*/
1109	static void kfd_process_wq_release(struct work_struct *work)
1110	{
1111	struct kfd_process *p = container_of(work, struct kfd_process,
1112	release_work);
1113	struct dma_fence *ef;
1114
1115	kfd_process_dequeue_from_all_devices(p);
1116	pqm_uninit(pqm: &p->pqm);
1117
1118	/* Signal the eviction fence after user mode queues are
1119	* destroyed. This allows any BOs to be freed without
1120	* triggering pointless evictions or waiting for fences.
1121	*/
1122	synchronize_rcu();
1123	ef = rcu_access_pointer(p->ef);
1124	dma_fence_signal(fence: ef);
1125
1126	kfd_process_remove_sysfs(p);
1127
1128	kfd_process_kunmap_signal_bo(p);
1129	kfd_process_free_outstanding_kfd_bos(p);
1130	svm_range_list_fini(p);
1131
1132	kfd_process_destroy_pdds(p);
1133	dma_fence_put(fence: ef);
1134
1135	kfd_event_free_process(p);
1136
1137	kfd_pasid_free(pasid: p->pasid);
1138	mutex_destroy(lock: &p->mutex);
1139
1140	put_task_struct(t: p->lead_thread);
1141
1142	kfree(objp: p);
1143	}
1144
1145	static void kfd_process_ref_release(struct kref *ref)
1146	{
1147	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1148
1149	INIT_WORK(&p->release_work, kfd_process_wq_release);
1150	queue_work(wq: kfd_process_wq, work: &p->release_work);
1151	}
1152
1153	static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1154	{
1155	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1156	struct kfd_process *p = find_process_by_mm(mm);
1157
1158	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1159
1160	return p ? &p->mmu_notifier : ERR_PTR(error: -ESRCH);
1161	}
1162
1163	static void kfd_process_free_notifier(struct mmu_notifier *mn)
1164	{
1165	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1166	}
1167
1168	static void kfd_process_notifier_release_internal(struct kfd_process *p)
1169	{
1170	int i;
1171
1172	cancel_delayed_work_sync(dwork: &p->eviction_work);
1173	cancel_delayed_work_sync(dwork: &p->restore_work);
1174
1175	for (i = 0; i < p->n_pdds; i++) {
1176	struct kfd_process_device *pdd = p->pdds[i];
1177
1178	/* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1179	if (!kfd_dbg_is_rlc_restore_supported(dev: pdd->dev) && p->runtime_info.ttmp_setup)
1180	amdgpu_gfx_off_ctrl(adev: pdd->dev->adev, enable: true);
1181	}
1182
1183	/* Indicate to other users that MM is no longer valid */
1184	p->mm = NULL;
1185	kfd_dbg_trap_disable(target: p);
1186
1187	if (atomic_read(v: &p->debugged_process_count) > 0) {
1188	struct kfd_process *target;
1189	unsigned int temp;
1190	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1191
1192	hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1193	if (target->debugger_process && target->debugger_process == p) {
1194	mutex_lock_nested(lock: &target->mutex, subclass: 1);
1195	kfd_dbg_trap_disable(target);
1196	mutex_unlock(lock: &target->mutex);
1197	if (atomic_read(v: &p->debugged_process_count) == 0)
1198	break;
1199	}
1200	}
1201
1202	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1203	}
1204
1205	mmu_notifier_put(subscription: &p->mmu_notifier);
1206	}
1207
1208	static void kfd_process_notifier_release(struct mmu_notifier *mn,
1209	struct mm_struct *mm)
1210	{
1211	struct kfd_process *p;
1212
1213	/*
1214	* The kfd_process structure can not be free because the
1215	* mmu_notifier srcu is read locked
1216	*/
1217	p = container_of(mn, struct kfd_process, mmu_notifier);
1218	if (WARN_ON(p->mm != mm))
1219	return;
1220
1221	mutex_lock(&kfd_processes_mutex);
1222	/*
1223	* Do early return if table is empty.
1224	*
1225	* This could potentially happen if this function is called concurrently
1226	* by mmu_notifier and by kfd_cleanup_pocesses.
1227	*
1228	*/
1229	if (hash_empty(kfd_processes_table)) {
1230	mutex_unlock(lock: &kfd_processes_mutex);
1231	return;
1232	}
1233	hash_del_rcu(node: &p->kfd_processes);
1234	mutex_unlock(lock: &kfd_processes_mutex);
1235	synchronize_srcu(ssp: &kfd_processes_srcu);
1236
1237	kfd_process_notifier_release_internal(p);
1238	}
1239
1240	static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1241	.release = kfd_process_notifier_release,
1242	.alloc_notifier = kfd_process_alloc_notifier,
1243	.free_notifier = kfd_process_free_notifier,
1244	};
1245
1246	/*
1247	* This code handles the case when driver is being unloaded before all
1248	* mm_struct are released. We need to safely free the kfd_process and
1249	* avoid race conditions with mmu_notifier that might try to free them.
1250	*
1251	*/
1252	void kfd_cleanup_processes(void)
1253	{
1254	struct kfd_process *p;
1255	struct hlist_node *p_temp;
1256	unsigned int temp;
1257	HLIST_HEAD(cleanup_list);
1258
1259	/*
1260	* Move all remaining kfd_process from the process table to a
1261	* temp list for processing. Once done, callback from mmu_notifier
1262	* release will not see the kfd_process in the table and do early return,
1263	* avoiding double free issues.
1264	*/
1265	mutex_lock(&kfd_processes_mutex);
1266	hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1267	hash_del_rcu(node: &p->kfd_processes);
1268	synchronize_srcu(ssp: &kfd_processes_srcu);
1269	hlist_add_head(n: &p->kfd_processes, h: &cleanup_list);
1270	}
1271	mutex_unlock(lock: &kfd_processes_mutex);
1272
1273	hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1274	kfd_process_notifier_release_internal(p);
1275
1276	/*
1277	* Ensures that all outstanding free_notifier get called, triggering
1278	* the release of the kfd_process struct.
1279	*/
1280	mmu_notifier_synchronize();
1281	}
1282
1283	int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1284	{
1285	unsigned long offset;
1286	int i;
1287
1288	if (p->has_cwsr)
1289	return 0;
1290
1291	for (i = 0; i < p->n_pdds; i++) {
1292	struct kfd_node *dev = p->pdds[i]->dev;
1293	struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1294
1295	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1296	continue;
1297
1298	offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1299	qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1300	KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1301	MAP_SHARED, offset);
1302
1303	if (IS_ERR_VALUE(qpd->tba_addr)) {
1304	int err = qpd->tba_addr;
1305
1306	pr_err("Failure to set tba address. error %d.\n", err);
1307	qpd->tba_addr = 0;
1308	qpd->cwsr_kaddr = NULL;
1309	return err;
1310	}
1311
1312	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1313
1314	kfd_process_set_trap_debug_flag(qpd, enabled: p->debug_trap_enabled);
1315
1316	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1317	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1318	qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1319	}
1320
1321	p->has_cwsr = true;
1322
1323	return 0;
1324	}
1325
1326	static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1327	{
1328	struct kfd_node *dev = pdd->dev;
1329	struct qcm_process_device *qpd = &pdd->qpd;
1330	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1331	| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1332	| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1333	struct kgd_mem *mem;
1334	void *kaddr;
1335	int ret;
1336
1337	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1338	return 0;
1339
1340	/* cwsr_base is only set for dGPU */
1341	ret = kfd_process_alloc_gpuvm(pdd, gpu_va: qpd->cwsr_base,
1342	KFD_CWSR_TBA_TMA_SIZE, flags, mem: &mem, kptr: &kaddr);
1343	if (ret)
1344	return ret;
1345
1346	qpd->cwsr_mem = mem;
1347	qpd->cwsr_kaddr = kaddr;
1348	qpd->tba_addr = qpd->cwsr_base;
1349
1350	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1351
1352	kfd_process_set_trap_debug_flag(qpd: &pdd->qpd,
1353	enabled: pdd->process->debug_trap_enabled);
1354
1355	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1356	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1357	qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1358
1359	return 0;
1360	}
1361
1362	static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1363	{
1364	struct kfd_node *dev = pdd->dev;
1365	struct qcm_process_device *qpd = &pdd->qpd;
1366
1367	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1368	return;
1369
1370	kfd_process_free_gpuvm(mem: qpd->cwsr_mem, pdd, kptr: &qpd->cwsr_kaddr);
1371	}
1372
1373	void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1374	uint64_t tba_addr,
1375	uint64_t tma_addr)
1376	{
1377	if (qpd->cwsr_kaddr) {
1378	/* KFD trap handler is bound, record as second-level TBA/TMA
1379	* in first-level TMA. First-level trap will jump to second.
1380	*/
1381	uint64_t *tma =
1382	(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1383	tma[0] = tba_addr;
1384	tma[1] = tma_addr;
1385	} else {
1386	/* No trap handler bound, bind as first-level TBA/TMA. */
1387	qpd->tba_addr = tba_addr;
1388	qpd->tma_addr = tma_addr;
1389	}
1390	}
1391
1392	bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1393	{
1394	int i;
1395
1396	/* On most GFXv9 GPUs, the retry mode in the SQ must match the
1397	* boot time retry setting. Mixing processes with different
1398	* XNACK/retry settings can hang the GPU.
1399	*
1400	* Different GPUs can have different noretry settings depending
1401	* on HW bugs or limitations. We need to find at least one
1402	* XNACK mode for this process that's compatible with all GPUs.
1403	* Fortunately GPUs with retry enabled (noretry=0) can run code
1404	* built for XNACK-off. On GFXv9 it may perform slower.
1405	*
1406	* Therefore applications built for XNACK-off can always be
1407	* supported and will be our fallback if any GPU does not
1408	* support retry.
1409	*/
1410	for (i = 0; i < p->n_pdds; i++) {
1411	struct kfd_node *dev = p->pdds[i]->dev;
1412
1413	/* Only consider GFXv9 and higher GPUs. Older GPUs don't
1414	* support the SVM APIs and don't need to be considered
1415	* for the XNACK mode selection.
1416	*/
1417	if (!KFD_IS_SOC15(dev))
1418	continue;
1419	/* Aldebaran can always support XNACK because it can support
1420	* per-process XNACK mode selection. But let the dev->noretry
1421	* setting still influence the default XNACK mode.
1422	*/
1423	if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1424	if (!amdgpu_sriov_xnack_support(adev: dev->kfd->adev)) {
1425	pr_debug("SRIOV platform xnack not supported\n");
1426	return false;
1427	}
1428	continue;
1429	}
1430
1431	/* GFXv10 and later GPUs do not support shader preemption
1432	* during page faults. This can lead to poor QoS for queue
1433	* management and memory-manager-related preemptions or
1434	* even deadlocks.
1435	*/
1436	if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1437	return false;
1438
1439	if (dev->kfd->noretry)
1440	return false;
1441	}
1442
1443	return true;
1444	}
1445
1446	void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1447	bool enabled)
1448	{
1449	if (qpd->cwsr_kaddr) {
1450	uint64_t *tma =
1451	(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1452	tma[2] = enabled;
1453	}
1454	}
1455
1456	/*
1457	* On return the kfd_process is fully operational and will be freed when the
1458	* mm is released
1459	*/
1460	static struct kfd_process *create_process(const struct task_struct *thread)
1461	{
1462	struct kfd_process *process;
1463	struct mmu_notifier *mn;
1464	int err = -ENOMEM;
1465
1466	process = kzalloc(size: sizeof(*process), GFP_KERNEL);
1467	if (!process)
1468	goto err_alloc_process;
1469
1470	kref_init(kref: &process->ref);
1471	mutex_init(&process->mutex);
1472	process->mm = thread->mm;
1473	process->lead_thread = thread->group_leader;
1474	process->n_pdds = 0;
1475	process->queues_paused = false;
1476	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1477	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1478	process->last_restore_timestamp = get_jiffies_64();
1479	err = kfd_event_init_process(p: process);
1480	if (err)
1481	goto err_event_init;
1482	process->is_32bit_user_mode = in_compat_syscall();
1483	process->debug_trap_enabled = false;
1484	process->debugger_process = NULL;
1485	process->exception_enable_mask = 0;
1486	atomic_set(v: &process->debugged_process_count, i: 0);
1487	sema_init(sem: &process->runtime_enable_sema, val: 0);
1488
1489	process->pasid = kfd_pasid_alloc();
1490	if (process->pasid == 0) {
1491	err = -ENOSPC;
1492	goto err_alloc_pasid;
1493	}
1494
1495	err = pqm_init(pqm: &process->pqm, p: process);
1496	if (err != 0)
1497	goto err_process_pqm_init;
1498
1499	/* init process apertures*/
1500	err = kfd_init_apertures(process);
1501	if (err != 0)
1502	goto err_init_apertures;
1503
1504	/* Check XNACK support after PDDs are created in kfd_init_apertures */
1505	process->xnack_enabled = kfd_process_xnack_mode(p: process, supported: false);
1506
1507	err = svm_range_list_init(p: process);
1508	if (err)
1509	goto err_init_svm_range_list;
1510
1511	/* alloc_notifier needs to find the process in the hash table */
1512	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1513	(uintptr_t)process->mm);
1514
1515	/* Avoid free_notifier to start kfd_process_wq_release if
1516	* mmu_notifier_get failed because of pending signal.
1517	*/
1518	kref_get(kref: &process->ref);
1519
1520	/* MMU notifier registration must be the last call that can fail
1521	* because after this point we cannot unwind the process creation.
1522	* After this point, mmu_notifier_put will trigger the cleanup by
1523	* dropping the last process reference in the free_notifier.
1524	*/
1525	mn = mmu_notifier_get(ops: &kfd_process_mmu_notifier_ops, mm: process->mm);
1526	if (IS_ERR(ptr: mn)) {
1527	err = PTR_ERR(ptr: mn);
1528	goto err_register_notifier;
1529	}
1530	BUG_ON(mn != &process->mmu_notifier);
1531
1532	kfd_unref_process(p: process);
1533	get_task_struct(t: process->lead_thread);
1534
1535	INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1536
1537	return process;
1538
1539	err_register_notifier:
1540	hash_del_rcu(node: &process->kfd_processes);
1541	svm_range_list_fini(p: process);
1542	err_init_svm_range_list:
1543	kfd_process_free_outstanding_kfd_bos(p: process);
1544	kfd_process_destroy_pdds(p: process);
1545	err_init_apertures:
1546	pqm_uninit(pqm: &process->pqm);
1547	err_process_pqm_init:
1548	kfd_pasid_free(pasid: process->pasid);
1549	err_alloc_pasid:
1550	kfd_event_free_process(p: process);
1551	err_event_init:
1552	mutex_destroy(lock: &process->mutex);
1553	kfree(objp: process);
1554	err_alloc_process:
1555	return ERR_PTR(error: err);
1556	}
1557
1558	struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1559	struct kfd_process *p)
1560	{
1561	int i;
1562
1563	for (i = 0; i < p->n_pdds; i++)
1564	if (p->pdds[i]->dev == dev)
1565	return p->pdds[i];
1566
1567	return NULL;
1568	}
1569
1570	struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1571	struct kfd_process *p)
1572	{
1573	struct kfd_process_device *pdd = NULL;
1574	int retval = 0;
1575
1576	if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1577	return NULL;
1578	pdd = kzalloc(size: sizeof(*pdd), GFP_KERNEL);
1579	if (!pdd)
1580	return NULL;
1581
1582	pdd->dev = dev;
1583	INIT_LIST_HEAD(list: &pdd->qpd.queues_list);
1584	INIT_LIST_HEAD(list: &pdd->qpd.priv_queue_list);
1585	pdd->qpd.dqm = dev->dqm;
1586	pdd->qpd.pqm = &p->pqm;
1587	pdd->qpd.evicted = 0;
1588	pdd->qpd.mapped_gws_queue = false;
1589	pdd->process = p;
1590	pdd->bound = PDD_UNBOUND;
1591	pdd->already_dequeued = false;
1592	pdd->runtime_inuse = false;
1593	pdd->vram_usage = 0;
1594	pdd->sdma_past_activity_counter = 0;
1595	pdd->user_gpu_id = dev->id;
1596	atomic64_set(v: &pdd->evict_duration_counter, i: 0);
1597
1598	if (dev->kfd->shared_resources.enable_mes) {
1599	retval = amdgpu_amdkfd_alloc_gtt_mem(adev: dev->adev,
1600	AMDGPU_MES_PROC_CTX_SIZE,
1601	mem_obj: &pdd->proc_ctx_bo,
1602	gpu_addr: &pdd->proc_ctx_gpu_addr,
1603	cpu_ptr: &pdd->proc_ctx_cpu_ptr,
1604	mqd_gfx9: false);
1605	if (retval) {
1606	pr_err("failed to allocate process context bo\n");
1607	goto err_free_pdd;
1608	}
1609	memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
1610	}
1611
1612	p->pdds[p->n_pdds++] = pdd;
1613	if (kfd_dbg_is_per_vmid_supported(dev: pdd->dev))
1614	pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1615	pdd->dev->adev,
1616	false,
1617	0);
1618
1619	/* Init idr used for memory handle translation */
1620	idr_init(idr: &pdd->alloc_idr);
1621
1622	return pdd;
1623
1624	err_free_pdd:
1625	kfree(objp: pdd);
1626	return NULL;
1627	}
1628
1629	/**
1630	* kfd_process_device_init_vm - Initialize a VM for a process-device
1631	*
1632	* @pdd: The process-device
1633	* @drm_file: Optional pointer to a DRM file descriptor
1634	*
1635	* If @drm_file is specified, it will be used to acquire the VM from
1636	* that file descriptor. If successful, the @pdd takes ownership of
1637	* the file descriptor.
1638	*
1639	* If @drm_file is NULL, a new VM is created.
1640	*
1641	* Returns 0 on success, -errno on failure.
1642	*/
1643	int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1644	struct file *drm_file)
1645	{
1646	struct amdgpu_fpriv *drv_priv;
1647	struct amdgpu_vm *avm;
1648	struct kfd_process *p;
1649	struct dma_fence *ef;
1650	struct kfd_node *dev;
1651	int ret;
1652
1653	if (!drm_file)
1654	return -EINVAL;
1655
1656	if (pdd->drm_priv)
1657	return -EBUSY;
1658
1659	ret = amdgpu_file_to_fpriv(filp: drm_file, fpriv: &drv_priv);
1660	if (ret)
1661	return ret;
1662	avm = &drv_priv->vm;
1663
1664	p = pdd->process;
1665	dev = pdd->dev;
1666
1667	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(adev: dev->adev, avm,
1668	process_info: &p->kgd_process_info,
1669	ef: &ef);
1670	if (ret) {
1671	pr_err("Failed to create process VM object\n");
1672	return ret;
1673	}
1674	RCU_INIT_POINTER(p->ef, ef);
1675	pdd->drm_priv = drm_file->private_data;
1676
1677	ret = kfd_process_device_reserve_ib_mem(pdd);
1678	if (ret)
1679	goto err_reserve_ib_mem;
1680	ret = kfd_process_device_init_cwsr_dgpu(pdd);
1681	if (ret)
1682	goto err_init_cwsr;
1683
1684	ret = amdgpu_amdkfd_gpuvm_set_vm_pasid(adev: dev->adev, avm, pasid: p->pasid);
1685	if (ret)
1686	goto err_set_pasid;
1687
1688	pdd->drm_file = drm_file;
1689
1690	return 0;
1691
1692	err_set_pasid:
1693	kfd_process_device_destroy_cwsr_dgpu(pdd);
1694	err_init_cwsr:
1695	kfd_process_device_destroy_ib_mem(pdd);
1696	err_reserve_ib_mem:
1697	pdd->drm_priv = NULL;
1698	amdgpu_amdkfd_gpuvm_destroy_cb(adev: dev->adev, vm: avm);
1699
1700	return ret;
1701	}
1702
1703	/*
1704	* Direct the IOMMU to bind the process (specifically the pasid->mm)
1705	* to the device.
1706	* Unbinding occurs when the process dies or the device is removed.
1707	*
1708	* Assumes that the process lock is held.
1709	*/
1710	struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1711	struct kfd_process *p)
1712	{
1713	struct kfd_process_device *pdd;
1714	int err;
1715
1716	pdd = kfd_get_process_device_data(dev, p);
1717	if (!pdd) {
1718	pr_err("Process device data doesn't exist\n");
1719	return ERR_PTR(error: -ENOMEM);
1720	}
1721
1722	if (!pdd->drm_priv)
1723	return ERR_PTR(error: -ENODEV);
1724
1725	/*
1726	* signal runtime-pm system to auto resume and prevent
1727	* further runtime suspend once device pdd is created until
1728	* pdd is destroyed.
1729	*/
1730	if (!pdd->runtime_inuse) {
1731	err = pm_runtime_get_sync(dev: adev_to_drm(adev: dev->adev)->dev);
1732	if (err < 0) {
1733	pm_runtime_put_autosuspend(dev: adev_to_drm(adev: dev->adev)->dev);
1734	return ERR_PTR(error: err);
1735	}
1736	}
1737
1738	/*
1739	* make sure that runtime_usage counter is incremented just once
1740	* per pdd
1741	*/
1742	pdd->runtime_inuse = true;
1743
1744	return pdd;
1745	}
1746
1747	/* Create specific handle mapped to mem from process local memory idr
1748	* Assumes that the process lock is held.
1749	*/
1750	int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1751	void *mem)
1752	{
1753	return idr_alloc(&pdd->alloc_idr, ptr: mem, start: 0, end: 0, GFP_KERNEL);
1754	}
1755
1756	/* Translate specific handle from process local memory idr
1757	* Assumes that the process lock is held.
1758	*/
1759	void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1760	int handle)
1761	{
1762	if (handle < 0)
1763	return NULL;
1764
1765	return idr_find(&pdd->alloc_idr, id: handle);
1766	}
1767
1768	/* Remove specific handle from process local memory idr
1769	* Assumes that the process lock is held.
1770	*/
1771	void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1772	int handle)
1773	{
1774	if (handle >= 0)
1775	idr_remove(&pdd->alloc_idr, id: handle);
1776	}
1777
1778	/* This increments the process->ref counter. */
1779	struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1780	{
1781	struct kfd_process *p, *ret_p = NULL;
1782	unsigned int temp;
1783
1784	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1785
1786	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1787	if (p->pasid == pasid) {
1788	kref_get(kref: &p->ref);
1789	ret_p = p;
1790	break;
1791	}
1792	}
1793
1794	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1795
1796	return ret_p;
1797	}
1798
1799	/* This increments the process->ref counter. */
1800	struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1801	{
1802	struct kfd_process *p;
1803
1804	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1805
1806	p = find_process_by_mm(mm);
1807	if (p)
1808	kref_get(kref: &p->ref);
1809
1810	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1811
1812	return p;
1813	}
1814
1815	/* kfd_process_evict_queues - Evict all user queues of a process
1816	*
1817	* Eviction is reference-counted per process-device. This means multiple
1818	* evictions from different sources can be nested safely.
1819	*/
1820	int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1821	{
1822	int r = 0;
1823	int i;
1824	unsigned int n_evicted = 0;
1825
1826	for (i = 0; i < p->n_pdds; i++) {
1827	struct kfd_process_device *pdd = p->pdds[i];
1828
1829	kfd_smi_event_queue_eviction(node: pdd->dev, pid: p->lead_thread->pid,
1830	trigger);
1831
1832	r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1833	&pdd->qpd);
1834	/* evict return -EIO if HWS is hang or asic is resetting, in this case
1835	* we would like to set all the queues to be in evicted state to prevent
1836	* them been add back since they actually not be saved right now.
1837	*/
1838	if (r && r != -EIO) {
1839	pr_err("Failed to evict process queues\n");
1840	goto fail;
1841	}
1842	n_evicted++;
1843	}
1844
1845	return r;
1846
1847	fail:
1848	/* To keep state consistent, roll back partial eviction by
1849	* restoring queues
1850	*/
1851	for (i = 0; i < p->n_pdds; i++) {
1852	struct kfd_process_device *pdd = p->pdds[i];
1853
1854	if (n_evicted == 0)
1855	break;
1856
1857	kfd_smi_event_queue_restore(node: pdd->dev, pid: p->lead_thread->pid);
1858
1859	if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1860	&pdd->qpd))
1861	pr_err("Failed to restore queues\n");
1862
1863	n_evicted--;
1864	}
1865
1866	return r;
1867	}
1868
1869	/* kfd_process_restore_queues - Restore all user queues of a process */
1870	int kfd_process_restore_queues(struct kfd_process *p)
1871	{
1872	int r, ret = 0;
1873	int i;
1874
1875	for (i = 0; i < p->n_pdds; i++) {
1876	struct kfd_process_device *pdd = p->pdds[i];
1877
1878	kfd_smi_event_queue_restore(node: pdd->dev, pid: p->lead_thread->pid);
1879
1880	r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1881	&pdd->qpd);
1882	if (r) {
1883	pr_err("Failed to restore process queues\n");
1884	if (!ret)
1885	ret = r;
1886	}
1887	}
1888
1889	return ret;
1890	}
1891
1892	int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1893	{
1894	int i;
1895
1896	for (i = 0; i < p->n_pdds; i++)
1897	if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1898	return i;
1899	return -EINVAL;
1900	}
1901
1902	int
1903	kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1904	uint32_t *gpuid, uint32_t *gpuidx)
1905	{
1906	int i;
1907
1908	for (i = 0; i < p->n_pdds; i++)
1909	if (p->pdds[i] && p->pdds[i]->dev == node) {
1910	*gpuid = p->pdds[i]->user_gpu_id;
1911	*gpuidx = i;
1912	return 0;
1913	}
1914	return -EINVAL;
1915	}
1916
1917	static int signal_eviction_fence(struct kfd_process *p)
1918	{
1919	struct dma_fence *ef;
1920	int ret;
1921
1922	rcu_read_lock();
1923	ef = dma_fence_get_rcu_safe(fencep: &p->ef);
1924	rcu_read_unlock();
1925
1926	ret = dma_fence_signal(fence: ef);
1927	dma_fence_put(fence: ef);
1928
1929	return ret;
1930	}
1931
1932	static void evict_process_worker(struct work_struct *work)
1933	{
1934	int ret;
1935	struct kfd_process *p;
1936	struct delayed_work *dwork;
1937
1938	dwork = to_delayed_work(work);
1939
1940	/* Process termination destroys this worker thread. So during the
1941	* lifetime of this thread, kfd_process p will be valid
1942	*/
1943	p = container_of(dwork, struct kfd_process, eviction_work);
1944
1945	pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1946	ret = kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_TRIGGER_TTM);
1947	if (!ret) {
1948	/* If another thread already signaled the eviction fence,
1949	* they are responsible stopping the queues and scheduling
1950	* the restore work.
1951	*/
1952	if (!signal_eviction_fence(p))
1953	queue_delayed_work(wq: kfd_restore_wq, dwork: &p->restore_work,
1954	delay: msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1955	else
1956	kfd_process_restore_queues(p);
1957
1958	pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1959	} else
1960	pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1961	}
1962
1963	static int restore_process_helper(struct kfd_process *p)
1964	{
1965	int ret = 0;
1966
1967	/* VMs may not have been acquired yet during debugging. */
1968	if (p->kgd_process_info) {
1969	ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
1970	process_info: p->kgd_process_info, ef: &p->ef);
1971	if (ret)
1972	return ret;
1973	}
1974
1975	ret = kfd_process_restore_queues(p);
1976	if (!ret)
1977	pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1978	else
1979	pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1980
1981	return ret;
1982	}
1983
1984	static void restore_process_worker(struct work_struct *work)
1985	{
1986	struct delayed_work *dwork;
1987	struct kfd_process *p;
1988	int ret = 0;
1989
1990	dwork = to_delayed_work(work);
1991
1992	/* Process termination destroys this worker thread. So during the
1993	* lifetime of this thread, kfd_process p will be valid
1994	*/
1995	p = container_of(dwork, struct kfd_process, restore_work);
1996	pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1997
1998	/* Setting last_restore_timestamp before successful restoration.
1999	* Otherwise this would have to be set by KGD (restore_process_bos)
2000	* before KFD BOs are unreserved. If not, the process can be evicted
2001	* again before the timestamp is set.
2002	* If restore fails, the timestamp will be set again in the next
2003	* attempt. This would mean that the minimum GPU quanta would be
2004	* PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2005	* functions)
2006	*/
2007
2008	p->last_restore_timestamp = get_jiffies_64();
2009
2010	ret = restore_process_helper(p);
2011	if (ret) {
2012	pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
2013	p->pasid, PROCESS_BACK_OFF_TIME_MS);
2014	ret = queue_delayed_work(wq: kfd_restore_wq, dwork: &p->restore_work,
2015	delay: msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
2016	WARN(!ret, "reschedule restore work failed\n");
2017	}
2018	}
2019
2020	void kfd_suspend_all_processes(void)
2021	{
2022	struct kfd_process *p;
2023	unsigned int temp;
2024	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
2025
2026	WARN(debug_evictions, "Evicting all processes");
2027	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2028	if (kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2029	pr_err("Failed to suspend process 0x%x\n", p->pasid);
2030	signal_eviction_fence(p);
2031	}
2032	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
2033	}
2034
2035	int kfd_resume_all_processes(void)
2036	{
2037	struct kfd_process *p;
2038	unsigned int temp;
2039	int ret = 0, idx = srcu_read_lock(ssp: &kfd_processes_srcu);
2040
2041	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2042	if (restore_process_helper(p)) {
2043	pr_err("Restore process %d failed during resume\n",
2044	p->pasid);
2045	ret = -EFAULT;
2046	}
2047	}
2048	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
2049	return ret;
2050	}
2051
2052	int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2053	struct vm_area_struct *vma)
2054	{
2055	struct kfd_process_device *pdd;
2056	struct qcm_process_device *qpd;
2057
2058	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2059	pr_err("Incorrect CWSR mapping size.\n");
2060	return -EINVAL;
2061	}
2062
2063	pdd = kfd_get_process_device_data(dev, p: process);
2064	if (!pdd)
2065	return -EINVAL;
2066	qpd = &pdd->qpd;
2067
2068	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2069	order: get_order(KFD_CWSR_TBA_TMA_SIZE));
2070	if (!qpd->cwsr_kaddr) {
2071	pr_err("Error allocating per process CWSR buffer.\n");
2072	return -ENOMEM;
2073	}
2074
2075	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2076	| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2077	/* Mapping pages to user process */
2078	return remap_pfn_range(vma, addr: vma->vm_start,
2079	PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2080	KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
2081	}
2082
2083	/* assumes caller holds process lock. */
2084	int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2085	{
2086	uint32_t irq_drain_fence[8];
2087	uint8_t node_id = 0;
2088	int r = 0;
2089
2090	if (!KFD_IS_SOC15(pdd->dev))
2091	return 0;
2092
2093	pdd->process->irq_drain_is_open = true;
2094
2095	memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2096	irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2097	KFD_IRQ_FENCE_CLIENTID;
2098	irq_drain_fence[3] = pdd->process->pasid;
2099
2100	/*
2101	* For GFX 9.4.3, send the NodeId also in IH cookie DW[3]
2102	*/
2103	if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3)) {
2104	node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2105	irq_drain_fence[3] |= node_id << 16;
2106	}
2107
2108	/* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2109	if (amdgpu_amdkfd_send_close_event_drain_irq(adev: pdd->dev->adev,
2110	payload: irq_drain_fence)) {
2111	pdd->process->irq_drain_is_open = false;
2112	return 0;
2113	}
2114
2115	r = wait_event_interruptible(pdd->process->wait_irq_drain,
2116	!READ_ONCE(pdd->process->irq_drain_is_open));
2117	if (r)
2118	pdd->process->irq_drain_is_open = false;
2119
2120	return r;
2121	}
2122
2123	void kfd_process_close_interrupt_drain(unsigned int pasid)
2124	{
2125	struct kfd_process *p;
2126
2127	p = kfd_lookup_process_by_pasid(pasid);
2128
2129	if (!p)
2130	return;
2131
2132	WRITE_ONCE(p->irq_drain_is_open, false);
2133	wake_up_all(&p->wait_irq_drain);
2134	kfd_unref_process(p);
2135	}
2136
2137	struct send_exception_work_handler_workarea {
2138	struct work_struct work;
2139	struct kfd_process *p;
2140	unsigned int queue_id;
2141	uint64_t error_reason;
2142	};
2143
2144	static void send_exception_work_handler(struct work_struct *work)
2145	{
2146	struct send_exception_work_handler_workarea *workarea;
2147	struct kfd_process *p;
2148	struct queue *q;
2149	struct mm_struct *mm;
2150	struct kfd_context_save_area_header __user *csa_header;
2151	uint64_t __user *err_payload_ptr;
2152	uint64_t cur_err;
2153	uint32_t ev_id;
2154
2155	workarea = container_of(work,
2156	struct send_exception_work_handler_workarea,
2157	work);
2158	p = workarea->p;
2159
2160	mm = get_task_mm(task: p->lead_thread);
2161
2162	if (!mm)
2163	return;
2164
2165	kthread_use_mm(mm);
2166
2167	q = pqm_get_user_queue(pqm: &p->pqm, qid: workarea->queue_id);
2168
2169	if (!q)
2170	goto out;
2171
2172	csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2173
2174	get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2175	get_user(cur_err, err_payload_ptr);
2176	cur_err |= workarea->error_reason;
2177	put_user(cur_err, err_payload_ptr);
2178	get_user(ev_id, &csa_header->err_event_id);
2179
2180	kfd_set_event(p, event_id: ev_id);
2181
2182	out:
2183	kthread_unuse_mm(mm);
2184	mmput(mm);
2185	}
2186
2187	int kfd_send_exception_to_runtime(struct kfd_process *p,
2188	unsigned int queue_id,
2189	uint64_t error_reason)
2190	{
2191	struct send_exception_work_handler_workarea worker;
2192
2193	INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2194
2195	worker.p = p;
2196	worker.queue_id = queue_id;
2197	worker.error_reason = error_reason;
2198
2199	schedule_work(work: &worker.work);
2200	flush_work(work: &worker.work);
2201	destroy_work_on_stack(work: &worker.work);
2202
2203	return 0;
2204	}
2205
2206	struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2207	{
2208	int i;
2209
2210	if (gpu_id) {
2211	for (i = 0; i < p->n_pdds; i++) {
2212	struct kfd_process_device *pdd = p->pdds[i];
2213
2214	if (pdd->user_gpu_id == gpu_id)
2215	return pdd;
2216	}
2217	}
2218	return NULL;
2219	}
2220
2221	int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2222	{
2223	int i;
2224
2225	if (!actual_gpu_id)
2226	return 0;
2227
2228	for (i = 0; i < p->n_pdds; i++) {
2229	struct kfd_process_device *pdd = p->pdds[i];
2230
2231	if (pdd->dev->id == actual_gpu_id)
2232	return pdd->user_gpu_id;
2233	}
2234	return -EINVAL;
2235	}
2236
2237	#if defined(CONFIG_DEBUG_FS)
2238
2239	int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2240	{
2241	struct kfd_process *p;
2242	unsigned int temp;
2243	int r = 0;
2244
2245	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
2246
2247	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2248	seq_printf(m, fmt: "Process %d PASID 0x%x:\n",
2249	p->lead_thread->tgid, p->pasid);
2250
2251	mutex_lock(&p->mutex);
2252	r = pqm_debugfs_mqds(m, data: &p->pqm);
2253	mutex_unlock(lock: &p->mutex);
2254
2255	if (r)
2256	break;
2257	}
2258
2259	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
2260
2261	return r;
2262	}
2263
2264	#endif
2265