1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Copyright 2014-2022 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23
24	#include <linux/mutex.h>
25	#include <linux/log2.h>
26	#include <linux/sched.h>
27	#include <linux/sched/mm.h>
28	#include <linux/sched/task.h>
29	#include <linux/mmu_context.h>
30	#include <linux/slab.h>
31	#include <linux/notifier.h>
32	#include <linux/compat.h>
33	#include <linux/mman.h>
34	#include <linux/file.h>
35	#include <linux/pm_runtime.h>
36	#include "amdgpu_amdkfd.h"
37	#include "amdgpu.h"
38	#include "amdgpu_reset.h"
39
40	struct mm_struct;
41
42	#include "kfd_priv.h"
43	#include "kfd_device_queue_manager.h"
44	#include "kfd_svm.h"
45	#include "kfd_smi_events.h"
46	#include "kfd_debug.h"
47
48	/*
49	* List of struct kfd_process (field kfd_process).
50	* Unique/indexed by mm_struct*
51	*/
52	DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
53	DEFINE_MUTEX(kfd_processes_mutex);
54
55	DEFINE_SRCU(kfd_processes_srcu);
56
57	/* For process termination handling */
58	static struct workqueue_struct *kfd_process_wq;
59
60	/* Ordered, single-threaded workqueue for restoring evicted
61	* processes. Restoring multiple processes concurrently under memory
62	* pressure can lead to processes blocking each other from validating
63	* their BOs and result in a live-lock situation where processes
64	* remain evicted indefinitely.
65	*/
66	static struct workqueue_struct *kfd_restore_wq;
67
68	static struct kfd_process *find_process(const struct task_struct *thread,
69	bool ref);
70	static void kfd_process_ref_release(struct kref *ref);
71	static struct kfd_process *create_process(const struct task_struct *thread);
72
73	static void evict_process_worker(struct work_struct *work);
74	static void restore_process_worker(struct work_struct *work);
75
76	static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77
78	struct kfd_procfs_tree {
79	struct kobject *kobj;
80	};
81
82	static struct kfd_procfs_tree procfs;
83
84	/*
85	* Structure for SDMA activity tracking
86	*/
87	struct kfd_sdma_activity_handler_workarea {
88	struct work_struct sdma_activity_work;
89	struct kfd_process_device *pdd;
90	uint64_t sdma_activity_counter;
91	};
92
93	struct temp_sdma_queue_list {
94	uint64_t __user *rptr;
95	uint64_t sdma_val;
96	unsigned int queue_id;
97	struct list_head list;
98	};
99
100	static void kfd_sdma_activity_worker(struct work_struct *work)
101	{
102	struct kfd_sdma_activity_handler_workarea *workarea;
103	struct kfd_process_device *pdd;
104	uint64_t val;
105	struct mm_struct *mm;
106	struct queue *q;
107	struct qcm_process_device *qpd;
108	struct device_queue_manager *dqm;
109	int ret = 0;
110	struct temp_sdma_queue_list sdma_q_list;
111	struct temp_sdma_queue_list *sdma_q, *next;
112
113	workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114	sdma_activity_work);
115
116	pdd = workarea->pdd;
117	if (!pdd)
118	return;
119	dqm = pdd->dev->dqm;
120	qpd = &pdd->qpd;
121	if (!dqm || !qpd)
122	return;
123	/*
124	* Total SDMA activity is current SDMA activity + past SDMA activity
125	* Past SDMA count is stored in pdd.
126	* To get the current activity counters for all active SDMA queues,
127	* we loop over all SDMA queues and get their counts from user-space.
128	*
129	* We cannot call get_user() with dqm_lock held as it can cause
130	* a circular lock dependency situation. To read the SDMA stats,
131	* we need to do the following:
132	*
133	* 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134	* with dqm_lock/dqm_unlock().
135	* 2. Call get_user() for each node in temporary list without dqm_lock.
136	* Save the SDMA count for each node and also add the count to the total
137	* SDMA count counter.
138	* Its possible, during this step, a few SDMA queue nodes got deleted
139	* from the qpd->queues_list.
140	* 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141	* If any node got deleted, its SDMA count would be captured in the sdma
142	* past activity counter. So subtract the SDMA counter stored in step 2
143	* for this node from the total SDMA count.
144	*/
145	INIT_LIST_HEAD(list: &sdma_q_list.list);
146
147	/*
148	* Create the temp list of all SDMA queues
149	*/
150	dqm_lock(dqm);
151
152	list_for_each_entry(q, &qpd->queues_list, list) {
153	if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154	(q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155	continue;
156
157	sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
158	if (!sdma_q) {
159	dqm_unlock(dqm);
160	goto cleanup;
161	}
162
163	INIT_LIST_HEAD(list: &sdma_q->list);
164	sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
165	sdma_q->queue_id = q->properties.queue_id;
166	list_add_tail(new: &sdma_q->list, head: &sdma_q_list.list);
167	}
168
169	/*
170	* If the temp list is empty, then no SDMA queues nodes were found in
171	* qpd->queues_list. Return the past activity count as the total sdma
172	* count
173	*/
174	if (list_empty(head: &sdma_q_list.list)) {
175	workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
176	dqm_unlock(dqm);
177	return;
178	}
179
180	dqm_unlock(dqm);
181
182	/*
183	* Get the usage count for each SDMA queue in temp_list.
184	*/
185	mm = get_task_mm(task: pdd->process->lead_thread);
186	if (!mm)
187	goto cleanup;
188
189	kthread_use_mm(mm);
190
191	list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
192	val = 0;
193	ret = read_sdma_queue_counter(q_rptr: sdma_q->rptr, val: &val);
194	if (ret) {
195	pr_debug("Failed to read SDMA queue active counter for queue id: %d",
196	sdma_q->queue_id);
197	} else {
198	sdma_q->sdma_val = val;
199	workarea->sdma_activity_counter += val;
200	}
201	}
202
203	kthread_unuse_mm(mm);
204	mmput(mm);
205
206	/*
207	* Do a second iteration over qpd_queues_list to check if any SDMA
208	* nodes got deleted while fetching SDMA counter.
209	*/
210	dqm_lock(dqm);
211
212	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
213
214	list_for_each_entry(q, &qpd->queues_list, list) {
215	if (list_empty(head: &sdma_q_list.list))
216	break;
217
218	if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
219	(q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
220	continue;
221
222	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
223	if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
224	(sdma_q->queue_id == q->properties.queue_id)) {
225	list_del(entry: &sdma_q->list);
226	kfree(objp: sdma_q);
227	break;
228	}
229	}
230	}
231
232	dqm_unlock(dqm);
233
234	/*
235	* If temp list is not empty, it implies some queues got deleted
236	* from qpd->queues_list during SDMA usage read. Subtract the SDMA
237	* count for each node from the total SDMA count.
238	*/
239	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
240	workarea->sdma_activity_counter -= sdma_q->sdma_val;
241	list_del(entry: &sdma_q->list);
242	kfree(objp: sdma_q);
243	}
244
245	return;
246
247	cleanup:
248	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
249	list_del(entry: &sdma_q->list);
250	kfree(objp: sdma_q);
251	}
252	}
253
254	/**
255	* kfd_get_cu_occupancy - Collect number of waves in-flight on this device
256	* by current process. Translates acquired wave count into number of compute units
257	* that are occupied.
258	*
259	* @attr: Handle of attribute that allows reporting of wave count. The attribute
260	* handle encapsulates GPU device it is associated with, thereby allowing collection
261	* of waves in flight, etc
262	* @buffer: Handle of user provided buffer updated with wave count
263	*
264	* Return: Number of bytes written to user buffer or an error value
265	*/
266	static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
267	{
268	int cu_cnt;
269	int wave_cnt;
270	int max_waves_per_cu;
271	struct kfd_node *dev = NULL;
272	struct kfd_process *proc = NULL;
273	struct kfd_process_device *pdd = NULL;
274	int i;
275	struct kfd_cu_occupancy *cu_occupancy;
276	u32 queue_format;
277
278	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
279	dev = pdd->dev;
280	if (dev->kfd2kgd->get_cu_occupancy == NULL)
281	return -EINVAL;
282
283	cu_cnt = 0;
284	proc = pdd->process;
285	if (pdd->qpd.queue_count == 0) {
286	pr_debug("Gpu-Id: %d has no active queues for process pid %d\n",
287	dev->id, (int)proc->lead_thread->pid);
288	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", cu_cnt);
289	}
290
291	/* Collect wave count from device if it supports */
292	wave_cnt = 0;
293	max_waves_per_cu = 0;
294
295	cu_occupancy = kcalloc(AMDGPU_MAX_QUEUES, sizeof(*cu_occupancy), GFP_KERNEL);
296	if (!cu_occupancy)
297	return -ENOMEM;
298
299	/*
300	* For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.
301	* For AQL queues, because of cooperative dispatch we multiply the wave count
302	* by number of XCCs in the partition to get the total wave counts across all
303	* XCCs in the partition.
304	* For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
305	*/
306	dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
307	&max_waves_per_cu, ffs(dev->xcc_mask) - 1);
308
309	for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
310	if (cu_occupancy[i].wave_cnt != 0 &&
311	kfd_dqm_is_queue_in_process(dqm: dev->dqm, qpd: &pdd->qpd,
312	doorbell_off: cu_occupancy[i].doorbell_off,
313	queue_format: &queue_format)) {
314	if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))
315	wave_cnt += cu_occupancy[i].wave_cnt;
316	else
317	wave_cnt += (NUM_XCC(dev->xcc_mask) *
318	cu_occupancy[i].wave_cnt);
319	}
320	}
321
322	/* Translate wave count to number of compute units */
323	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
324	kfree(objp: cu_occupancy);
325	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", cu_cnt);
326	}
327
328	static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
329	char *buffer)
330	{
331	if (strcmp(attr->name, "pasid") == 0)
332	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d\n", 0);
333	else if (strncmp(attr->name, "vram_", 5) == 0) {
334	struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
335	attr_vram);
336	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%llu\n", atomic64_read(v: &pdd->vram_usage));
337	} else if (strncmp(attr->name, "sdma_", 5) == 0) {
338	struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
339	attr_sdma);
340	struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
341
342	INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
343	kfd_sdma_activity_worker);
344
345	sdma_activity_work_handler.pdd = pdd;
346	sdma_activity_work_handler.sdma_activity_counter = 0;
347
348	schedule_work(work: &sdma_activity_work_handler.sdma_activity_work);
349
350	flush_work(work: &sdma_activity_work_handler.sdma_activity_work);
351	destroy_work_on_stack(work: &sdma_activity_work_handler.sdma_activity_work);
352
353	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%llu\n",
354	(sdma_activity_work_handler.sdma_activity_counter)/
355	SDMA_ACTIVITY_DIVISOR);
356	} else {
357	pr_err("Invalid attribute");
358	return -EINVAL;
359	}
360
361	return 0;
362	}
363
364	static void kfd_procfs_kobj_release(struct kobject *kobj)
365	{
366	kfree(objp: kobj);
367	}
368
369	static const struct sysfs_ops kfd_procfs_ops = {
370	.show = kfd_procfs_show,
371	};
372
373	static const struct kobj_type procfs_type = {
374	.release = kfd_procfs_kobj_release,
375	.sysfs_ops = &kfd_procfs_ops,
376	};
377
378	void kfd_procfs_init(void)
379	{
380	int ret = 0;
381
382	procfs.kobj = kfd_alloc_struct(procfs.kobj);
383	if (!procfs.kobj)
384	return;
385
386	ret = kobject_init_and_add(kobj: procfs.kobj, ktype: &procfs_type,
387	parent: &kfd_device->kobj, fmt: "proc");
388	if (ret) {
389	pr_warn("Could not create procfs proc folder");
390	/* If we fail to create the procfs, clean up */
391	kfd_procfs_shutdown();
392	}
393	}
394
395	void kfd_procfs_shutdown(void)
396	{
397	if (procfs.kobj) {
398	kobject_del(kobj: procfs.kobj);
399	kobject_put(kobj: procfs.kobj);
400	procfs.kobj = NULL;
401	}
402	}
403
404	static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
405	struct attribute *attr, char *buffer)
406	{
407	struct queue *q = container_of(kobj, struct queue, kobj);
408
409	if (!strcmp(attr->name, "size"))
410	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%llu",
411	q->properties.queue_size);
412	else if (!strcmp(attr->name, "type"))
413	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%d", q->properties.type);
414	else if (!strcmp(attr->name, "gpuid"))
415	return snprintf(buf: buffer, PAGE_SIZE, fmt: "%u", q->device->id);
416	else
417	pr_err("Invalid attribute");
418
419	return 0;
420	}
421
422	static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
423	struct attribute *attr, char *buffer)
424	{
425	if (strcmp(attr->name, "evicted_ms") == 0) {
426	struct kfd_process_device *pdd = container_of(attr,
427	struct kfd_process_device,
428	attr_evict);
429	uint64_t evict_jiffies;
430
431	evict_jiffies = atomic64_read(v: &pdd->evict_duration_counter);
432
433	return snprintf(buf: buffer,
434	PAGE_SIZE,
435	fmt: "%llu\n",
436	jiffies64_to_msecs(j: evict_jiffies));
437
438	/* Sysfs handle that gets CU occupancy is per device */
439	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
440	return kfd_get_cu_occupancy(attr, buffer);
441	} else {
442	pr_err("Invalid attribute");
443	}
444
445	return 0;
446	}
447
448	static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
449	struct attribute *attr, char *buf)
450	{
451	struct kfd_process_device *pdd;
452
453	if (!strcmp(attr->name, "faults")) {
454	pdd = container_of(attr, struct kfd_process_device,
455	attr_faults);
456	return sysfs_emit(buf, fmt: "%llu\n", READ_ONCE(pdd->faults));
457	}
458	if (!strcmp(attr->name, "page_in")) {
459	pdd = container_of(attr, struct kfd_process_device,
460	attr_page_in);
461	return sysfs_emit(buf, fmt: "%llu\n", READ_ONCE(pdd->page_in));
462	}
463	if (!strcmp(attr->name, "page_out")) {
464	pdd = container_of(attr, struct kfd_process_device,
465	attr_page_out);
466	return sysfs_emit(buf, fmt: "%llu\n", READ_ONCE(pdd->page_out));
467	}
468	return 0;
469	}
470
471	static struct attribute attr_queue_size = {
472	.name = "size",
473	.mode = KFD_SYSFS_FILE_MODE
474	};
475
476	static struct attribute attr_queue_type = {
477	.name = "type",
478	.mode = KFD_SYSFS_FILE_MODE
479	};
480
481	static struct attribute attr_queue_gpuid = {
482	.name = "gpuid",
483	.mode = KFD_SYSFS_FILE_MODE
484	};
485
486	static struct attribute *procfs_queue_attrs[] = {
487	&attr_queue_size,
488	&attr_queue_type,
489	&attr_queue_gpuid,
490	NULL
491	};
492	ATTRIBUTE_GROUPS(procfs_queue);
493
494	static const struct sysfs_ops procfs_queue_ops = {
495	.show = kfd_procfs_queue_show,
496	};
497
498	static const struct kobj_type procfs_queue_type = {
499	.sysfs_ops = &procfs_queue_ops,
500	.default_groups = procfs_queue_groups,
501	};
502
503	static const struct sysfs_ops procfs_stats_ops = {
504	.show = kfd_procfs_stats_show,
505	};
506
507	static const struct kobj_type procfs_stats_type = {
508	.sysfs_ops = &procfs_stats_ops,
509	.release = kfd_procfs_kobj_release,
510	};
511
512	static const struct sysfs_ops sysfs_counters_ops = {
513	.show = kfd_sysfs_counters_show,
514	};
515
516	static const struct kobj_type sysfs_counters_type = {
517	.sysfs_ops = &sysfs_counters_ops,
518	.release = kfd_procfs_kobj_release,
519	};
520
521	int kfd_procfs_add_queue(struct queue *q)
522	{
523	struct kfd_process *proc;
524	int ret;
525
526	if (!q || !q->process)
527	return -EINVAL;
528	proc = q->process;
529
530	/* Create proc/<pid>/queues/<queue id> folder */
531	if (!proc->kobj_queues)
532	return -EFAULT;
533	ret = kobject_init_and_add(kobj: &q->kobj, ktype: &procfs_queue_type,
534	parent: proc->kobj_queues, fmt: "%u", q->properties.queue_id);
535	if (ret < 0) {
536	pr_warn("Creating proc/<pid>/queues/%u failed",
537	q->properties.queue_id);
538	kobject_put(kobj: &q->kobj);
539	return ret;
540	}
541
542	return 0;
543	}
544
545	static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
546	char *name)
547	{
548	int ret;
549
550	if (!kobj || !attr || !name)
551	return;
552
553	attr->name = name;
554	attr->mode = KFD_SYSFS_FILE_MODE;
555	sysfs_attr_init(attr);
556
557	ret = sysfs_create_file(kobj, attr);
558	if (ret)
559	pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
560	}
561
562	static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
563	{
564	int ret;
565	int i;
566	char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
567
568	if (!p || !p->kobj)
569	return;
570
571	/*
572	* Create sysfs files for each GPU:
573	* - proc/<pid>/stats_<gpuid>/
574	* - proc/<pid>/stats_<gpuid>/evicted_ms
575	* - proc/<pid>/stats_<gpuid>/cu_occupancy
576	*/
577	for (i = 0; i < p->n_pdds; i++) {
578	struct kfd_process_device *pdd = p->pdds[i];
579
580	snprintf(buf: stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
581	fmt: "stats_%u", pdd->dev->id);
582	pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
583	if (!pdd->kobj_stats)
584	return;
585
586	ret = kobject_init_and_add(kobj: pdd->kobj_stats,
587	ktype: &procfs_stats_type,
588	parent: p->kobj,
589	fmt: stats_dir_filename);
590
591	if (ret) {
592	pr_warn("Creating KFD proc/stats_%s folder failed",
593	stats_dir_filename);
594	kobject_put(kobj: pdd->kobj_stats);
595	pdd->kobj_stats = NULL;
596	return;
597	}
598
599	kfd_sysfs_create_file(kobj: pdd->kobj_stats, attr: &pdd->attr_evict,
600	name: "evicted_ms");
601	/* Add sysfs file to report compute unit occupancy */
602	if (pdd->dev->kfd2kgd->get_cu_occupancy)
603	kfd_sysfs_create_file(kobj: pdd->kobj_stats,
604	attr: &pdd->attr_cu_occupancy,
605	name: "cu_occupancy");
606	}
607	}
608
609	static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
610	{
611	int ret = 0;
612	int i;
613	char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
614
615	if (!p || !p->kobj)
616	return;
617
618	/*
619	* Create sysfs files for each GPU which supports SVM
620	* - proc/<pid>/counters_<gpuid>/
621	* - proc/<pid>/counters_<gpuid>/faults
622	* - proc/<pid>/counters_<gpuid>/page_in
623	* - proc/<pid>/counters_<gpuid>/page_out
624	*/
625	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
626	struct kfd_process_device *pdd = p->pdds[i];
627	struct kobject *kobj_counters;
628
629	snprintf(buf: counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
630	fmt: "counters_%u", pdd->dev->id);
631	kobj_counters = kfd_alloc_struct(kobj_counters);
632	if (!kobj_counters)
633	return;
634
635	ret = kobject_init_and_add(kobj: kobj_counters, ktype: &sysfs_counters_type,
636	parent: p->kobj, fmt: counters_dir_filename);
637	if (ret) {
638	pr_warn("Creating KFD proc/%s folder failed",
639	counters_dir_filename);
640	kobject_put(kobj: kobj_counters);
641	return;
642	}
643
644	pdd->kobj_counters = kobj_counters;
645	kfd_sysfs_create_file(kobj: kobj_counters, attr: &pdd->attr_faults,
646	name: "faults");
647	kfd_sysfs_create_file(kobj: kobj_counters, attr: &pdd->attr_page_in,
648	name: "page_in");
649	kfd_sysfs_create_file(kobj: kobj_counters, attr: &pdd->attr_page_out,
650	name: "page_out");
651	}
652	}
653
654	static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
655	{
656	int i;
657
658	if (!p || !p->kobj)
659	return;
660
661	/*
662	* Create sysfs files for each GPU:
663	* - proc/<pid>/vram_<gpuid>
664	* - proc/<pid>/sdma_<gpuid>
665	*/
666	for (i = 0; i < p->n_pdds; i++) {
667	struct kfd_process_device *pdd = p->pdds[i];
668
669	snprintf(buf: pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, fmt: "vram_%u",
670	pdd->dev->id);
671	kfd_sysfs_create_file(kobj: p->kobj, attr: &pdd->attr_vram,
672	name: pdd->vram_filename);
673
674	snprintf(buf: pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, fmt: "sdma_%u",
675	pdd->dev->id);
676	kfd_sysfs_create_file(kobj: p->kobj, attr: &pdd->attr_sdma,
677	name: pdd->sdma_filename);
678	}
679	}
680
681	void kfd_procfs_del_queue(struct queue *q)
682	{
683	if (!q)
684	return;
685
686	kobject_del(kobj: &q->kobj);
687	kobject_put(kobj: &q->kobj);
688	}
689
690	int kfd_process_create_wq(void)
691	{
692	if (!kfd_process_wq)
693	kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
694	if (!kfd_restore_wq)
695	kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
696	WQ_FREEZABLE);
697
698	if (!kfd_process_wq || !kfd_restore_wq) {
699	kfd_process_destroy_wq();
700	return -ENOMEM;
701	}
702
703	return 0;
704	}
705
706	void kfd_process_destroy_wq(void)
707	{
708	if (kfd_process_wq) {
709	destroy_workqueue(wq: kfd_process_wq);
710	kfd_process_wq = NULL;
711	}
712	if (kfd_restore_wq) {
713	destroy_workqueue(wq: kfd_restore_wq);
714	kfd_restore_wq = NULL;
715	}
716	}
717
718	static void kfd_process_free_gpuvm(struct kgd_mem *mem,
719	struct kfd_process_device *pdd, void **kptr)
720	{
721	struct kfd_node *dev = pdd->dev;
722
723	if (kptr && *kptr) {
724	amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
725	*kptr = NULL;
726	}
727
728	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(adev: dev->adev, mem, drm_priv: pdd->drm_priv);
729	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: dev->adev, mem, drm_priv: pdd->drm_priv,
730	NULL);
731	}
732
733	/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
734	* This function should be only called right after the process
735	* is created and when kfd_processes_mutex is still being held
736	* to avoid concurrency. Because of that exclusiveness, we do
737	* not need to take p->mutex.
738	*/
739	static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
740	uint64_t gpu_va, uint32_t size,
741	uint32_t flags, struct kgd_mem **mem, void **kptr)
742	{
743	struct kfd_node *kdev = pdd->dev;
744	int err;
745
746	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(adev: kdev->adev, va: gpu_va, size,
747	drm_priv: pdd->drm_priv, mem, NULL,
748	flags, criu_resume: false);
749	if (err)
750	goto err_alloc_mem;
751
752	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(adev: kdev->adev, mem: *mem,
753	drm_priv: pdd->drm_priv);
754	if (err)
755	goto err_map_mem;
756
757	err = amdgpu_amdkfd_gpuvm_sync_memory(adev: kdev->adev, mem: *mem, intr: true);
758	if (err) {
759	pr_debug("Sync memory failed, wait interrupted by user signal\n");
760	goto sync_memory_failed;
761	}
762
763	if (kptr) {
764	err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
765	mem: (struct kgd_mem *)*mem, kptr, NULL);
766	if (err) {
767	pr_debug("Map GTT BO to kernel failed\n");
768	goto sync_memory_failed;
769	}
770	}
771
772	return err;
773
774	sync_memory_failed:
775	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(adev: kdev->adev, mem: *mem, drm_priv: pdd->drm_priv);
776
777	err_map_mem:
778	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: kdev->adev, mem: *mem, drm_priv: pdd->drm_priv,
779	NULL);
780	err_alloc_mem:
781	*mem = NULL;
782	*kptr = NULL;
783	return err;
784	}
785
786	/* kfd_process_device_reserve_ib_mem - Reserve memory inside the
787	* process for IB usage The memory reserved is for KFD to submit
788	* IB to AMDGPU from kernel. If the memory is reserved
789	* successfully, ib_kaddr will have the CPU/kernel
790	* address. Check ib_kaddr before accessing the memory.
791	*/
792	static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
793	{
794	struct qcm_process_device *qpd = &pdd->qpd;
795	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
796	KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
797	KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
798	KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
799	struct kgd_mem *mem;
800	void *kaddr;
801	int ret;
802
803	if (qpd->ib_kaddr || !qpd->ib_base)
804	return 0;
805
806	/* ib_base is only set for dGPU */
807	ret = kfd_process_alloc_gpuvm(pdd, gpu_va: qpd->ib_base, PAGE_SIZE, flags,
808	mem: &mem, kptr: &kaddr);
809	if (ret)
810	return ret;
811
812	qpd->ib_mem = mem;
813	qpd->ib_kaddr = kaddr;
814
815	return 0;
816	}
817
818	static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
819	{
820	struct qcm_process_device *qpd = &pdd->qpd;
821
822	if (!qpd->ib_kaddr || !qpd->ib_base)
823	return;
824
825	kfd_process_free_gpuvm(mem: qpd->ib_mem, pdd, kptr: &qpd->ib_kaddr);
826	}
827
828	struct kfd_process *kfd_create_process(struct task_struct *thread)
829	{
830	struct kfd_process *process;
831	int ret;
832
833	if (!(thread->mm && mmget_not_zero(mm: thread->mm)))
834	return ERR_PTR(error: -EINVAL);
835
836	/* Only the pthreads threading model is supported. */
837	if (thread->group_leader->mm != thread->mm) {
838	mmput(thread->mm);
839	return ERR_PTR(error: -EINVAL);
840	}
841
842	/* If the process just called exec(3), it is possible that the
843	* cleanup of the kfd_process (following the release of the mm
844	* of the old process image) is still in the cleanup work queue.
845	* Make sure to drain any job before trying to recreate any
846	* resource for this process.
847	*/
848	flush_workqueue(kfd_process_wq);
849
850	/*
851	* take kfd processes mutex before starting of process creation
852	* so there won't be a case where two threads of the same process
853	* create two kfd_process structures
854	*/
855	mutex_lock(&kfd_processes_mutex);
856
857	if (kfd_is_locked(NULL)) {
858	pr_debug("KFD is locked! Cannot create process");
859	process = ERR_PTR(error: -EINVAL);
860	goto out;
861	}
862
863	/* A prior open of /dev/kfd could have already created the process.
864	* find_process will increase process kref in this case
865	*/
866	process = find_process(thread, ref: true);
867	if (process) {
868	pr_debug("Process already found\n");
869	} else {
870	process = create_process(thread);
871	if (IS_ERR(ptr: process))
872	goto out;
873
874	if (!procfs.kobj)
875	goto out;
876
877	process->kobj = kfd_alloc_struct(process->kobj);
878	if (!process->kobj) {
879	pr_warn("Creating procfs kobject failed");
880	goto out;
881	}
882	ret = kobject_init_and_add(kobj: process->kobj, ktype: &procfs_type,
883	parent: procfs.kobj, fmt: "%d",
884	(int)process->lead_thread->pid);
885	if (ret) {
886	pr_warn("Creating procfs pid directory failed");
887	kobject_put(kobj: process->kobj);
888	goto out;
889	}
890
891	kfd_sysfs_create_file(kobj: process->kobj, attr: &process->attr_pasid,
892	name: "pasid");
893
894	process->kobj_queues = kobject_create_and_add(name: "queues",
895	parent: process->kobj);
896	if (!process->kobj_queues)
897	pr_warn("Creating KFD proc/queues folder failed");
898
899	kfd_procfs_add_sysfs_stats(p: process);
900	kfd_procfs_add_sysfs_files(p: process);
901	kfd_procfs_add_sysfs_counters(p: process);
902
903	kfd_debugfs_add_process(p: process);
904
905	init_waitqueue_head(&process->wait_irq_drain);
906	}
907	out:
908	mutex_unlock(lock: &kfd_processes_mutex);
909	mmput(thread->mm);
910
911	return process;
912	}
913
914	struct kfd_process *kfd_get_process(const struct task_struct *thread)
915	{
916	struct kfd_process *process;
917
918	if (!thread->mm)
919	return ERR_PTR(error: -EINVAL);
920
921	/* Only the pthreads threading model is supported. */
922	if (thread->group_leader->mm != thread->mm)
923	return ERR_PTR(error: -EINVAL);
924
925	process = find_process(thread, ref: false);
926	if (!process)
927	return ERR_PTR(error: -EINVAL);
928
929	return process;
930	}
931
932	static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
933	{
934	struct kfd_process *process;
935
936	hash_for_each_possible_rcu(kfd_processes_table, process,
937	kfd_processes, (uintptr_t)mm)
938	if (process->mm == mm)
939	return process;
940
941	return NULL;
942	}
943
944	static struct kfd_process *find_process(const struct task_struct *thread,
945	bool ref)
946	{
947	struct kfd_process *p;
948	int idx;
949
950	idx = srcu_read_lock(ssp: &kfd_processes_srcu);
951	p = find_process_by_mm(mm: thread->mm);
952	if (p && ref)
953	kref_get(kref: &p->ref);
954	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
955
956	return p;
957	}
958
959	void kfd_unref_process(struct kfd_process *p)
960	{
961	kref_put(kref: &p->ref, release: kfd_process_ref_release);
962	}
963
964	/* This increments the process->ref counter. */
965	struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
966	{
967	struct task_struct *task = NULL;
968	struct kfd_process *p = NULL;
969
970	if (!pid) {
971	task = current;
972	get_task_struct(t: task);
973	} else {
974	task = get_pid_task(pid, PIDTYPE_PID);
975	}
976
977	if (task) {
978	p = find_process(thread: task, ref: true);
979	put_task_struct(t: task);
980	}
981
982	return p;
983	}
984
985	static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
986	{
987	struct kfd_process *p = pdd->process;
988	void *mem;
989	int id;
990	int i;
991
992	/*
993	* Remove all handles from idr and release appropriate
994	* local memory object
995	*/
996	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
997
998	for (i = 0; i < p->n_pdds; i++) {
999	struct kfd_process_device *peer_pdd = p->pdds[i];
1000
1001	if (!peer_pdd->drm_priv)
1002	continue;
1003	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1004	adev: peer_pdd->dev->adev, mem, drm_priv: peer_pdd->drm_priv);
1005	}
1006
1007	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev, mem,
1008	drm_priv: pdd->drm_priv, NULL);
1009	kfd_process_device_remove_obj_handle(pdd, handle: id);
1010	}
1011	}
1012
1013	/*
1014	* Just kunmap and unpin signal BO here. It will be freed in
1015	* kfd_process_free_outstanding_kfd_bos()
1016	*/
1017	static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
1018	{
1019	struct kfd_process_device *pdd;
1020	struct kfd_node *kdev;
1021	void *mem;
1022
1023	kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
1024	if (!kdev)
1025	return;
1026
1027	mutex_lock(&p->mutex);
1028
1029	pdd = kfd_get_process_device_data(dev: kdev, p);
1030	if (!pdd)
1031	goto out;
1032
1033	mem = kfd_process_device_translate_handle(
1034	p: pdd, GET_IDR_HANDLE(p->signal_handle));
1035	if (!mem)
1036	goto out;
1037
1038	amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1039
1040	out:
1041	mutex_unlock(lock: &p->mutex);
1042	}
1043
1044	static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1045	{
1046	int i;
1047
1048	for (i = 0; i < p->n_pdds; i++)
1049	kfd_process_device_free_bos(pdd: p->pdds[i]);
1050	}
1051
1052	static void kfd_process_destroy_pdds(struct kfd_process *p)
1053	{
1054	int i;
1055
1056	for (i = 0; i < p->n_pdds; i++) {
1057	struct kfd_process_device *pdd = p->pdds[i];
1058
1059	kfd_smi_event_process(pdd, start: false);
1060
1061	pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
1062	pdd->dev->id, p->lead_thread->pid);
1063	kfd_process_device_destroy_cwsr_dgpu(pdd);
1064	kfd_process_device_destroy_ib_mem(pdd);
1065
1066	if (pdd->drm_file)
1067	fput(pdd->drm_file);
1068
1069	if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1070	free_pages(addr: (unsigned long)pdd->qpd.cwsr_kaddr,
1071	order: get_order(KFD_CWSR_TBA_TMA_SIZE));
1072
1073	idr_destroy(&pdd->alloc_idr);
1074
1075	kfd_free_process_doorbells(kfd: pdd->dev->kfd, pdd);
1076
1077	if (pdd->dev->kfd->shared_resources.enable_mes &&
1078	pdd->proc_ctx_cpu_ptr)
1079	amdgpu_amdkfd_free_gtt_mem(adev: pdd->dev->adev,
1080	mem_obj: &pdd->proc_ctx_bo);
1081	/*
1082	* before destroying pdd, make sure to report availability
1083	* for auto suspend
1084	*/
1085	if (pdd->runtime_inuse) {
1086	pm_runtime_put_autosuspend(dev: adev_to_drm(adev: pdd->dev->adev)->dev);
1087	pdd->runtime_inuse = false;
1088	}
1089
1090	atomic_dec(v: &pdd->dev->kfd->kfd_processes_count);
1091
1092	kfree(objp: pdd);
1093	p->pdds[i] = NULL;
1094	}
1095	p->n_pdds = 0;
1096	}
1097
1098	static void kfd_process_remove_sysfs(struct kfd_process *p)
1099	{
1100	struct kfd_process_device *pdd;
1101	int i;
1102
1103	if (!p->kobj)
1104	return;
1105
1106	sysfs_remove_file(kobj: p->kobj, attr: &p->attr_pasid);
1107	kobject_del(kobj: p->kobj_queues);
1108	kobject_put(kobj: p->kobj_queues);
1109	p->kobj_queues = NULL;
1110
1111	for (i = 0; i < p->n_pdds; i++) {
1112	pdd = p->pdds[i];
1113
1114	sysfs_remove_file(kobj: p->kobj, attr: &pdd->attr_vram);
1115	sysfs_remove_file(kobj: p->kobj, attr: &pdd->attr_sdma);
1116
1117	sysfs_remove_file(kobj: pdd->kobj_stats, attr: &pdd->attr_evict);
1118	if (pdd->dev->kfd2kgd->get_cu_occupancy)
1119	sysfs_remove_file(kobj: pdd->kobj_stats,
1120	attr: &pdd->attr_cu_occupancy);
1121	kobject_del(kobj: pdd->kobj_stats);
1122	kobject_put(kobj: pdd->kobj_stats);
1123	pdd->kobj_stats = NULL;
1124	}
1125
1126	for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1127	pdd = p->pdds[i];
1128
1129	sysfs_remove_file(kobj: pdd->kobj_counters, attr: &pdd->attr_faults);
1130	sysfs_remove_file(kobj: pdd->kobj_counters, attr: &pdd->attr_page_in);
1131	sysfs_remove_file(kobj: pdd->kobj_counters, attr: &pdd->attr_page_out);
1132	kobject_del(kobj: pdd->kobj_counters);
1133	kobject_put(kobj: pdd->kobj_counters);
1134	pdd->kobj_counters = NULL;
1135	}
1136
1137	kobject_del(kobj: p->kobj);
1138	kobject_put(kobj: p->kobj);
1139	p->kobj = NULL;
1140	}
1141
1142	/*
1143	* If any GPU is ongoing reset, wait for reset complete.
1144	*/
1145	static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1146	{
1147	int i;
1148
1149	for (i = 0; i < p->n_pdds; i++)
1150	flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1151	}
1152
1153	/* No process locking is needed in this function, because the process
1154	* is not findable any more. We must assume that no other thread is
1155	* using it any more, otherwise we couldn't safely free the process
1156	* structure in the end.
1157	*/
1158	static void kfd_process_wq_release(struct work_struct *work)
1159	{
1160	struct kfd_process *p = container_of(work, struct kfd_process,
1161	release_work);
1162	struct dma_fence *ef;
1163
1164	/*
1165	* If GPU in reset, user queues may still running, wait for reset complete.
1166	*/
1167	kfd_process_wait_gpu_reset_complete(p);
1168
1169	/* Signal the eviction fence after user mode queues are
1170	* destroyed. This allows any BOs to be freed without
1171	* triggering pointless evictions or waiting for fences.
1172	*/
1173	synchronize_rcu();
1174	ef = rcu_access_pointer(p->ef);
1175	if (ef)
1176	dma_fence_signal(fence: ef);
1177
1178	kfd_process_remove_sysfs(p);
1179	kfd_debugfs_remove_process(p);
1180
1181	kfd_process_kunmap_signal_bo(p);
1182	kfd_process_free_outstanding_kfd_bos(p);
1183	svm_range_list_fini(p);
1184
1185	kfd_process_destroy_pdds(p);
1186	dma_fence_put(fence: ef);
1187
1188	kfd_event_free_process(p);
1189
1190	mutex_destroy(lock: &p->mutex);
1191
1192	put_task_struct(t: p->lead_thread);
1193
1194	kfree(objp: p);
1195	}
1196
1197	static void kfd_process_ref_release(struct kref *ref)
1198	{
1199	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1200
1201	INIT_WORK(&p->release_work, kfd_process_wq_release);
1202	queue_work(wq: kfd_process_wq, work: &p->release_work);
1203	}
1204
1205	static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1206	{
1207	/* This increments p->ref counter if kfd process p exists */
1208	struct kfd_process *p = kfd_lookup_process_by_mm(mm);
1209
1210	return p ? &p->mmu_notifier : ERR_PTR(error: -ESRCH);
1211	}
1212
1213	static void kfd_process_free_notifier(struct mmu_notifier *mn)
1214	{
1215	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1216	}
1217
1218	static void kfd_process_notifier_release_internal(struct kfd_process *p)
1219	{
1220	int i;
1221
1222	cancel_delayed_work_sync(dwork: &p->eviction_work);
1223	cancel_delayed_work_sync(dwork: &p->restore_work);
1224
1225	/*
1226	* Dequeue and destroy user queues, it is not safe for GPU to access
1227	* system memory after mmu release notifier callback returns because
1228	* exit_mmap free process memory afterwards.
1229	*/
1230	kfd_process_dequeue_from_all_devices(p);
1231	pqm_uninit(pqm: &p->pqm);
1232
1233	for (i = 0; i < p->n_pdds; i++) {
1234	struct kfd_process_device *pdd = p->pdds[i];
1235
1236	/* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1237	if (!kfd_dbg_is_rlc_restore_supported(dev: pdd->dev) && p->runtime_info.ttmp_setup)
1238	amdgpu_gfx_off_ctrl(adev: pdd->dev->adev, enable: true);
1239	}
1240
1241	/* Indicate to other users that MM is no longer valid */
1242	p->mm = NULL;
1243	kfd_dbg_trap_disable(target: p);
1244
1245	if (atomic_read(v: &p->debugged_process_count) > 0) {
1246	struct kfd_process *target;
1247	unsigned int temp;
1248	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1249
1250	hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1251	if (target->debugger_process && target->debugger_process == p) {
1252	mutex_lock_nested(lock: &target->mutex, subclass: 1);
1253	kfd_dbg_trap_disable(target);
1254	mutex_unlock(lock: &target->mutex);
1255	if (atomic_read(v: &p->debugged_process_count) == 0)
1256	break;
1257	}
1258	}
1259
1260	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1261	}
1262
1263	mmu_notifier_put(subscription: &p->mmu_notifier);
1264	}
1265
1266	static void kfd_process_notifier_release(struct mmu_notifier *mn,
1267	struct mm_struct *mm)
1268	{
1269	struct kfd_process *p;
1270
1271	/*
1272	* The kfd_process structure can not be free because the
1273	* mmu_notifier srcu is read locked
1274	*/
1275	p = container_of(mn, struct kfd_process, mmu_notifier);
1276	if (WARN_ON(p->mm != mm))
1277	return;
1278
1279	mutex_lock(&kfd_processes_mutex);
1280	/*
1281	* Do early return if table is empty.
1282	*
1283	* This could potentially happen if this function is called concurrently
1284	* by mmu_notifier and by kfd_cleanup_pocesses.
1285	*
1286	*/
1287	if (hash_empty(kfd_processes_table)) {
1288	mutex_unlock(lock: &kfd_processes_mutex);
1289	return;
1290	}
1291	hash_del_rcu(node: &p->kfd_processes);
1292	mutex_unlock(lock: &kfd_processes_mutex);
1293	synchronize_srcu(ssp: &kfd_processes_srcu);
1294
1295	kfd_process_notifier_release_internal(p);
1296	}
1297
1298	static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1299	.release = kfd_process_notifier_release,
1300	.alloc_notifier = kfd_process_alloc_notifier,
1301	.free_notifier = kfd_process_free_notifier,
1302	};
1303
1304	/*
1305	* This code handles the case when driver is being unloaded before all
1306	* mm_struct are released. We need to safely free the kfd_process and
1307	* avoid race conditions with mmu_notifier that might try to free them.
1308	*
1309	*/
1310	void kfd_cleanup_processes(void)
1311	{
1312	struct kfd_process *p;
1313	struct hlist_node *p_temp;
1314	unsigned int temp;
1315	HLIST_HEAD(cleanup_list);
1316
1317	/*
1318	* Move all remaining kfd_process from the process table to a
1319	* temp list for processing. Once done, callback from mmu_notifier
1320	* release will not see the kfd_process in the table and do early return,
1321	* avoiding double free issues.
1322	*/
1323	mutex_lock(&kfd_processes_mutex);
1324	hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1325	hash_del_rcu(node: &p->kfd_processes);
1326	synchronize_srcu(ssp: &kfd_processes_srcu);
1327	hlist_add_head(n: &p->kfd_processes, h: &cleanup_list);
1328	}
1329	mutex_unlock(lock: &kfd_processes_mutex);
1330
1331	hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1332	kfd_process_notifier_release_internal(p);
1333
1334	/*
1335	* Ensures that all outstanding free_notifier get called, triggering
1336	* the release of the kfd_process struct.
1337	*/
1338	mmu_notifier_synchronize();
1339	}
1340
1341	int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1342	{
1343	unsigned long offset;
1344	int i;
1345
1346	if (p->has_cwsr)
1347	return 0;
1348
1349	for (i = 0; i < p->n_pdds; i++) {
1350	struct kfd_node *dev = p->pdds[i]->dev;
1351	struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1352
1353	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1354	continue;
1355
1356	offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1357	qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1358	KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1359	MAP_SHARED, offset);
1360
1361	if (IS_ERR_VALUE(qpd->tba_addr)) {
1362	int err = qpd->tba_addr;
1363
1364	dev_err(dev->adev->dev,
1365	"Failure to set tba address. error %d.\n", err);
1366	qpd->tba_addr = 0;
1367	qpd->cwsr_kaddr = NULL;
1368	return err;
1369	}
1370
1371	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1372
1373	kfd_process_set_trap_debug_flag(qpd, enabled: p->debug_trap_enabled);
1374
1375	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1376	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1377	qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1378	}
1379
1380	p->has_cwsr = true;
1381
1382	return 0;
1383	}
1384
1385	static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1386	{
1387	struct kfd_node *dev = pdd->dev;
1388	struct qcm_process_device *qpd = &pdd->qpd;
1389	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1390	| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1391	| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1392	struct kgd_mem *mem;
1393	void *kaddr;
1394	int ret;
1395
1396	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1397	return 0;
1398
1399	/* cwsr_base is only set for dGPU */
1400	ret = kfd_process_alloc_gpuvm(pdd, gpu_va: qpd->cwsr_base,
1401	KFD_CWSR_TBA_TMA_SIZE, flags, mem: &mem, kptr: &kaddr);
1402	if (ret)
1403	return ret;
1404
1405	qpd->cwsr_mem = mem;
1406	qpd->cwsr_kaddr = kaddr;
1407	qpd->tba_addr = qpd->cwsr_base;
1408
1409	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1410
1411	kfd_process_set_trap_debug_flag(qpd: &pdd->qpd,
1412	enabled: pdd->process->debug_trap_enabled);
1413
1414	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1415	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1416	qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1417
1418	return 0;
1419	}
1420
1421	static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1422	{
1423	struct kfd_node *dev = pdd->dev;
1424	struct qcm_process_device *qpd = &pdd->qpd;
1425
1426	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1427	return;
1428
1429	kfd_process_free_gpuvm(mem: qpd->cwsr_mem, pdd, kptr: &qpd->cwsr_kaddr);
1430	}
1431
1432	void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1433	uint64_t tba_addr,
1434	uint64_t tma_addr)
1435	{
1436	if (qpd->cwsr_kaddr) {
1437	/* KFD trap handler is bound, record as second-level TBA/TMA
1438	* in first-level TMA. First-level trap will jump to second.
1439	*/
1440	uint64_t *tma =
1441	(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1442	tma[0] = tba_addr;
1443	tma[1] = tma_addr;
1444	} else {
1445	/* No trap handler bound, bind as first-level TBA/TMA. */
1446	qpd->tba_addr = tba_addr;
1447	qpd->tma_addr = tma_addr;
1448	}
1449	}
1450
1451	bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1452	{
1453	int i;
1454
1455	/* On most GFXv9 GPUs, the retry mode in the SQ must match the
1456	* boot time retry setting. Mixing processes with different
1457	* XNACK/retry settings can hang the GPU.
1458	*
1459	* Different GPUs can have different noretry settings depending
1460	* on HW bugs or limitations. We need to find at least one
1461	* XNACK mode for this process that's compatible with all GPUs.
1462	* Fortunately GPUs with retry enabled (noretry=0) can run code
1463	* built for XNACK-off. On GFXv9 it may perform slower.
1464	*
1465	* Therefore applications built for XNACK-off can always be
1466	* supported and will be our fallback if any GPU does not
1467	* support retry.
1468	*/
1469	for (i = 0; i < p->n_pdds; i++) {
1470	struct kfd_node *dev = p->pdds[i]->dev;
1471
1472	/* Only consider GFXv9 and higher GPUs. Older GPUs don't
1473	* support the SVM APIs and don't need to be considered
1474	* for the XNACK mode selection.
1475	*/
1476	if (!KFD_IS_SOC15(dev))
1477	continue;
1478	/* Aldebaran can always support XNACK because it can support
1479	* per-process XNACK mode selection. But let the dev->noretry
1480	* setting still influence the default XNACK mode.
1481	*/
1482	if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1483	if (!amdgpu_sriov_xnack_support(adev: dev->kfd->adev)) {
1484	pr_debug("SRIOV platform xnack not supported\n");
1485	return false;
1486	}
1487	continue;
1488	}
1489
1490	/* GFXv10 and later GPUs do not support shader preemption
1491	* during page faults. This can lead to poor QoS for queue
1492	* management and memory-manager-related preemptions or
1493	* even deadlocks.
1494	*/
1495	if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1496	return false;
1497
1498	if (dev->kfd->noretry)
1499	return false;
1500	}
1501
1502	return true;
1503	}
1504
1505	void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1506	bool enabled)
1507	{
1508	if (qpd->cwsr_kaddr) {
1509	uint64_t *tma =
1510	(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1511	tma[2] = enabled;
1512	}
1513	}
1514
1515	/*
1516	* On return the kfd_process is fully operational and will be freed when the
1517	* mm is released
1518	*/
1519	static struct kfd_process *create_process(const struct task_struct *thread)
1520	{
1521	struct kfd_process *process;
1522	struct mmu_notifier *mn;
1523	int err = -ENOMEM;
1524
1525	process = kzalloc(sizeof(*process), GFP_KERNEL);
1526	if (!process)
1527	goto err_alloc_process;
1528
1529	kref_init(kref: &process->ref);
1530	mutex_init(&process->mutex);
1531	process->mm = thread->mm;
1532	process->lead_thread = thread->group_leader;
1533	process->n_pdds = 0;
1534	process->queues_paused = false;
1535	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1536	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1537	process->last_restore_timestamp = get_jiffies_64();
1538	err = kfd_event_init_process(p: process);
1539	if (err)
1540	goto err_event_init;
1541	process->is_32bit_user_mode = in_compat_syscall();
1542	process->debug_trap_enabled = false;
1543	process->debugger_process = NULL;
1544	process->exception_enable_mask = 0;
1545	atomic_set(v: &process->debugged_process_count, i: 0);
1546	sema_init(sem: &process->runtime_enable_sema, val: 0);
1547
1548	err = pqm_init(pqm: &process->pqm, p: process);
1549	if (err != 0)
1550	goto err_process_pqm_init;
1551
1552	/* init process apertures*/
1553	err = kfd_init_apertures(process);
1554	if (err != 0)
1555	goto err_init_apertures;
1556
1557	/* Check XNACK support after PDDs are created in kfd_init_apertures */
1558	process->xnack_enabled = kfd_process_xnack_mode(p: process, supported: false);
1559
1560	err = svm_range_list_init(p: process);
1561	if (err)
1562	goto err_init_svm_range_list;
1563
1564	/* alloc_notifier needs to find the process in the hash table */
1565	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1566	(uintptr_t)process->mm);
1567
1568	/* Avoid free_notifier to start kfd_process_wq_release if
1569	* mmu_notifier_get failed because of pending signal.
1570	*/
1571	kref_get(kref: &process->ref);
1572
1573	/* MMU notifier registration must be the last call that can fail
1574	* because after this point we cannot unwind the process creation.
1575	* After this point, mmu_notifier_put will trigger the cleanup by
1576	* dropping the last process reference in the free_notifier.
1577	*/
1578	mn = mmu_notifier_get(ops: &kfd_process_mmu_notifier_ops, mm: process->mm);
1579	if (IS_ERR(ptr: mn)) {
1580	err = PTR_ERR(ptr: mn);
1581	goto err_register_notifier;
1582	}
1583	BUG_ON(mn != &process->mmu_notifier);
1584
1585	kfd_unref_process(p: process);
1586	get_task_struct(t: process->lead_thread);
1587
1588	INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1589
1590	return process;
1591
1592	err_register_notifier:
1593	hash_del_rcu(node: &process->kfd_processes);
1594	svm_range_list_fini(p: process);
1595	err_init_svm_range_list:
1596	kfd_process_free_outstanding_kfd_bos(p: process);
1597	kfd_process_destroy_pdds(p: process);
1598	err_init_apertures:
1599	pqm_uninit(pqm: &process->pqm);
1600	err_process_pqm_init:
1601	kfd_event_free_process(p: process);
1602	err_event_init:
1603	mutex_destroy(lock: &process->mutex);
1604	kfree(objp: process);
1605	err_alloc_process:
1606	return ERR_PTR(error: err);
1607	}
1608
1609	struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1610	struct kfd_process *p)
1611	{
1612	int i;
1613
1614	for (i = 0; i < p->n_pdds; i++)
1615	if (p->pdds[i]->dev == dev)
1616	return p->pdds[i];
1617
1618	return NULL;
1619	}
1620
1621	struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1622	struct kfd_process *p)
1623	{
1624	struct kfd_process_device *pdd = NULL;
1625
1626	if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1627	return NULL;
1628	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1629	if (!pdd)
1630	return NULL;
1631
1632	pdd->dev = dev;
1633	INIT_LIST_HEAD(list: &pdd->qpd.queues_list);
1634	INIT_LIST_HEAD(list: &pdd->qpd.priv_queue_list);
1635	pdd->qpd.dqm = dev->dqm;
1636	pdd->qpd.pqm = &p->pqm;
1637	pdd->qpd.evicted = 0;
1638	pdd->qpd.mapped_gws_queue = false;
1639	pdd->process = p;
1640	pdd->bound = PDD_UNBOUND;
1641	pdd->already_dequeued = false;
1642	pdd->runtime_inuse = false;
1643	atomic64_set(v: &pdd->vram_usage, i: 0);
1644	pdd->sdma_past_activity_counter = 0;
1645	pdd->user_gpu_id = dev->id;
1646	atomic64_set(v: &pdd->evict_duration_counter, i: 0);
1647
1648	p->pdds[p->n_pdds++] = pdd;
1649	if (kfd_dbg_is_per_vmid_supported(dev: pdd->dev))
1650	pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1651	pdd->dev->adev,
1652	false,
1653	0);
1654
1655	/* Init idr used for memory handle translation */
1656	idr_init(idr: &pdd->alloc_idr);
1657
1658	atomic_inc(v: &dev->kfd->kfd_processes_count);
1659
1660	return pdd;
1661	}
1662
1663	/**
1664	* kfd_process_device_init_vm - Initialize a VM for a process-device
1665	*
1666	* @pdd: The process-device
1667	* @drm_file: Optional pointer to a DRM file descriptor
1668	*
1669	* If @drm_file is specified, it will be used to acquire the VM from
1670	* that file descriptor. If successful, the @pdd takes ownership of
1671	* the file descriptor.
1672	*
1673	* If @drm_file is NULL, a new VM is created.
1674	*
1675	* Returns 0 on success, -errno on failure.
1676	*/
1677	int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1678	struct file *drm_file)
1679	{
1680	struct amdgpu_fpriv *drv_priv;
1681	struct amdgpu_vm *avm;
1682	struct kfd_process *p;
1683	struct dma_fence *ef;
1684	struct kfd_node *dev;
1685	int ret;
1686
1687	if (!drm_file)
1688	return -EINVAL;
1689
1690	if (pdd->drm_priv)
1691	return -EBUSY;
1692
1693	ret = amdgpu_file_to_fpriv(filp: drm_file, fpriv: &drv_priv);
1694	if (ret)
1695	return ret;
1696	avm = &drv_priv->vm;
1697
1698	p = pdd->process;
1699	dev = pdd->dev;
1700
1701	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(adev: dev->adev, avm,
1702	process_info: &p->kgd_process_info,
1703	ef: p->ef ? NULL : &ef);
1704	if (ret) {
1705	dev_err(dev->adev->dev, "Failed to create process VM object\n");
1706	return ret;
1707	}
1708
1709	if (!p->ef)
1710	RCU_INIT_POINTER(p->ef, ef);
1711
1712	pdd->drm_priv = drm_file->private_data;
1713
1714	ret = kfd_process_device_reserve_ib_mem(pdd);
1715	if (ret)
1716	goto err_reserve_ib_mem;
1717	ret = kfd_process_device_init_cwsr_dgpu(pdd);
1718	if (ret)
1719	goto err_init_cwsr;
1720
1721	if (unlikely(!avm->pasid)) {
1722	dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated",
1723	avm);
1724	ret = -EINVAL;
1725	goto err_get_pasid;
1726	}
1727
1728	pdd->pasid = avm->pasid;
1729	pdd->drm_file = drm_file;
1730
1731	kfd_smi_event_process(pdd, start: true);
1732
1733	return 0;
1734
1735	err_get_pasid:
1736	kfd_process_device_destroy_cwsr_dgpu(pdd);
1737	err_init_cwsr:
1738	kfd_process_device_destroy_ib_mem(pdd);
1739	err_reserve_ib_mem:
1740	pdd->drm_priv = NULL;
1741	amdgpu_amdkfd_gpuvm_destroy_cb(adev: dev->adev, vm: avm);
1742
1743	return ret;
1744	}
1745
1746	/*
1747	* Direct the IOMMU to bind the process (specifically the pasid->mm)
1748	* to the device.
1749	* Unbinding occurs when the process dies or the device is removed.
1750	*
1751	* Assumes that the process lock is held.
1752	*/
1753	struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1754	struct kfd_process *p)
1755	{
1756	struct kfd_process_device *pdd;
1757	int err;
1758
1759	pdd = kfd_get_process_device_data(dev, p);
1760	if (!pdd) {
1761	dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1762	return ERR_PTR(error: -ENOMEM);
1763	}
1764
1765	if (!pdd->drm_priv)
1766	return ERR_PTR(error: -ENODEV);
1767
1768	/*
1769	* signal runtime-pm system to auto resume and prevent
1770	* further runtime suspend once device pdd is created until
1771	* pdd is destroyed.
1772	*/
1773	if (!pdd->runtime_inuse) {
1774	err = pm_runtime_get_sync(dev: adev_to_drm(adev: dev->adev)->dev);
1775	if (err < 0) {
1776	pm_runtime_put_autosuspend(dev: adev_to_drm(adev: dev->adev)->dev);
1777	return ERR_PTR(error: err);
1778	}
1779	}
1780
1781	/*
1782	* make sure that runtime_usage counter is incremented just once
1783	* per pdd
1784	*/
1785	pdd->runtime_inuse = true;
1786
1787	return pdd;
1788	}
1789
1790	/* Create specific handle mapped to mem from process local memory idr
1791	* Assumes that the process lock is held.
1792	*/
1793	int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1794	void *mem)
1795	{
1796	return idr_alloc(&pdd->alloc_idr, ptr: mem, start: 0, end: 0, GFP_KERNEL);
1797	}
1798
1799	/* Translate specific handle from process local memory idr
1800	* Assumes that the process lock is held.
1801	*/
1802	void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1803	int handle)
1804	{
1805	if (handle < 0)
1806	return NULL;
1807
1808	return idr_find(&pdd->alloc_idr, id: handle);
1809	}
1810
1811	/* Remove specific handle from process local memory idr
1812	* Assumes that the process lock is held.
1813	*/
1814	void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1815	int handle)
1816	{
1817	if (handle >= 0)
1818	idr_remove(&pdd->alloc_idr, id: handle);
1819	}
1820
1821	static struct kfd_process_device *kfd_lookup_process_device_by_pasid(u32 pasid)
1822	{
1823	struct kfd_process_device *ret_p = NULL;
1824	struct kfd_process *p;
1825	unsigned int temp;
1826	int i;
1827
1828	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1829	for (i = 0; i < p->n_pdds; i++) {
1830	if (p->pdds[i]->pasid == pasid) {
1831	ret_p = p->pdds[i];
1832	break;
1833	}
1834	}
1835	if (ret_p)
1836	break;
1837	}
1838	return ret_p;
1839	}
1840
1841	/* This increments the process->ref counter. */
1842	struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
1843	struct kfd_process_device **pdd)
1844	{
1845	struct kfd_process_device *ret_p;
1846
1847	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1848
1849	ret_p = kfd_lookup_process_device_by_pasid(pasid);
1850	if (ret_p) {
1851	if (pdd)
1852	*pdd = ret_p;
1853	kref_get(kref: &ret_p->process->ref);
1854
1855	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1856	return ret_p->process;
1857	}
1858
1859	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1860
1861	if (pdd)
1862	*pdd = NULL;
1863
1864	return NULL;
1865	}
1866
1867	/* This increments the process->ref counter. */
1868	struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1869	{
1870	struct kfd_process *p;
1871
1872	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
1873
1874	p = find_process_by_mm(mm);
1875	if (p)
1876	kref_get(kref: &p->ref);
1877
1878	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
1879
1880	return p;
1881	}
1882
1883	/* kfd_process_evict_queues - Evict all user queues of a process
1884	*
1885	* Eviction is reference-counted per process-device. This means multiple
1886	* evictions from different sources can be nested safely.
1887	*/
1888	int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1889	{
1890	int r = 0;
1891	int i;
1892	unsigned int n_evicted = 0;
1893
1894	for (i = 0; i < p->n_pdds; i++) {
1895	struct kfd_process_device *pdd = p->pdds[i];
1896	struct device *dev = pdd->dev->adev->dev;
1897
1898	kfd_smi_event_queue_eviction(node: pdd->dev, pid: p->lead_thread->pid,
1899	trigger);
1900
1901	r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1902	&pdd->qpd);
1903	/* evict return -EIO if HWS is hang or asic is resetting, in this case
1904	* we would like to set all the queues to be in evicted state to prevent
1905	* them been add back since they actually not be saved right now.
1906	*/
1907	if (r && r != -EIO) {
1908	dev_err(dev, "Failed to evict process queues\n");
1909	goto fail;
1910	}
1911	n_evicted++;
1912
1913	pdd->dev->dqm->is_hws_hang = false;
1914	}
1915
1916	return r;
1917
1918	fail:
1919	/* To keep state consistent, roll back partial eviction by
1920	* restoring queues
1921	*/
1922	for (i = 0; i < p->n_pdds; i++) {
1923	struct kfd_process_device *pdd = p->pdds[i];
1924
1925	if (n_evicted == 0)
1926	break;
1927
1928	kfd_smi_event_queue_restore(node: pdd->dev, pid: p->lead_thread->pid);
1929
1930	if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1931	&pdd->qpd))
1932	dev_err(pdd->dev->adev->dev,
1933	"Failed to restore queues\n");
1934
1935	n_evicted--;
1936	}
1937
1938	return r;
1939	}
1940
1941	/* kfd_process_restore_queues - Restore all user queues of a process */
1942	int kfd_process_restore_queues(struct kfd_process *p)
1943	{
1944	int r, ret = 0;
1945	int i;
1946
1947	for (i = 0; i < p->n_pdds; i++) {
1948	struct kfd_process_device *pdd = p->pdds[i];
1949	struct device *dev = pdd->dev->adev->dev;
1950
1951	kfd_smi_event_queue_restore(node: pdd->dev, pid: p->lead_thread->pid);
1952
1953	r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1954	&pdd->qpd);
1955	if (r) {
1956	dev_err(dev, "Failed to restore process queues\n");
1957	if (!ret)
1958	ret = r;
1959	}
1960	}
1961
1962	return ret;
1963	}
1964
1965	int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1966	{
1967	int i;
1968
1969	for (i = 0; i < p->n_pdds; i++)
1970	if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1971	return i;
1972	return -EINVAL;
1973	}
1974
1975	int
1976	kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1977	uint32_t *gpuid, uint32_t *gpuidx)
1978	{
1979	int i;
1980
1981	for (i = 0; i < p->n_pdds; i++)
1982	if (p->pdds[i] && p->pdds[i]->dev == node) {
1983	*gpuid = p->pdds[i]->user_gpu_id;
1984	*gpuidx = i;
1985	return 0;
1986	}
1987	return -EINVAL;
1988	}
1989
1990	static int signal_eviction_fence(struct kfd_process *p)
1991	{
1992	struct dma_fence *ef;
1993	int ret;
1994
1995	rcu_read_lock();
1996	ef = dma_fence_get_rcu_safe(fencep: &p->ef);
1997	rcu_read_unlock();
1998	if (!ef)
1999	return -EINVAL;
2000
2001	ret = dma_fence_signal(fence: ef);
2002	dma_fence_put(fence: ef);
2003
2004	return ret;
2005	}
2006
2007	static void evict_process_worker(struct work_struct *work)
2008	{
2009	int ret;
2010	struct kfd_process *p;
2011	struct delayed_work *dwork;
2012
2013	dwork = to_delayed_work(work);
2014
2015	/* Process termination destroys this worker thread. So during the
2016	* lifetime of this thread, kfd_process p will be valid
2017	*/
2018	p = container_of(dwork, struct kfd_process, eviction_work);
2019
2020	pr_debug("Started evicting process pid %d\n", p->lead_thread->pid);
2021	ret = kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_TRIGGER_TTM);
2022	if (!ret) {
2023	/* If another thread already signaled the eviction fence,
2024	* they are responsible stopping the queues and scheduling
2025	* the restore work.
2026	*/
2027	if (signal_eviction_fence(p) ||
2028	mod_delayed_work(wq: kfd_restore_wq, dwork: &p->restore_work,
2029	delay: msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2030	kfd_process_restore_queues(p);
2031
2032	pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid);
2033	} else
2034	pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid);
2035	}
2036
2037	static int restore_process_helper(struct kfd_process *p)
2038	{
2039	int ret = 0;
2040
2041	/* VMs may not have been acquired yet during debugging. */
2042	if (p->kgd_process_info) {
2043	ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
2044	process_info: p->kgd_process_info, ef: &p->ef);
2045	if (ret)
2046	return ret;
2047	}
2048
2049	ret = kfd_process_restore_queues(p);
2050	if (!ret)
2051	pr_debug("Finished restoring process pid %d\n",
2052	p->lead_thread->pid);
2053	else
2054	pr_err("Failed to restore queues of process pid %d\n",
2055	p->lead_thread->pid);
2056
2057	return ret;
2058	}
2059
2060	static void restore_process_worker(struct work_struct *work)
2061	{
2062	struct delayed_work *dwork;
2063	struct kfd_process *p;
2064	int ret = 0;
2065
2066	dwork = to_delayed_work(work);
2067
2068	/* Process termination destroys this worker thread. So during the
2069	* lifetime of this thread, kfd_process p will be valid
2070	*/
2071	p = container_of(dwork, struct kfd_process, restore_work);
2072	pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid);
2073
2074	/* Setting last_restore_timestamp before successful restoration.
2075	* Otherwise this would have to be set by KGD (restore_process_bos)
2076	* before KFD BOs are unreserved. If not, the process can be evicted
2077	* again before the timestamp is set.
2078	* If restore fails, the timestamp will be set again in the next
2079	* attempt. This would mean that the minimum GPU quanta would be
2080	* PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2081	* functions)
2082	*/
2083
2084	p->last_restore_timestamp = get_jiffies_64();
2085
2086	ret = restore_process_helper(p);
2087	if (ret) {
2088	pr_debug("Failed to restore BOs of process pid %d, retry after %d ms\n",
2089	p->lead_thread->pid, PROCESS_BACK_OFF_TIME_MS);
2090	if (mod_delayed_work(wq: kfd_restore_wq, dwork: &p->restore_work,
2091	delay: msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2092	kfd_process_restore_queues(p);
2093	}
2094	}
2095
2096	void kfd_suspend_all_processes(void)
2097	{
2098	struct kfd_process *p;
2099	unsigned int temp;
2100	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
2101
2102	WARN(debug_evictions, "Evicting all processes");
2103	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2104	if (kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2105	pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid);
2106	signal_eviction_fence(p);
2107	}
2108	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
2109	}
2110
2111	int kfd_resume_all_processes(void)
2112	{
2113	struct kfd_process *p;
2114	unsigned int temp;
2115	int ret = 0, idx = srcu_read_lock(ssp: &kfd_processes_srcu);
2116
2117	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2118	if (restore_process_helper(p)) {
2119	pr_err("Restore process pid %d failed during resume\n",
2120	p->lead_thread->pid);
2121	ret = -EFAULT;
2122	}
2123	}
2124	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
2125	return ret;
2126	}
2127
2128	int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2129	struct vm_area_struct *vma)
2130	{
2131	struct kfd_process_device *pdd;
2132	struct qcm_process_device *qpd;
2133
2134	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2135	dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
2136	return -EINVAL;
2137	}
2138
2139	pdd = kfd_get_process_device_data(dev, p: process);
2140	if (!pdd)
2141	return -EINVAL;
2142	qpd = &pdd->qpd;
2143
2144	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2145	get_order(KFD_CWSR_TBA_TMA_SIZE));
2146	if (!qpd->cwsr_kaddr) {
2147	dev_err(dev->adev->dev,
2148	"Error allocating per process CWSR buffer.\n");
2149	return -ENOMEM;
2150	}
2151
2152	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2153	| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2154	/* Mapping pages to user process */
2155	return remap_pfn_range(vma, addr: vma->vm_start,
2156	PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2157	KFD_CWSR_TBA_TMA_SIZE, pgprot: vma->vm_page_prot);
2158	}
2159
2160	/* assumes caller holds process lock. */
2161	int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2162	{
2163	uint32_t irq_drain_fence[8];
2164	uint8_t node_id = 0;
2165	int r = 0;
2166
2167	if (!KFD_IS_SOC15(pdd->dev))
2168	return 0;
2169
2170	pdd->process->irq_drain_is_open = true;
2171
2172	memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2173	irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2174	KFD_IRQ_FENCE_CLIENTID;
2175	irq_drain_fence[3] = pdd->pasid;
2176
2177	/*
2178	* For GFX 9.4.3/9.5.0, send the NodeId also in IH cookie DW[3]
2179	*/
2180	if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
2181	KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||
2182	KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) {
2183	node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2184	irq_drain_fence[3] |= node_id << 16;
2185	}
2186
2187	/* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2188	if (amdgpu_amdkfd_send_close_event_drain_irq(adev: pdd->dev->adev,
2189	payload: irq_drain_fence)) {
2190	pdd->process->irq_drain_is_open = false;
2191	return 0;
2192	}
2193
2194	r = wait_event_interruptible(pdd->process->wait_irq_drain,
2195	!READ_ONCE(pdd->process->irq_drain_is_open));
2196	if (r)
2197	pdd->process->irq_drain_is_open = false;
2198
2199	return r;
2200	}
2201
2202	void kfd_process_close_interrupt_drain(unsigned int pasid)
2203	{
2204	struct kfd_process *p;
2205
2206	p = kfd_lookup_process_by_pasid(pasid, NULL);
2207
2208	if (!p)
2209	return;
2210
2211	WRITE_ONCE(p->irq_drain_is_open, false);
2212	wake_up_all(&p->wait_irq_drain);
2213	kfd_unref_process(p);
2214	}
2215
2216	struct send_exception_work_handler_workarea {
2217	struct work_struct work;
2218	struct kfd_process *p;
2219	unsigned int queue_id;
2220	uint64_t error_reason;
2221	};
2222
2223	static void send_exception_work_handler(struct work_struct *work)
2224	{
2225	struct send_exception_work_handler_workarea *workarea;
2226	struct kfd_process *p;
2227	struct queue *q;
2228	struct mm_struct *mm;
2229	struct kfd_context_save_area_header __user *csa_header;
2230	uint64_t __user *err_payload_ptr;
2231	uint64_t cur_err;
2232	uint32_t ev_id;
2233
2234	workarea = container_of(work,
2235	struct send_exception_work_handler_workarea,
2236	work);
2237	p = workarea->p;
2238
2239	mm = get_task_mm(task: p->lead_thread);
2240
2241	if (!mm)
2242	return;
2243
2244	kthread_use_mm(mm);
2245
2246	q = pqm_get_user_queue(pqm: &p->pqm, qid: workarea->queue_id);
2247
2248	if (!q)
2249	goto out;
2250
2251	csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2252
2253	get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2254	get_user(cur_err, err_payload_ptr);
2255	cur_err |= workarea->error_reason;
2256	put_user(cur_err, err_payload_ptr);
2257	get_user(ev_id, &csa_header->err_event_id);
2258
2259	kfd_set_event(p, event_id: ev_id);
2260
2261	out:
2262	kthread_unuse_mm(mm);
2263	mmput(mm);
2264	}
2265
2266	int kfd_send_exception_to_runtime(struct kfd_process *p,
2267	unsigned int queue_id,
2268	uint64_t error_reason)
2269	{
2270	struct send_exception_work_handler_workarea worker;
2271
2272	INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2273
2274	worker.p = p;
2275	worker.queue_id = queue_id;
2276	worker.error_reason = error_reason;
2277
2278	schedule_work(work: &worker.work);
2279	flush_work(work: &worker.work);
2280	destroy_work_on_stack(work: &worker.work);
2281
2282	return 0;
2283	}
2284
2285	struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2286	{
2287	int i;
2288
2289	if (gpu_id) {
2290	for (i = 0; i < p->n_pdds; i++) {
2291	struct kfd_process_device *pdd = p->pdds[i];
2292
2293	if (pdd->user_gpu_id == gpu_id)
2294	return pdd;
2295	}
2296	}
2297	return NULL;
2298	}
2299
2300	int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2301	{
2302	int i;
2303
2304	if (!actual_gpu_id)
2305	return 0;
2306
2307	for (i = 0; i < p->n_pdds; i++) {
2308	struct kfd_process_device *pdd = p->pdds[i];
2309
2310	if (pdd->dev->id == actual_gpu_id)
2311	return pdd->user_gpu_id;
2312	}
2313	return -EINVAL;
2314	}
2315
2316	#if defined(CONFIG_DEBUG_FS)
2317
2318	int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2319	{
2320	struct kfd_process *p;
2321	unsigned int temp;
2322	int r = 0;
2323
2324	int idx = srcu_read_lock(ssp: &kfd_processes_srcu);
2325
2326	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2327	seq_printf(m, fmt: "Process %d PASID %d:\n",
2328	p->lead_thread->tgid, p->lead_thread->pid);
2329
2330	mutex_lock(&p->mutex);
2331	r = pqm_debugfs_mqds(m, data: &p->pqm);
2332	mutex_unlock(lock: &p->mutex);
2333
2334	if (r)
2335	break;
2336	}
2337
2338	srcu_read_unlock(ssp: &kfd_processes_srcu, idx);
2339
2340	return r;
2341	}
2342
2343	#endif
2344