1	/*
2	* Copyright 2019 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/firmware.h>
25	#include <drm/drm_exec.h>
26
27	#include "amdgpu_mes.h"
28	#include "amdgpu.h"
29	#include "soc15_common.h"
30	#include "amdgpu_mes_ctx.h"
31
32	#define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
33	#define AMDGPU_ONE_DOORBELL_SIZE 8
34
35	int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
36	{
37	return roundup(AMDGPU_ONE_DOORBELL_SIZE *
38	AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,
39	PAGE_SIZE);
40	}
41
42	static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,
43	struct amdgpu_mes_process *process,
44	int ip_type, uint64_t *doorbell_index)
45	{
46	unsigned int offset, found;
47	struct amdgpu_mes *mes = &adev->mes;
48
49	if (ip_type == AMDGPU_RING_TYPE_SDMA)
50	offset = adev->doorbell_index.sdma_engine[0];
51	else
52	offset = 0;
53
54	found = find_next_zero_bit(addr: mes->doorbell_bitmap, size: mes->num_mes_dbs, offset);
55	if (found >= mes->num_mes_dbs) {
56	DRM_WARN("No doorbell available\n");
57	return -ENOSPC;
58	}
59
60	set_bit(nr: found, addr: mes->doorbell_bitmap);
61
62	/* Get the absolute doorbell index on BAR */
63	*doorbell_index = mes->db_start_dw_offset + found * 2;
64	return 0;
65	}
66
67	static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev,
68	struct amdgpu_mes_process *process,
69	uint32_t doorbell_index)
70	{
71	unsigned int old, rel_index;
72	struct amdgpu_mes *mes = &adev->mes;
73
74	/* Find the relative index of the doorbell in this object */
75	rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;
76	old = test_and_clear_bit(nr: rel_index, addr: mes->doorbell_bitmap);
77	WARN_ON(!old);
78	}
79
80	static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev)
81	{
82	int i;
83	struct amdgpu_mes *mes = &adev->mes;
84
85	/* Bitmap for dynamic allocation of kernel doorbells */
86	mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);
87	if (!mes->doorbell_bitmap) {
88	DRM_ERROR("Failed to allocate MES doorbell bitmap\n");
89	return -ENOMEM;
90	}
91
92	mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;
93	for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) {
94	adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;
95	set_bit(nr: i, addr: mes->doorbell_bitmap);
96	}
97
98	return 0;
99	}
100
101	static int amdgpu_mes_event_log_init(struct amdgpu_device *adev)
102	{
103	int r;
104
105	if (!amdgpu_mes_log_enable)
106	return 0;
107
108	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_LOG_BUFFER_SIZE, PAGE_SIZE,
109	AMDGPU_GEM_DOMAIN_GTT,
110	bo_ptr: &adev->mes.event_log_gpu_obj,
111	gpu_addr: &adev->mes.event_log_gpu_addr,
112	cpu_addr: &adev->mes.event_log_cpu_addr);
113	if (r) {
114	dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r);
115	return r;
116	}
117
118	memset(adev->mes.event_log_cpu_addr, 0, PAGE_SIZE);
119
120	return 0;
121
122	}
123
124	static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev)
125	{
126	bitmap_free(bitmap: adev->mes.doorbell_bitmap);
127	}
128
129	int amdgpu_mes_init(struct amdgpu_device *adev)
130	{
131	int i, r;
132
133	adev->mes.adev = adev;
134
135	idr_init(idr: &adev->mes.pasid_idr);
136	idr_init(idr: &adev->mes.gang_id_idr);
137	idr_init(idr: &adev->mes.queue_id_idr);
138	ida_init(ida: &adev->mes.doorbell_ida);
139	spin_lock_init(&adev->mes.queue_id_lock);
140	spin_lock_init(&adev->mes.ring_lock);
141	mutex_init(&adev->mes.mutex_hidden);
142
143	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
144	adev->mes.vmid_mask_mmhub = 0xffffff00;
145	adev->mes.vmid_mask_gfxhub = 0xffffff00;
146
147	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
148	/* use only 1st MEC pipes */
149	if (i >= 4)
150	continue;
151	adev->mes.compute_hqd_mask[i] = 0xc;
152	}
153
154	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
155	adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;
156
157	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
158	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) <
159	IP_VERSION(6, 0, 0))
160	adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;
161	/* zero sdma_hqd_mask for non-existent engine */
162	else if (adev->sdma.num_instances == 1)
163	adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;
164	else
165	adev->mes.sdma_hqd_mask[i] = 0xfc;
166	}
167
168	r = amdgpu_device_wb_get(adev, wb: &adev->mes.sch_ctx_offs);
169	if (r) {
170	dev_err(adev->dev,
171	"(%d) ring trail_fence_offs wb alloc failed\n", r);
172	goto error_ids;
173	}
174	adev->mes.sch_ctx_gpu_addr =
175	adev->wb.gpu_addr + (adev->mes.sch_ctx_offs * 4);
176	adev->mes.sch_ctx_ptr =
177	(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs];
178
179	r = amdgpu_device_wb_get(adev, wb: &adev->mes.query_status_fence_offs);
180	if (r) {
181	amdgpu_device_wb_free(adev, wb: adev->mes.sch_ctx_offs);
182	dev_err(adev->dev,
183	"(%d) query_status_fence_offs wb alloc failed\n", r);
184	goto error_ids;
185	}
186	adev->mes.query_status_fence_gpu_addr =
187	adev->wb.gpu_addr + (adev->mes.query_status_fence_offs * 4);
188	adev->mes.query_status_fence_ptr =
189	(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs];
190
191	r = amdgpu_device_wb_get(adev, wb: &adev->mes.read_val_offs);
192	if (r) {
193	amdgpu_device_wb_free(adev, wb: adev->mes.sch_ctx_offs);
194	amdgpu_device_wb_free(adev, wb: adev->mes.query_status_fence_offs);
195	dev_err(adev->dev,
196	"(%d) read_val_offs alloc failed\n", r);
197	goto error_ids;
198	}
199	adev->mes.read_val_gpu_addr =
200	adev->wb.gpu_addr + (adev->mes.read_val_offs * 4);
201	adev->mes.read_val_ptr =
202	(uint32_t *)&adev->wb.wb[adev->mes.read_val_offs];
203
204	r = amdgpu_mes_doorbell_init(adev);
205	if (r)
206	goto error;
207
208	r = amdgpu_mes_event_log_init(adev);
209	if (r)
210	goto error_doorbell;
211
212	return 0;
213
214	error_doorbell:
215	amdgpu_mes_doorbell_free(adev);
216	error:
217	amdgpu_device_wb_free(adev, wb: adev->mes.sch_ctx_offs);
218	amdgpu_device_wb_free(adev, wb: adev->mes.query_status_fence_offs);
219	amdgpu_device_wb_free(adev, wb: adev->mes.read_val_offs);
220	error_ids:
221	idr_destroy(&adev->mes.pasid_idr);
222	idr_destroy(&adev->mes.gang_id_idr);
223	idr_destroy(&adev->mes.queue_id_idr);
224	ida_destroy(ida: &adev->mes.doorbell_ida);
225	mutex_destroy(lock: &adev->mes.mutex_hidden);
226	return r;
227	}
228
229	void amdgpu_mes_fini(struct amdgpu_device *adev)
230	{
231	amdgpu_bo_free_kernel(bo: &adev->mes.event_log_gpu_obj,
232	gpu_addr: &adev->mes.event_log_gpu_addr,
233	cpu_addr: &adev->mes.event_log_cpu_addr);
234
235	amdgpu_device_wb_free(adev, wb: adev->mes.sch_ctx_offs);
236	amdgpu_device_wb_free(adev, wb: adev->mes.query_status_fence_offs);
237	amdgpu_device_wb_free(adev, wb: adev->mes.read_val_offs);
238	amdgpu_mes_doorbell_free(adev);
239
240	idr_destroy(&adev->mes.pasid_idr);
241	idr_destroy(&adev->mes.gang_id_idr);
242	idr_destroy(&adev->mes.queue_id_idr);
243	ida_destroy(ida: &adev->mes.doorbell_ida);
244	mutex_destroy(lock: &adev->mes.mutex_hidden);
245	}
246
247	static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q)
248	{
249	amdgpu_bo_free_kernel(bo: &q->mqd_obj,
250	gpu_addr: &q->mqd_gpu_addr,
251	cpu_addr: &q->mqd_cpu_ptr);
252	}
253
254	int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid,
255	struct amdgpu_vm *vm)
256	{
257	struct amdgpu_mes_process *process;
258	int r;
259
260	/* allocate the mes process buffer */
261	process = kzalloc(size: sizeof(struct amdgpu_mes_process), GFP_KERNEL);
262	if (!process) {
263	DRM_ERROR("no more memory to create mes process\n");
264	return -ENOMEM;
265	}
266
267	/* allocate the process context bo and map it */
268	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE,
269	AMDGPU_GEM_DOMAIN_GTT,
270	bo_ptr: &process->proc_ctx_bo,
271	gpu_addr: &process->proc_ctx_gpu_addr,
272	cpu_addr: &process->proc_ctx_cpu_ptr);
273	if (r) {
274	DRM_ERROR("failed to allocate process context bo\n");
275	goto clean_up_memory;
276	}
277	memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
278
279	/*
280	* Avoid taking any other locks under MES lock to avoid circular
281	* lock dependencies.
282	*/
283	amdgpu_mes_lock(mes: &adev->mes);
284
285	/* add the mes process to idr list */
286	r = idr_alloc(&adev->mes.pasid_idr, ptr: process, start: pasid, end: pasid + 1,
287	GFP_KERNEL);
288	if (r < 0) {
289	DRM_ERROR("failed to lock pasid=%d\n", pasid);
290	goto clean_up_ctx;
291	}
292
293	INIT_LIST_HEAD(list: &process->gang_list);
294	process->vm = vm;
295	process->pasid = pasid;
296	process->process_quantum = adev->mes.default_process_quantum;
297	process->pd_gpu_addr = amdgpu_bo_gpu_offset(bo: vm->root.bo);
298
299	amdgpu_mes_unlock(mes: &adev->mes);
300	return 0;
301
302	clean_up_ctx:
303	amdgpu_mes_unlock(mes: &adev->mes);
304	amdgpu_bo_free_kernel(bo: &process->proc_ctx_bo,
305	gpu_addr: &process->proc_ctx_gpu_addr,
306	cpu_addr: &process->proc_ctx_cpu_ptr);
307	clean_up_memory:
308	kfree(objp: process);
309	return r;
310	}
311
312	void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid)
313	{
314	struct amdgpu_mes_process *process;
315	struct amdgpu_mes_gang *gang, *tmp1;
316	struct amdgpu_mes_queue *queue, *tmp2;
317	struct mes_remove_queue_input queue_input;
318	unsigned long flags;
319	int r;
320
321	/*
322	* Avoid taking any other locks under MES lock to avoid circular
323	* lock dependencies.
324	*/
325	amdgpu_mes_lock(mes: &adev->mes);
326
327	process = idr_find(&adev->mes.pasid_idr, id: pasid);
328	if (!process) {
329	DRM_WARN("pasid %d doesn't exist\n", pasid);
330	amdgpu_mes_unlock(mes: &adev->mes);
331	return;
332	}
333
334	/* Remove all queues from hardware */
335	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
336	list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
337	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
338	idr_remove(&adev->mes.queue_id_idr, id: queue->queue_id);
339	spin_unlock_irqrestore(lock: &adev->mes.queue_id_lock, flags);
340
341	queue_input.doorbell_offset = queue->doorbell_off;
342	queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
343
344	r = adev->mes.funcs->remove_hw_queue(&adev->mes,
345	&queue_input);
346	if (r)
347	DRM_WARN("failed to remove hardware queue\n");
348	}
349
350	idr_remove(&adev->mes.gang_id_idr, id: gang->gang_id);
351	}
352
353	idr_remove(&adev->mes.pasid_idr, id: pasid);
354	amdgpu_mes_unlock(mes: &adev->mes);
355
356	/* free all memory allocated by the process */
357	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
358	/* free all queues in the gang */
359	list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
360	amdgpu_mes_queue_free_mqd(q: queue);
361	list_del(entry: &queue->list);
362	kfree(objp: queue);
363	}
364	amdgpu_bo_free_kernel(bo: &gang->gang_ctx_bo,
365	gpu_addr: &gang->gang_ctx_gpu_addr,
366	cpu_addr: &gang->gang_ctx_cpu_ptr);
367	list_del(entry: &gang->list);
368	kfree(objp: gang);
369
370	}
371	amdgpu_bo_free_kernel(bo: &process->proc_ctx_bo,
372	gpu_addr: &process->proc_ctx_gpu_addr,
373	cpu_addr: &process->proc_ctx_cpu_ptr);
374	kfree(objp: process);
375	}
376
377	int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid,
378	struct amdgpu_mes_gang_properties *gprops,
379	int *gang_id)
380	{
381	struct amdgpu_mes_process *process;
382	struct amdgpu_mes_gang *gang;
383	int r;
384
385	/* allocate the mes gang buffer */
386	gang = kzalloc(size: sizeof(struct amdgpu_mes_gang), GFP_KERNEL);
387	if (!gang) {
388	return -ENOMEM;
389	}
390
391	/* allocate the gang context bo and map it to cpu space */
392	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE,
393	AMDGPU_GEM_DOMAIN_GTT,
394	bo_ptr: &gang->gang_ctx_bo,
395	gpu_addr: &gang->gang_ctx_gpu_addr,
396	cpu_addr: &gang->gang_ctx_cpu_ptr);
397	if (r) {
398	DRM_ERROR("failed to allocate process context bo\n");
399	goto clean_up_mem;
400	}
401	memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
402
403	/*
404	* Avoid taking any other locks under MES lock to avoid circular
405	* lock dependencies.
406	*/
407	amdgpu_mes_lock(mes: &adev->mes);
408
409	process = idr_find(&adev->mes.pasid_idr, id: pasid);
410	if (!process) {
411	DRM_ERROR("pasid %d doesn't exist\n", pasid);
412	r = -EINVAL;
413	goto clean_up_ctx;
414	}
415
416	/* add the mes gang to idr list */
417	r = idr_alloc(&adev->mes.gang_id_idr, ptr: gang, start: 1, end: 0,
418	GFP_KERNEL);
419	if (r < 0) {
420	DRM_ERROR("failed to allocate idr for gang\n");
421	goto clean_up_ctx;
422	}
423
424	gang->gang_id = r;
425	*gang_id = r;
426
427	INIT_LIST_HEAD(list: &gang->queue_list);
428	gang->process = process;
429	gang->priority = gprops->priority;
430	gang->gang_quantum = gprops->gang_quantum ?
431	gprops->gang_quantum : adev->mes.default_gang_quantum;
432	gang->global_priority_level = gprops->global_priority_level;
433	gang->inprocess_gang_priority = gprops->inprocess_gang_priority;
434	list_add_tail(new: &gang->list, head: &process->gang_list);
435
436	amdgpu_mes_unlock(mes: &adev->mes);
437	return 0;
438
439	clean_up_ctx:
440	amdgpu_mes_unlock(mes: &adev->mes);
441	amdgpu_bo_free_kernel(bo: &gang->gang_ctx_bo,
442	gpu_addr: &gang->gang_ctx_gpu_addr,
443	cpu_addr: &gang->gang_ctx_cpu_ptr);
444	clean_up_mem:
445	kfree(objp: gang);
446	return r;
447	}
448
449	int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id)
450	{
451	struct amdgpu_mes_gang *gang;
452
453	/*
454	* Avoid taking any other locks under MES lock to avoid circular
455	* lock dependencies.
456	*/
457	amdgpu_mes_lock(mes: &adev->mes);
458
459	gang = idr_find(&adev->mes.gang_id_idr, id: gang_id);
460	if (!gang) {
461	DRM_ERROR("gang id %d doesn't exist\n", gang_id);
462	amdgpu_mes_unlock(mes: &adev->mes);
463	return -EINVAL;
464	}
465
466	if (!list_empty(head: &gang->queue_list)) {
467	DRM_ERROR("queue list is not empty\n");
468	amdgpu_mes_unlock(mes: &adev->mes);
469	return -EBUSY;
470	}
471
472	idr_remove(&adev->mes.gang_id_idr, id: gang->gang_id);
473	list_del(entry: &gang->list);
474	amdgpu_mes_unlock(mes: &adev->mes);
475
476	amdgpu_bo_free_kernel(bo: &gang->gang_ctx_bo,
477	gpu_addr: &gang->gang_ctx_gpu_addr,
478	cpu_addr: &gang->gang_ctx_cpu_ptr);
479
480	kfree(objp: gang);
481
482	return 0;
483	}
484
485	int amdgpu_mes_suspend(struct amdgpu_device *adev)
486	{
487	struct idr *idp;
488	struct amdgpu_mes_process *process;
489	struct amdgpu_mes_gang *gang;
490	struct mes_suspend_gang_input input;
491	int r, pasid;
492
493	/*
494	* Avoid taking any other locks under MES lock to avoid circular
495	* lock dependencies.
496	*/
497	amdgpu_mes_lock(mes: &adev->mes);
498
499	idp = &adev->mes.pasid_idr;
500
501	idr_for_each_entry(idp, process, pasid) {
502	list_for_each_entry(gang, &process->gang_list, list) {
503	r = adev->mes.funcs->suspend_gang(&adev->mes, &input);
504	if (r)
505	DRM_ERROR("failed to suspend pasid %d gangid %d",
506	pasid, gang->gang_id);
507	}
508	}
509
510	amdgpu_mes_unlock(mes: &adev->mes);
511	return 0;
512	}
513
514	int amdgpu_mes_resume(struct amdgpu_device *adev)
515	{
516	struct idr *idp;
517	struct amdgpu_mes_process *process;
518	struct amdgpu_mes_gang *gang;
519	struct mes_resume_gang_input input;
520	int r, pasid;
521
522	/*
523	* Avoid taking any other locks under MES lock to avoid circular
524	* lock dependencies.
525	*/
526	amdgpu_mes_lock(mes: &adev->mes);
527
528	idp = &adev->mes.pasid_idr;
529
530	idr_for_each_entry(idp, process, pasid) {
531	list_for_each_entry(gang, &process->gang_list, list) {
532	r = adev->mes.funcs->resume_gang(&adev->mes, &input);
533	if (r)
534	DRM_ERROR("failed to resume pasid %d gangid %d",
535	pasid, gang->gang_id);
536	}
537	}
538
539	amdgpu_mes_unlock(mes: &adev->mes);
540	return 0;
541	}
542
543	static int amdgpu_mes_queue_alloc_mqd(struct amdgpu_device *adev,
544	struct amdgpu_mes_queue *q,
545	struct amdgpu_mes_queue_properties *p)
546	{
547	struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
548	u32 mqd_size = mqd_mgr->mqd_size;
549	int r;
550
551	r = amdgpu_bo_create_kernel(adev, size: mqd_size, PAGE_SIZE,
552	AMDGPU_GEM_DOMAIN_GTT,
553	bo_ptr: &q->mqd_obj,
554	gpu_addr: &q->mqd_gpu_addr, cpu_addr: &q->mqd_cpu_ptr);
555	if (r) {
556	dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r);
557	return r;
558	}
559	memset(q->mqd_cpu_ptr, 0, mqd_size);
560
561	r = amdgpu_bo_reserve(bo: q->mqd_obj, no_intr: false);
562	if (unlikely(r != 0))
563	goto clean_up;
564
565	return 0;
566
567	clean_up:
568	amdgpu_bo_free_kernel(bo: &q->mqd_obj,
569	gpu_addr: &q->mqd_gpu_addr,
570	cpu_addr: &q->mqd_cpu_ptr);
571	return r;
572	}
573
574	static void amdgpu_mes_queue_init_mqd(struct amdgpu_device *adev,
575	struct amdgpu_mes_queue *q,
576	struct amdgpu_mes_queue_properties *p)
577	{
578	struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
579	struct amdgpu_mqd_prop mqd_prop = {0};
580
581	mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr;
582	mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr;
583	mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr;
584	mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr;
585	mqd_prop.queue_size = p->queue_size;
586	mqd_prop.use_doorbell = true;
587	mqd_prop.doorbell_index = p->doorbell_off;
588	mqd_prop.eop_gpu_addr = p->eop_gpu_addr;
589	mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority;
590	mqd_prop.hqd_queue_priority = p->hqd_queue_priority;
591	mqd_prop.hqd_active = false;
592
593	if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
594	p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
595	mutex_lock(&adev->srbm_mutex);
596	amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0);
597	}
598
599	mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop);
600
601	if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
602	p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
603	amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
604	mutex_unlock(lock: &adev->srbm_mutex);
605	}
606
607	amdgpu_bo_unreserve(bo: q->mqd_obj);
608	}
609
610	int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
611	struct amdgpu_mes_queue_properties *qprops,
612	int *queue_id)
613	{
614	struct amdgpu_mes_queue *queue;
615	struct amdgpu_mes_gang *gang;
616	struct mes_add_queue_input queue_input;
617	unsigned long flags;
618	int r;
619
620	memset(&queue_input, 0, sizeof(struct mes_add_queue_input));
621
622	/* allocate the mes queue buffer */
623	queue = kzalloc(size: sizeof(struct amdgpu_mes_queue), GFP_KERNEL);
624	if (!queue) {
625	DRM_ERROR("Failed to allocate memory for queue\n");
626	return -ENOMEM;
627	}
628
629	/* Allocate the queue mqd */
630	r = amdgpu_mes_queue_alloc_mqd(adev, q: queue, p: qprops);
631	if (r)
632	goto clean_up_memory;
633
634	/*
635	* Avoid taking any other locks under MES lock to avoid circular
636	* lock dependencies.
637	*/
638	amdgpu_mes_lock(mes: &adev->mes);
639
640	gang = idr_find(&adev->mes.gang_id_idr, id: gang_id);
641	if (!gang) {
642	DRM_ERROR("gang id %d doesn't exist\n", gang_id);
643	r = -EINVAL;
644	goto clean_up_mqd;
645	}
646
647	/* add the mes gang to idr list */
648	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
649	r = idr_alloc(&adev->mes.queue_id_idr, ptr: queue, start: 1, end: 0,
650	GFP_ATOMIC);
651	if (r < 0) {
652	spin_unlock_irqrestore(lock: &adev->mes.queue_id_lock, flags);
653	goto clean_up_mqd;
654	}
655	spin_unlock_irqrestore(lock: &adev->mes.queue_id_lock, flags);
656	*queue_id = queue->queue_id = r;
657
658	/* allocate a doorbell index for the queue */
659	r = amdgpu_mes_kernel_doorbell_get(adev, process: gang->process,
660	ip_type: qprops->queue_type,
661	doorbell_index: &qprops->doorbell_off);
662	if (r)
663	goto clean_up_queue_id;
664
665	/* initialize the queue mqd */
666	amdgpu_mes_queue_init_mqd(adev, q: queue, p: qprops);
667
668	/* add hw queue to mes */
669	queue_input.process_id = gang->process->pasid;
670
671	queue_input.page_table_base_addr =
672	adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr -
673	adev->gmc.vram_start;
674
675	queue_input.process_va_start = 0;
676	queue_input.process_va_end =
677	(adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
678	queue_input.process_quantum = gang->process->process_quantum;
679	queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr;
680	queue_input.gang_quantum = gang->gang_quantum;
681	queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
682	queue_input.inprocess_gang_priority = gang->inprocess_gang_priority;
683	queue_input.gang_global_priority_level = gang->global_priority_level;
684	queue_input.doorbell_offset = qprops->doorbell_off;
685	queue_input.mqd_addr = queue->mqd_gpu_addr;
686	queue_input.wptr_addr = qprops->wptr_gpu_addr;
687	queue_input.wptr_mc_addr = qprops->wptr_mc_addr;
688	queue_input.queue_type = qprops->queue_type;
689	queue_input.paging = qprops->paging;
690	queue_input.is_kfd_process = 0;
691
692	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
693	if (r) {
694	DRM_ERROR("failed to add hardware queue to MES, doorbell=0x%llx\n",
695	qprops->doorbell_off);
696	goto clean_up_doorbell;
697	}
698
699	DRM_DEBUG("MES hw queue was added, pasid=%d, gang id=%d, "
700	"queue type=%d, doorbell=0x%llx\n",
701	gang->process->pasid, gang_id, qprops->queue_type,
702	qprops->doorbell_off);
703
704	queue->ring = qprops->ring;
705	queue->doorbell_off = qprops->doorbell_off;
706	queue->wptr_gpu_addr = qprops->wptr_gpu_addr;
707	queue->queue_type = qprops->queue_type;
708	queue->paging = qprops->paging;
709	queue->gang = gang;
710	queue->ring->mqd_ptr = queue->mqd_cpu_ptr;
711	list_add_tail(new: &queue->list, head: &gang->queue_list);
712
713	amdgpu_mes_unlock(mes: &adev->mes);
714	return 0;
715
716	clean_up_doorbell:
717	amdgpu_mes_kernel_doorbell_free(adev, process: gang->process,
718	doorbell_index: qprops->doorbell_off);
719	clean_up_queue_id:
720	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
721	idr_remove(&adev->mes.queue_id_idr, id: queue->queue_id);
722	spin_unlock_irqrestore(lock: &adev->mes.queue_id_lock, flags);
723	clean_up_mqd:
724	amdgpu_mes_unlock(mes: &adev->mes);
725	amdgpu_mes_queue_free_mqd(q: queue);
726	clean_up_memory:
727	kfree(objp: queue);
728	return r;
729	}
730
731	int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id)
732	{
733	unsigned long flags;
734	struct amdgpu_mes_queue *queue;
735	struct amdgpu_mes_gang *gang;
736	struct mes_remove_queue_input queue_input;
737	int r;
738
739	/*
740	* Avoid taking any other locks under MES lock to avoid circular
741	* lock dependencies.
742	*/
743	amdgpu_mes_lock(mes: &adev->mes);
744
745	/* remove the mes gang from idr list */
746	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
747
748	queue = idr_find(&adev->mes.queue_id_idr, id: queue_id);
749	if (!queue) {
750	spin_unlock_irqrestore(lock: &adev->mes.queue_id_lock, flags);
751	amdgpu_mes_unlock(mes: &adev->mes);
752	DRM_ERROR("queue id %d doesn't exist\n", queue_id);
753	return -EINVAL;
754	}
755
756	idr_remove(&adev->mes.queue_id_idr, id: queue_id);
757	spin_unlock_irqrestore(lock: &adev->mes.queue_id_lock, flags);
758
759	DRM_DEBUG("try to remove queue, doorbell off = 0x%llx\n",
760	queue->doorbell_off);
761
762	gang = queue->gang;
763	queue_input.doorbell_offset = queue->doorbell_off;
764	queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
765
766	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
767	if (r)
768	DRM_ERROR("failed to remove hardware queue, queue id = %d\n",
769	queue_id);
770
771	list_del(entry: &queue->list);
772	amdgpu_mes_kernel_doorbell_free(adev, process: gang->process,
773	doorbell_index: queue->doorbell_off);
774	amdgpu_mes_unlock(mes: &adev->mes);
775
776	amdgpu_mes_queue_free_mqd(q: queue);
777	kfree(objp: queue);
778	return 0;
779	}
780
781	int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
782	struct amdgpu_ring *ring,
783	enum amdgpu_unmap_queues_action action,
784	u64 gpu_addr, u64 seq)
785	{
786	struct mes_unmap_legacy_queue_input queue_input;
787	int r;
788
789	queue_input.action = action;
790	queue_input.queue_type = ring->funcs->type;
791	queue_input.doorbell_offset = ring->doorbell_index;
792	queue_input.pipe_id = ring->pipe;
793	queue_input.queue_id = ring->queue;
794	queue_input.trail_fence_addr = gpu_addr;
795	queue_input.trail_fence_data = seq;
796
797	r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);
798	if (r)
799	DRM_ERROR("failed to unmap legacy queue\n");
800
801	return r;
802	}
803
804	uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
805	{
806	struct mes_misc_op_input op_input;
807	int r, val = 0;
808
809	op_input.op = MES_MISC_OP_READ_REG;
810	op_input.read_reg.reg_offset = reg;
811	op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr;
812
813	if (!adev->mes.funcs->misc_op) {
814	DRM_ERROR("mes rreg is not supported!\n");
815	goto error;
816	}
817
818	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
819	if (r)
820	DRM_ERROR("failed to read reg (0x%x)\n", reg);
821	else
822	val = *(adev->mes.read_val_ptr);
823
824	error:
825	return val;
826	}
827
828	int amdgpu_mes_wreg(struct amdgpu_device *adev,
829	uint32_t reg, uint32_t val)
830	{
831	struct mes_misc_op_input op_input;
832	int r;
833
834	op_input.op = MES_MISC_OP_WRITE_REG;
835	op_input.write_reg.reg_offset = reg;
836	op_input.write_reg.reg_value = val;
837
838	if (!adev->mes.funcs->misc_op) {
839	DRM_ERROR("mes wreg is not supported!\n");
840	r = -EINVAL;
841	goto error;
842	}
843
844	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
845	if (r)
846	DRM_ERROR("failed to write reg (0x%x)\n", reg);
847
848	error:
849	return r;
850	}
851
852	int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
853	uint32_t reg0, uint32_t reg1,
854	uint32_t ref, uint32_t mask)
855	{
856	struct mes_misc_op_input op_input;
857	int r;
858
859	op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT;
860	op_input.wrm_reg.reg0 = reg0;
861	op_input.wrm_reg.reg1 = reg1;
862	op_input.wrm_reg.ref = ref;
863	op_input.wrm_reg.mask = mask;
864
865	if (!adev->mes.funcs->misc_op) {
866	DRM_ERROR("mes reg_write_reg_wait is not supported!\n");
867	r = -EINVAL;
868	goto error;
869	}
870
871	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
872	if (r)
873	DRM_ERROR("failed to reg_write_reg_wait\n");
874
875	error:
876	return r;
877	}
878
879	int amdgpu_mes_reg_wait(struct amdgpu_device *adev, uint32_t reg,
880	uint32_t val, uint32_t mask)
881	{
882	struct mes_misc_op_input op_input;
883	int r;
884
885	op_input.op = MES_MISC_OP_WRM_REG_WAIT;
886	op_input.wrm_reg.reg0 = reg;
887	op_input.wrm_reg.ref = val;
888	op_input.wrm_reg.mask = mask;
889
890	if (!adev->mes.funcs->misc_op) {
891	DRM_ERROR("mes reg wait is not supported!\n");
892	r = -EINVAL;
893	goto error;
894	}
895
896	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
897	if (r)
898	DRM_ERROR("failed to reg_write_reg_wait\n");
899
900	error:
901	return r;
902	}
903
904	int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
905	uint64_t process_context_addr,
906	uint32_t spi_gdbg_per_vmid_cntl,
907	const uint32_t *tcp_watch_cntl,
908	uint32_t flags,
909	bool trap_en)
910	{
911	struct mes_misc_op_input op_input = {0};
912	int r;
913
914	if (!adev->mes.funcs->misc_op) {
915	DRM_ERROR("mes set shader debugger is not supported!\n");
916	return -EINVAL;
917	}
918
919	op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
920	op_input.set_shader_debugger.process_context_addr = process_context_addr;
921	op_input.set_shader_debugger.flags.u32all = flags;
922
923	/* use amdgpu mes_flush_shader_debugger instead */
924	if (op_input.set_shader_debugger.flags.process_ctx_flush)
925	return -EINVAL;
926
927	op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl;
928	memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl,
929	sizeof(op_input.set_shader_debugger.tcp_watch_cntl));
930
931	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
932	AMDGPU_MES_API_VERSION_SHIFT) >= 14)
933	op_input.set_shader_debugger.trap_en = trap_en;
934
935	amdgpu_mes_lock(mes: &adev->mes);
936
937	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
938	if (r)
939	DRM_ERROR("failed to set_shader_debugger\n");
940
941	amdgpu_mes_unlock(mes: &adev->mes);
942
943	return r;
944	}
945
946	int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
947	uint64_t process_context_addr)
948	{
949	struct mes_misc_op_input op_input = {0};
950	int r;
951
952	if (!adev->mes.funcs->misc_op) {
953	DRM_ERROR("mes flush shader debugger is not supported!\n");
954	return -EINVAL;
955	}
956
957	op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
958	op_input.set_shader_debugger.process_context_addr = process_context_addr;
959	op_input.set_shader_debugger.flags.process_ctx_flush = true;
960
961	amdgpu_mes_lock(mes: &adev->mes);
962
963	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
964	if (r)
965	DRM_ERROR("failed to set_shader_debugger\n");
966
967	amdgpu_mes_unlock(mes: &adev->mes);
968
969	return r;
970	}
971
972	static void
973	amdgpu_mes_ring_to_queue_props(struct amdgpu_device *adev,
974	struct amdgpu_ring *ring,
975	struct amdgpu_mes_queue_properties *props)
976	{
977	props->queue_type = ring->funcs->type;
978	props->hqd_base_gpu_addr = ring->gpu_addr;
979	props->rptr_gpu_addr = ring->rptr_gpu_addr;
980	props->wptr_gpu_addr = ring->wptr_gpu_addr;
981	props->wptr_mc_addr =
982	ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs;
983	props->queue_size = ring->ring_size;
984	props->eop_gpu_addr = ring->eop_gpu_addr;
985	props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
986	props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
987	props->paging = false;
988	props->ring = ring;
989	}
990
991	#define DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(_eng) \
992	do { \
993	if (id_offs < AMDGPU_MES_CTX_MAX_OFFS) \
994	return offsetof(struct amdgpu_mes_ctx_meta_data, \
995	_eng[ring->idx].slots[id_offs]); \
996	else if (id_offs == AMDGPU_MES_CTX_RING_OFFS) \
997	return offsetof(struct amdgpu_mes_ctx_meta_data, \
998	_eng[ring->idx].ring); \
999	else if (id_offs == AMDGPU_MES_CTX_IB_OFFS) \
1000	return offsetof(struct amdgpu_mes_ctx_meta_data, \
1001	_eng[ring->idx].ib); \
1002	else if (id_offs == AMDGPU_MES_CTX_PADDING_OFFS) \
1003	return offsetof(struct amdgpu_mes_ctx_meta_data, \
1004	_eng[ring->idx].padding); \
1005	} while(0)
1006
1007	int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs)
1008	{
1009	switch (ring->funcs->type) {
1010	case AMDGPU_RING_TYPE_GFX:
1011	DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx);
1012	break;
1013	case AMDGPU_RING_TYPE_COMPUTE:
1014	DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(compute);
1015	break;
1016	case AMDGPU_RING_TYPE_SDMA:
1017	DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(sdma);
1018	break;
1019	default:
1020	break;
1021	}
1022
1023	WARN_ON(1);
1024	return -EINVAL;
1025	}
1026
1027	int amdgpu_mes_add_ring(struct amdgpu_device *adev, int gang_id,
1028	int queue_type, int idx,
1029	struct amdgpu_mes_ctx_data *ctx_data,
1030	struct amdgpu_ring **out)
1031	{
1032	struct amdgpu_ring *ring;
1033	struct amdgpu_mes_gang *gang;
1034	struct amdgpu_mes_queue_properties qprops = {0};
1035	int r, queue_id, pasid;
1036
1037	/*
1038	* Avoid taking any other locks under MES lock to avoid circular
1039	* lock dependencies.
1040	*/
1041	amdgpu_mes_lock(mes: &adev->mes);
1042	gang = idr_find(&adev->mes.gang_id_idr, id: gang_id);
1043	if (!gang) {
1044	DRM_ERROR("gang id %d doesn't exist\n", gang_id);
1045	amdgpu_mes_unlock(mes: &adev->mes);
1046	return -EINVAL;
1047	}
1048	pasid = gang->process->pasid;
1049
1050	ring = kzalloc(size: sizeof(struct amdgpu_ring), GFP_KERNEL);
1051	if (!ring) {
1052	amdgpu_mes_unlock(mes: &adev->mes);
1053	return -ENOMEM;
1054	}
1055
1056	ring->ring_obj = NULL;
1057	ring->use_doorbell = true;
1058	ring->is_mes_queue = true;
1059	ring->mes_ctx = ctx_data;
1060	ring->idx = idx;
1061	ring->no_scheduler = true;
1062
1063	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1064	int offset = offsetof(struct amdgpu_mes_ctx_meta_data,
1065	compute[ring->idx].mec_hpd);
1066	ring->eop_gpu_addr =
1067	amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
1068	}
1069
1070	switch (queue_type) {
1071	case AMDGPU_RING_TYPE_GFX:
1072	ring->funcs = adev->gfx.gfx_ring[0].funcs;
1073	ring->me = adev->gfx.gfx_ring[0].me;
1074	ring->pipe = adev->gfx.gfx_ring[0].pipe;
1075	break;
1076	case AMDGPU_RING_TYPE_COMPUTE:
1077	ring->funcs = adev->gfx.compute_ring[0].funcs;
1078	ring->me = adev->gfx.compute_ring[0].me;
1079	ring->pipe = adev->gfx.compute_ring[0].pipe;
1080	break;
1081	case AMDGPU_RING_TYPE_SDMA:
1082	ring->funcs = adev->sdma.instance[0].ring.funcs;
1083	break;
1084	default:
1085	BUG();
1086	}
1087
1088	r = amdgpu_ring_init(adev, ring, max_dw: 1024, NULL, irq_type: 0,
1089	hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
1090	if (r)
1091	goto clean_up_memory;
1092
1093	amdgpu_mes_ring_to_queue_props(adev, ring, props: &qprops);
1094
1095	dma_fence_wait(fence: gang->process->vm->last_update, intr: false);
1096	dma_fence_wait(fence: ctx_data->meta_data_va->last_pt_update, intr: false);
1097	amdgpu_mes_unlock(mes: &adev->mes);
1098
1099	r = amdgpu_mes_add_hw_queue(adev, gang_id, qprops: &qprops, queue_id: &queue_id);
1100	if (r)
1101	goto clean_up_ring;
1102
1103	ring->hw_queue_id = queue_id;
1104	ring->doorbell_index = qprops.doorbell_off;
1105
1106	if (queue_type == AMDGPU_RING_TYPE_GFX)
1107	sprintf(buf: ring->name, fmt: "gfx_%d.%d.%d", pasid, gang_id, queue_id);
1108	else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
1109	sprintf(buf: ring->name, fmt: "compute_%d.%d.%d", pasid, gang_id,
1110	queue_id);
1111	else if (queue_type == AMDGPU_RING_TYPE_SDMA)
1112	sprintf(buf: ring->name, fmt: "sdma_%d.%d.%d", pasid, gang_id,
1113	queue_id);
1114	else
1115	BUG();
1116
1117	*out = ring;
1118	return 0;
1119
1120	clean_up_ring:
1121	amdgpu_ring_fini(ring);
1122	clean_up_memory:
1123	kfree(objp: ring);
1124	amdgpu_mes_unlock(mes: &adev->mes);
1125	return r;
1126	}
1127
1128	void amdgpu_mes_remove_ring(struct amdgpu_device *adev,
1129	struct amdgpu_ring *ring)
1130	{
1131	if (!ring)
1132	return;
1133
1134	amdgpu_mes_remove_hw_queue(adev, queue_id: ring->hw_queue_id);
1135	amdgpu_ring_fini(ring);
1136	kfree(objp: ring);
1137	}
1138
1139	uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev,
1140	enum amdgpu_mes_priority_level prio)
1141	{
1142	return adev->mes.aggregated_doorbells[prio];
1143	}
1144
1145	int amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device *adev,
1146	struct amdgpu_mes_ctx_data *ctx_data)
1147	{
1148	int r;
1149
1150	r = amdgpu_bo_create_kernel(adev,
1151	size: sizeof(struct amdgpu_mes_ctx_meta_data),
1152	PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1153	bo_ptr: &ctx_data->meta_data_obj,
1154	gpu_addr: &ctx_data->meta_data_mc_addr,
1155	cpu_addr: &ctx_data->meta_data_ptr);
1156	if (r) {
1157	dev_warn(adev->dev, "(%d) create CTX bo failed\n", r);
1158	return r;
1159	}
1160
1161	if (!ctx_data->meta_data_obj)
1162	return -ENOMEM;
1163
1164	memset(ctx_data->meta_data_ptr, 0,
1165	sizeof(struct amdgpu_mes_ctx_meta_data));
1166
1167	return 0;
1168	}
1169
1170	void amdgpu_mes_ctx_free_meta_data(struct amdgpu_mes_ctx_data *ctx_data)
1171	{
1172	if (ctx_data->meta_data_obj)
1173	amdgpu_bo_free_kernel(bo: &ctx_data->meta_data_obj,
1174	gpu_addr: &ctx_data->meta_data_mc_addr,
1175	cpu_addr: &ctx_data->meta_data_ptr);
1176	}
1177
1178	int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev,
1179	struct amdgpu_vm *vm,
1180	struct amdgpu_mes_ctx_data *ctx_data)
1181	{
1182	struct amdgpu_bo_va *bo_va;
1183	struct amdgpu_sync sync;
1184	struct drm_exec exec;
1185	int r;
1186
1187	amdgpu_sync_create(sync: &sync);
1188
1189	drm_exec_init(exec: &exec, flags: 0, nr: 0);
1190	drm_exec_until_all_locked(&exec) {
1191	r = drm_exec_lock_obj(exec: &exec,
1192	obj: &ctx_data->meta_data_obj->tbo.base);
1193	drm_exec_retry_on_contention(&exec);
1194	if (unlikely(r))
1195	goto error_fini_exec;
1196
1197	r = amdgpu_vm_lock_pd(vm, exec: &exec, num_fences: 0);
1198	drm_exec_retry_on_contention(&exec);
1199	if (unlikely(r))
1200	goto error_fini_exec;
1201	}
1202
1203	bo_va = amdgpu_vm_bo_add(adev, vm, bo: ctx_data->meta_data_obj);
1204	if (!bo_va) {
1205	DRM_ERROR("failed to create bo_va for meta data BO\n");
1206	r = -ENOMEM;
1207	goto error_fini_exec;
1208	}
1209
1210	r = amdgpu_vm_bo_map(adev, bo_va, addr: ctx_data->meta_data_gpu_addr, offset: 0,
1211	size: sizeof(struct amdgpu_mes_ctx_meta_data),
1212	AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
1213	AMDGPU_PTE_EXECUTABLE);
1214
1215	if (r) {
1216	DRM_ERROR("failed to do bo_map on meta data, err=%d\n", r);
1217	goto error_del_bo_va;
1218	}
1219
1220	r = amdgpu_vm_bo_update(adev, bo_va, clear: false);
1221	if (r) {
1222	DRM_ERROR("failed to do vm_bo_update on meta data\n");
1223	goto error_del_bo_va;
1224	}
1225	amdgpu_sync_fence(sync: &sync, f: bo_va->last_pt_update);
1226
1227	r = amdgpu_vm_update_pdes(adev, vm, immediate: false);
1228	if (r) {
1229	DRM_ERROR("failed to update pdes on meta data\n");
1230	goto error_del_bo_va;
1231	}
1232	amdgpu_sync_fence(sync: &sync, f: vm->last_update);
1233
1234	amdgpu_sync_wait(sync: &sync, intr: false);
1235	drm_exec_fini(exec: &exec);
1236
1237	amdgpu_sync_free(sync: &sync);
1238	ctx_data->meta_data_va = bo_va;
1239	return 0;
1240
1241	error_del_bo_va:
1242	amdgpu_vm_bo_del(adev, bo_va);
1243
1244	error_fini_exec:
1245	drm_exec_fini(exec: &exec);
1246	amdgpu_sync_free(sync: &sync);
1247	return r;
1248	}
1249
1250	int amdgpu_mes_ctx_unmap_meta_data(struct amdgpu_device *adev,
1251	struct amdgpu_mes_ctx_data *ctx_data)
1252	{
1253	struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va;
1254	struct amdgpu_bo *bo = ctx_data->meta_data_obj;
1255	struct amdgpu_vm *vm = bo_va->base.vm;
1256	struct dma_fence *fence;
1257	struct drm_exec exec;
1258	long r;
1259
1260	drm_exec_init(exec: &exec, flags: 0, nr: 0);
1261	drm_exec_until_all_locked(&exec) {
1262	r = drm_exec_lock_obj(exec: &exec,
1263	obj: &ctx_data->meta_data_obj->tbo.base);
1264	drm_exec_retry_on_contention(&exec);
1265	if (unlikely(r))
1266	goto out_unlock;
1267
1268	r = amdgpu_vm_lock_pd(vm, exec: &exec, num_fences: 0);
1269	drm_exec_retry_on_contention(&exec);
1270	if (unlikely(r))
1271	goto out_unlock;
1272	}
1273
1274	amdgpu_vm_bo_del(adev, bo_va);
1275	if (!amdgpu_vm_ready(vm))
1276	goto out_unlock;
1277
1278	r = dma_resv_get_singleton(obj: bo->tbo.base.resv, usage: DMA_RESV_USAGE_BOOKKEEP,
1279	fence: &fence);
1280	if (r)
1281	goto out_unlock;
1282	if (fence) {
1283	amdgpu_bo_fence(bo, fence, shared: true);
1284	fence = NULL;
1285	}
1286
1287	r = amdgpu_vm_clear_freed(adev, vm, fence: &fence);
1288	if (r || !fence)
1289	goto out_unlock;
1290
1291	dma_fence_wait(fence, intr: false);
1292	amdgpu_bo_fence(bo, fence, shared: true);
1293	dma_fence_put(fence);
1294
1295	out_unlock:
1296	if (unlikely(r < 0))
1297	dev_err(adev->dev, "failed to clear page tables (%ld)\n", r);
1298	drm_exec_fini(exec: &exec);
1299
1300	return r;
1301	}
1302
1303	static int amdgpu_mes_test_create_gang_and_queues(struct amdgpu_device *adev,
1304	int pasid, int *gang_id,
1305	int queue_type, int num_queue,
1306	struct amdgpu_ring **added_rings,
1307	struct amdgpu_mes_ctx_data *ctx_data)
1308	{
1309	struct amdgpu_ring *ring;
1310	struct amdgpu_mes_gang_properties gprops = {0};
1311	int r, j;
1312
1313	/* create a gang for the process */
1314	gprops.priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1315	gprops.gang_quantum = adev->mes.default_gang_quantum;
1316	gprops.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1317	gprops.priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1318	gprops.global_priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1319
1320	r = amdgpu_mes_add_gang(adev, pasid, gprops: &gprops, gang_id);
1321	if (r) {
1322	DRM_ERROR("failed to add gang\n");
1323	return r;
1324	}
1325
1326	/* create queues for the gang */
1327	for (j = 0; j < num_queue; j++) {
1328	r = amdgpu_mes_add_ring(adev, gang_id: *gang_id, queue_type, idx: j,
1329	ctx_data, out: &ring);
1330	if (r) {
1331	DRM_ERROR("failed to add ring\n");
1332	break;
1333	}
1334
1335	DRM_INFO("ring %s was added\n", ring->name);
1336	added_rings[j] = ring;
1337	}
1338
1339	return 0;
1340	}
1341
1342	static int amdgpu_mes_test_queues(struct amdgpu_ring **added_rings)
1343	{
1344	struct amdgpu_ring *ring;
1345	int i, r;
1346
1347	for (i = 0; i < AMDGPU_MES_CTX_MAX_RINGS; i++) {
1348	ring = added_rings[i];
1349	if (!ring)
1350	continue;
1351
1352	r = amdgpu_ring_test_helper(ring);
1353	if (r)
1354	return r;
1355
1356	r = amdgpu_ring_test_ib(ring, 1000 * 10);
1357	if (r) {
1358	DRM_DEV_ERROR(ring->adev->dev,
1359	"ring %s ib test failed (%d)\n",
1360	ring->name, r);
1361	return r;
1362	} else
1363	DRM_INFO("ring %s ib test pass\n", ring->name);
1364	}
1365
1366	return 0;
1367	}
1368
1369	int amdgpu_mes_self_test(struct amdgpu_device *adev)
1370	{
1371	struct amdgpu_vm *vm = NULL;
1372	struct amdgpu_mes_ctx_data ctx_data = {0};
1373	struct amdgpu_ring *added_rings[AMDGPU_MES_CTX_MAX_RINGS] = { NULL };
1374	int gang_ids[3] = {0};
1375	int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX, 1 },
1376	{ AMDGPU_RING_TYPE_COMPUTE, 1 },
1377	{ AMDGPU_RING_TYPE_SDMA, 1} };
1378	int i, r, pasid, k = 0;
1379
1380	pasid = amdgpu_pasid_alloc(bits: 16);
1381	if (pasid < 0) {
1382	dev_warn(adev->dev, "No more PASIDs available!");
1383	pasid = 0;
1384	}
1385
1386	vm = kzalloc(size: sizeof(*vm), GFP_KERNEL);
1387	if (!vm) {
1388	r = -ENOMEM;
1389	goto error_pasid;
1390	}
1391
1392	r = amdgpu_vm_init(adev, vm, xcp_id: -1);
1393	if (r) {
1394	DRM_ERROR("failed to initialize vm\n");
1395	goto error_pasid;
1396	}
1397
1398	r = amdgpu_mes_ctx_alloc_meta_data(adev, ctx_data: &ctx_data);
1399	if (r) {
1400	DRM_ERROR("failed to alloc ctx meta data\n");
1401	goto error_fini;
1402	}
1403
1404	ctx_data.meta_data_gpu_addr = AMDGPU_VA_RESERVED_BOTTOM;
1405	r = amdgpu_mes_ctx_map_meta_data(adev, vm, ctx_data: &ctx_data);
1406	if (r) {
1407	DRM_ERROR("failed to map ctx meta data\n");
1408	goto error_vm;
1409	}
1410
1411	r = amdgpu_mes_create_process(adev, pasid, vm);
1412	if (r) {
1413	DRM_ERROR("failed to create MES process\n");
1414	goto error_vm;
1415	}
1416
1417	for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
1418	/* On GFX v10.3, fw hasn't supported to map sdma queue. */
1419	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >=
1420	IP_VERSION(10, 3, 0) &&
1421	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) <
1422	IP_VERSION(11, 0, 0) &&
1423	queue_types[i][0] == AMDGPU_RING_TYPE_SDMA)
1424	continue;
1425
1426	r = amdgpu_mes_test_create_gang_and_queues(adev, pasid,
1427	gang_id: &gang_ids[i],
1428	queue_type: queue_types[i][0],
1429	num_queue: queue_types[i][1],
1430	added_rings: &added_rings[k],
1431	ctx_data: &ctx_data);
1432	if (r)
1433	goto error_queues;
1434
1435	k += queue_types[i][1];
1436	}
1437
1438	/* start ring test and ib test for MES queues */
1439	amdgpu_mes_test_queues(added_rings);
1440
1441	error_queues:
1442	/* remove all queues */
1443	for (i = 0; i < ARRAY_SIZE(added_rings); i++) {
1444	if (!added_rings[i])
1445	continue;
1446	amdgpu_mes_remove_ring(adev, ring: added_rings[i]);
1447	}
1448
1449	for (i = 0; i < ARRAY_SIZE(gang_ids); i++) {
1450	if (!gang_ids[i])
1451	continue;
1452	amdgpu_mes_remove_gang(adev, gang_id: gang_ids[i]);
1453	}
1454
1455	amdgpu_mes_destroy_process(adev, pasid);
1456
1457	error_vm:
1458	amdgpu_mes_ctx_unmap_meta_data(adev, ctx_data: &ctx_data);
1459
1460	error_fini:
1461	amdgpu_vm_fini(adev, vm);
1462
1463	error_pasid:
1464	if (pasid)
1465	amdgpu_pasid_free(pasid);
1466
1467	amdgpu_mes_ctx_free_meta_data(ctx_data: &ctx_data);
1468	kfree(objp: vm);
1469	return 0;
1470	}
1471
1472	int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe)
1473	{
1474	const struct mes_firmware_header_v1_0 *mes_hdr;
1475	struct amdgpu_firmware_info *info;
1476	char ucode_prefix[30];
1477	char fw_name[40];
1478	bool need_retry = false;
1479	int r;
1480
1481	amdgpu_ucode_ip_version_decode(adev, block_type: GC_HWIP, ucode_prefix,
1482	len: sizeof(ucode_prefix));
1483	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(11, 0, 0)) {
1484	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mes%s.bin",
1485	ucode_prefix,
1486	pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1");
1487	need_retry = true;
1488	} else {
1489	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mes%s.bin",
1490	ucode_prefix,
1491	pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1");
1492	}
1493
1494	r = amdgpu_ucode_request(adev, fw: &adev->mes.fw[pipe], fw_name);
1495	if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) {
1496	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mes.bin",
1497	ucode_prefix);
1498	DRM_INFO("try to fall back to %s\n", fw_name);
1499	r = amdgpu_ucode_request(adev, fw: &adev->mes.fw[pipe],
1500	fw_name);
1501	}
1502
1503	if (r)
1504	goto out;
1505
1506	mes_hdr = (const struct mes_firmware_header_v1_0 *)
1507	adev->mes.fw[pipe]->data;
1508	adev->mes.uc_start_addr[pipe] =
1509	le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
1510	((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
1511	adev->mes.data_start_addr[pipe] =
1512	le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
1513	((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
1514
1515	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1516	int ucode, ucode_data;
1517
1518	if (pipe == AMDGPU_MES_SCHED_PIPE) {
1519	ucode = AMDGPU_UCODE_ID_CP_MES;
1520	ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA;
1521	} else {
1522	ucode = AMDGPU_UCODE_ID_CP_MES1;
1523	ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA;
1524	}
1525
1526	info = &adev->firmware.ucode[ucode];
1527	info->ucode_id = ucode;
1528	info->fw = adev->mes.fw[pipe];
1529	adev->firmware.fw_size +=
1530	ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes),
1531	PAGE_SIZE);
1532
1533	info = &adev->firmware.ucode[ucode_data];
1534	info->ucode_id = ucode_data;
1535	info->fw = adev->mes.fw[pipe];
1536	adev->firmware.fw_size +=
1537	ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes),
1538	PAGE_SIZE);
1539	}
1540
1541	return 0;
1542	out:
1543	amdgpu_ucode_release(fw: &adev->mes.fw[pipe]);
1544	return r;
1545	}
1546
1547	#if defined(CONFIG_DEBUG_FS)
1548
1549	static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
1550	{
1551	struct amdgpu_device *adev = m->private;
1552	uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr);
1553
1554	seq_hex_dump(m, prefix_str: "", prefix_type: DUMP_PREFIX_OFFSET, rowsize: 32, groupsize: 4,
1555	buf: mem, AMDGPU_MES_LOG_BUFFER_SIZE, ascii: false);
1556
1557	return 0;
1558	}
1559
1560	DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_mes_event_log);
1561
1562	#endif
1563
1564	void amdgpu_debugfs_mes_event_log_init(struct amdgpu_device *adev)
1565	{
1566
1567	#if defined(CONFIG_DEBUG_FS)
1568	struct drm_minor *minor = adev_to_drm(adev)->primary;
1569	struct dentry *root = minor->debugfs_root;
1570	if (adev->enable_mes && amdgpu_mes_log_enable)
1571	debugfs_create_file(name: "amdgpu_mes_event_log", mode: 0444, parent: root,
1572	data: adev, fops: &amdgpu_debugfs_mes_event_log_fops);
1573
1574	#endif
1575	}
1576