1	/*
2	* Copyright 2019 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/firmware.h>
25	#include "amdgpu.h"
26	#include "amdgpu_vcn.h"
27	#include "amdgpu_pm.h"
28	#include "amdgpu_cs.h"
29	#include "soc15.h"
30	#include "soc15d.h"
31	#include "vcn_v2_0.h"
32	#include "mmsch_v3_0.h"
33	#include "vcn_sw_ring.h"
34
35	#include "vcn/vcn_3_0_0_offset.h"
36	#include "vcn/vcn_3_0_0_sh_mask.h"
37	#include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
38
39	#include <drm/drm_drv.h>
40
41	#define VCN_VID_SOC_ADDRESS_2_0 0x1fa00
42	#define VCN1_VID_SOC_ADDRESS_3_0 0x48200
43
44	#define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
45	#define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
46	#define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
47	#define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
48	#define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
49	#define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
50	#define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
51
52	#define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
53	#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
54	#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
55	#define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
56
57	#define VCN_INSTANCES_SIENNA_CICHLID 2
58	#define DEC_SW_RING_ENABLED FALSE
59
60	#define RDECODE_MSG_CREATE 0x00000000
61	#define RDECODE_MESSAGE_CREATE 0x00000001
62
63	static int amdgpu_ih_clientid_vcns[] = {
64	SOC15_IH_CLIENTID_VCN,
65	SOC15_IH_CLIENTID_VCN1
66	};
67
68	static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
69	static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
70	static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
71	static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
72	static int vcn_v3_0_set_powergating_state(void *handle,
73	enum amd_powergating_state state);
74	static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
75	int inst_idx, struct dpg_pause_state *new_state);
76
77	static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
78	static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
79
80	/**
81	* vcn_v3_0_early_init - set function pointers and load microcode
82	*
83	* @handle: amdgpu_device pointer
84	*
85	* Set ring and irq function pointers
86	* Load microcode from filesystem
87	*/
88	static int vcn_v3_0_early_init(void *handle)
89	{
90	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
91
92	if (amdgpu_sriov_vf(adev)) {
93	adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
94	adev->vcn.harvest_config = 0;
95	adev->vcn.num_enc_rings = 1;
96
97	} else {
98	if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
99	AMDGPU_VCN_HARVEST_VCN1))
100	/* both instances are harvested, disable the block */
101	return -ENOENT;
102
103	if (amdgpu_ip_version(adev, ip: UVD_HWIP, inst: 0) ==
104	IP_VERSION(3, 0, 33))
105	adev->vcn.num_enc_rings = 0;
106	else
107	adev->vcn.num_enc_rings = 2;
108	}
109
110	vcn_v3_0_set_dec_ring_funcs(adev);
111	vcn_v3_0_set_enc_ring_funcs(adev);
112	vcn_v3_0_set_irq_funcs(adev);
113
114	return amdgpu_vcn_early_init(adev);
115	}
116
117	/**
118	* vcn_v3_0_sw_init - sw init for VCN block
119	*
120	* @handle: amdgpu_device pointer
121	*
122	* Load firmware and sw initialization
123	*/
124	static int vcn_v3_0_sw_init(void *handle)
125	{
126	struct amdgpu_ring *ring;
127	int i, j, r;
128	int vcn_doorbell_index = 0;
129	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
130
131	r = amdgpu_vcn_sw_init(adev);
132	if (r)
133	return r;
134
135	amdgpu_vcn_setup_ucode(adev);
136
137	r = amdgpu_vcn_resume(adev);
138	if (r)
139	return r;
140
141	/*
142	* Note: doorbell assignment is fixed for SRIOV multiple VCN engines
143	* Formula:
144	* vcn_db_base = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
145	* dec_ring_i = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
146	* enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
147	*/
148	if (amdgpu_sriov_vf(adev)) {
149	vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
150	/* get DWORD offset */
151	vcn_doorbell_index = vcn_doorbell_index << 1;
152	}
153
154	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
155	volatile struct amdgpu_fw_shared *fw_shared;
156
157	if (adev->vcn.harvest_config & (1 << i))
158	continue;
159
160	adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
161	adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
162	adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
163	adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
164	adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
165	adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
166
167	adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
168	adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
169	adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
170	adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
171	adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
172	adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
173	adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
174	adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
175	adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
176	adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
177
178	/* VCN DEC TRAP */
179	r = amdgpu_irq_add_id(adev, client_id: amdgpu_ih_clientid_vcns[i],
180	VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, source: &adev->vcn.inst[i].irq);
181	if (r)
182	return r;
183
184	atomic_set(v: &adev->vcn.inst[i].sched_score, i: 0);
185
186	ring = &adev->vcn.inst[i].ring_dec;
187	ring->use_doorbell = true;
188	if (amdgpu_sriov_vf(adev)) {
189	ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
190	} else {
191	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
192	}
193	ring->vm_hub = AMDGPU_MMHUB0(0);
194	sprintf(buf: ring->name, fmt: "vcn_dec_%d", i);
195	r = amdgpu_ring_init(adev, ring, max_dw: 512, irq_src: &adev->vcn.inst[i].irq, irq_type: 0,
196	hw_prio: AMDGPU_RING_PRIO_DEFAULT,
197	sched_score: &adev->vcn.inst[i].sched_score);
198	if (r)
199	return r;
200
201	for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
202	enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(ring: j);
203
204	/* VCN ENC TRAP */
205	r = amdgpu_irq_add_id(adev, client_id: amdgpu_ih_clientid_vcns[i],
206	src_id: j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, source: &adev->vcn.inst[i].irq);
207	if (r)
208	return r;
209
210	ring = &adev->vcn.inst[i].ring_enc[j];
211	ring->use_doorbell = true;
212	if (amdgpu_sriov_vf(adev)) {
213	ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
214	} else {
215	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
216	}
217	ring->vm_hub = AMDGPU_MMHUB0(0);
218	sprintf(buf: ring->name, fmt: "vcn_enc_%d.%d", i, j);
219	r = amdgpu_ring_init(adev, ring, max_dw: 512, irq_src: &adev->vcn.inst[i].irq, irq_type: 0,
220	hw_prio, sched_score: &adev->vcn.inst[i].sched_score);
221	if (r)
222	return r;
223	}
224
225	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
226	fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
227	cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |
228	cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);
229	fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
230	fw_shared->present_flag_0 |= AMDGPU_VCN_SMU_VERSION_INFO_FLAG;
231	if (amdgpu_ip_version(adev, ip: UVD_HWIP, inst: 0) == IP_VERSION(3, 1, 2))
232	fw_shared->smu_interface_info.smu_interface_type = 2;
233	else if (amdgpu_ip_version(adev, ip: UVD_HWIP, inst: 0) ==
234	IP_VERSION(3, 1, 1))
235	fw_shared->smu_interface_info.smu_interface_type = 1;
236
237	if (amdgpu_vcnfw_log)
238	amdgpu_vcn_fwlog_init(vcn: &adev->vcn.inst[i]);
239	}
240
241	if (amdgpu_sriov_vf(adev)) {
242	r = amdgpu_virt_alloc_mm_table(adev);
243	if (r)
244	return r;
245	}
246	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
247	adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
248
249	return 0;
250	}
251
252	/**
253	* vcn_v3_0_sw_fini - sw fini for VCN block
254	*
255	* @handle: amdgpu_device pointer
256	*
257	* VCN suspend and free up sw allocation
258	*/
259	static int vcn_v3_0_sw_fini(void *handle)
260	{
261	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
262	int i, r, idx;
263
264	if (drm_dev_enter(dev: adev_to_drm(adev), idx: &idx)) {
265	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
266	volatile struct amdgpu_fw_shared *fw_shared;
267
268	if (adev->vcn.harvest_config & (1 << i))
269	continue;
270	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
271	fw_shared->present_flag_0 = 0;
272	fw_shared->sw_ring.is_enabled = false;
273	}
274
275	drm_dev_exit(idx);
276	}
277
278	if (amdgpu_sriov_vf(adev))
279	amdgpu_virt_free_mm_table(adev);
280
281	r = amdgpu_vcn_suspend(adev);
282	if (r)
283	return r;
284
285	r = amdgpu_vcn_sw_fini(adev);
286
287	return r;
288	}
289
290	/**
291	* vcn_v3_0_hw_init - start and test VCN block
292	*
293	* @handle: amdgpu_device pointer
294	*
295	* Initialize the hardware, boot up the VCPU and do some testing
296	*/
297	static int vcn_v3_0_hw_init(void *handle)
298	{
299	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
300	struct amdgpu_ring *ring;
301	int i, j, r;
302
303	if (amdgpu_sriov_vf(adev)) {
304	r = vcn_v3_0_start_sriov(adev);
305	if (r)
306	goto done;
307
308	/* initialize VCN dec and enc ring buffers */
309	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
310	if (adev->vcn.harvest_config & (1 << i))
311	continue;
312
313	ring = &adev->vcn.inst[i].ring_dec;
314	if (amdgpu_vcn_is_disabled_vcn(adev, type: VCN_DECODE_RING, vcn_instance: i)) {
315	ring->sched.ready = false;
316	ring->no_scheduler = true;
317	dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
318	} else {
319	ring->wptr = 0;
320	ring->wptr_old = 0;
321	vcn_v3_0_dec_ring_set_wptr(ring);
322	ring->sched.ready = true;
323	}
324
325	for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
326	ring = &adev->vcn.inst[i].ring_enc[j];
327	if (amdgpu_vcn_is_disabled_vcn(adev, type: VCN_ENCODE_RING, vcn_instance: i)) {
328	ring->sched.ready = false;
329	ring->no_scheduler = true;
330	dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
331	} else {
332	ring->wptr = 0;
333	ring->wptr_old = 0;
334	vcn_v3_0_enc_ring_set_wptr(ring);
335	ring->sched.ready = true;
336	}
337	}
338	}
339	} else {
340	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
341	if (adev->vcn.harvest_config & (1 << i))
342	continue;
343
344	ring = &adev->vcn.inst[i].ring_dec;
345
346	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
347	ring->doorbell_index, i);
348
349	r = amdgpu_ring_test_helper(ring);
350	if (r)
351	goto done;
352
353	for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
354	ring = &adev->vcn.inst[i].ring_enc[j];
355	r = amdgpu_ring_test_helper(ring);
356	if (r)
357	goto done;
358	}
359	}
360	}
361
362	done:
363	if (!r)
364	DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
365	(adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
366
367	return r;
368	}
369
370	/**
371	* vcn_v3_0_hw_fini - stop the hardware block
372	*
373	* @handle: amdgpu_device pointer
374	*
375	* Stop the VCN block, mark ring as not ready any more
376	*/
377	static int vcn_v3_0_hw_fini(void *handle)
378	{
379	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
380	int i;
381
382	cancel_delayed_work_sync(dwork: &adev->vcn.idle_work);
383
384	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
385	if (adev->vcn.harvest_config & (1 << i))
386	continue;
387
388	if (!amdgpu_sriov_vf(adev)) {
389	if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
390	(adev->vcn.cur_state != AMD_PG_STATE_GATE &&
391	RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
392	vcn_v3_0_set_powergating_state(handle: adev, state: AMD_PG_STATE_GATE);
393	}
394	}
395	}
396
397	return 0;
398	}
399
400	/**
401	* vcn_v3_0_suspend - suspend VCN block
402	*
403	* @handle: amdgpu_device pointer
404	*
405	* HW fini and suspend VCN block
406	*/
407	static int vcn_v3_0_suspend(void *handle)
408	{
409	int r;
410	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
411
412	r = vcn_v3_0_hw_fini(handle: adev);
413	if (r)
414	return r;
415
416	r = amdgpu_vcn_suspend(adev);
417
418	return r;
419	}
420
421	/**
422	* vcn_v3_0_resume - resume VCN block
423	*
424	* @handle: amdgpu_device pointer
425	*
426	* Resume firmware and hw init VCN block
427	*/
428	static int vcn_v3_0_resume(void *handle)
429	{
430	int r;
431	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
432
433	r = amdgpu_vcn_resume(adev);
434	if (r)
435	return r;
436
437	r = vcn_v3_0_hw_init(handle: adev);
438
439	return r;
440	}
441
442	/**
443	* vcn_v3_0_mc_resume - memory controller programming
444	*
445	* @adev: amdgpu_device pointer
446	* @inst: instance number
447	*
448	* Let the VCN memory controller know it's offsets
449	*/
450	static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
451	{
452	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[inst]->size + 4);
453	uint32_t offset;
454
455	/* cache window 0: fw */
456	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
457	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
458	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
459	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
460	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
461	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
462	offset = 0;
463	} else {
464	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
465	lower_32_bits(adev->vcn.inst[inst].gpu_addr));
466	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
467	upper_32_bits(adev->vcn.inst[inst].gpu_addr));
468	offset = size;
469	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
470	AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
471	}
472	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
473
474	/* cache window 1: stack */
475	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
476	lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
477	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
478	upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
479	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
480	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
481
482	/* cache window 2: context */
483	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
484	lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
485	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
486	upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
487	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
488	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
489
490	/* non-cache window */
491	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
492	lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
493	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
494	upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
495	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
496	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
497	AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
498	}
499
500	static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
501	{
502	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[inst_idx]->size + 4);
503	uint32_t offset;
504
505	/* cache window 0: fw */
506	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
507	if (!indirect) {
508	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
509	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
510	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
511	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
512	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
513	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
514	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
515	VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
516	} else {
517	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
518	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
519	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
520	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
521	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
522	VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
523	}
524	offset = 0;
525	} else {
526	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
527	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
528	lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
529	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
530	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
531	upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
532	offset = size;
533	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
534	VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
535	AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
536	}
537
538	if (!indirect)
539	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
540	VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
541	else
542	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
543	VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
544
545	/* cache window 1: stack */
546	if (!indirect) {
547	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
548	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
549	lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
550	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
551	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
552	upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
553	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
554	VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
555	} else {
556	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
557	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
558	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
559	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
560	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
561	VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
562	}
563	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
564	VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
565
566	/* cache window 2: context */
567	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
568	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
569	lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
570	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
571	VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
572	upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
573	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
574	VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
575	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
576	VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
577
578	/* non-cache window */
579	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
580	VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
581	lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
582	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
583	VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
584	upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
585	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
586	VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
587	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
588	VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
589	AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
590
591	/* VCN global tiling registers */
592	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
593	UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
594	}
595
596	static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
597	{
598	uint32_t data = 0;
599
600	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
601	data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
602	| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
603	| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
604	| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
605	| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
606	| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
607	| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
608	| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
609	| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
610	| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
611	| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
612	| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
613	| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
614	| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
615
616	WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
617	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
618	UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
619	} else {
620	data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
621	| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
622	| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
623	| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
624	| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
625	| 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
626	| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
627	| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
628	| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
629	| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
630	| 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
631	| 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
632	| 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
633	| 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
634	WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
635	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0, 0x3F3FFFFF);
636	}
637
638	data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
639	data &= ~0x103;
640	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
641	data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
642	UVD_POWER_STATUS__UVD_PG_EN_MASK;
643
644	WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
645	}
646
647	static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
648	{
649	uint32_t data;
650
651	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
652	/* Before power off, this indicator has to be turned on */
653	data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
654	data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
655	data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
656	WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
657
658	data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
659	| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
660	| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
661	| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
662	| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
663	| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
664	| 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
665	| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
666	| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
667	| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
668	| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
669	| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
670	| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
671	| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
672	WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
673
674	data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
675	| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
676	| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
677	| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
678	| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
679	| 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
680	| 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
681	| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
682	| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
683	| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
684	| 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
685	| 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
686	| 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
687	| 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
688	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
689	}
690	}
691
692	/**
693	* vcn_v3_0_disable_clock_gating - disable VCN clock gating
694	*
695	* @adev: amdgpu_device pointer
696	* @inst: instance number
697	*
698	* Disable clock gating for VCN block
699	*/
700	static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
701	{
702	uint32_t data;
703
704	/* VCN disable CGC */
705	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
706	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
707	data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
708	else
709	data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
710	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
711	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
712	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
713
714	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
715	data &= ~(UVD_CGC_GATE__SYS_MASK
716	| UVD_CGC_GATE__UDEC_MASK
717	| UVD_CGC_GATE__MPEG2_MASK
718	| UVD_CGC_GATE__REGS_MASK
719	| UVD_CGC_GATE__RBC_MASK
720	| UVD_CGC_GATE__LMI_MC_MASK
721	| UVD_CGC_GATE__LMI_UMC_MASK
722	| UVD_CGC_GATE__IDCT_MASK
723	| UVD_CGC_GATE__MPRD_MASK
724	| UVD_CGC_GATE__MPC_MASK
725	| UVD_CGC_GATE__LBSI_MASK
726	| UVD_CGC_GATE__LRBBM_MASK
727	| UVD_CGC_GATE__UDEC_RE_MASK
728	| UVD_CGC_GATE__UDEC_CM_MASK
729	| UVD_CGC_GATE__UDEC_IT_MASK
730	| UVD_CGC_GATE__UDEC_DB_MASK
731	| UVD_CGC_GATE__UDEC_MP_MASK
732	| UVD_CGC_GATE__WCB_MASK
733	| UVD_CGC_GATE__VCPU_MASK
734	| UVD_CGC_GATE__MMSCH_MASK);
735
736	WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
737
738	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);
739
740	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
741	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
742	| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
743	| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
744	| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
745	| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
746	| UVD_CGC_CTRL__SYS_MODE_MASK
747	| UVD_CGC_CTRL__UDEC_MODE_MASK
748	| UVD_CGC_CTRL__MPEG2_MODE_MASK
749	| UVD_CGC_CTRL__REGS_MODE_MASK
750	| UVD_CGC_CTRL__RBC_MODE_MASK
751	| UVD_CGC_CTRL__LMI_MC_MODE_MASK
752	| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
753	| UVD_CGC_CTRL__IDCT_MODE_MASK
754	| UVD_CGC_CTRL__MPRD_MODE_MASK
755	| UVD_CGC_CTRL__MPC_MODE_MASK
756	| UVD_CGC_CTRL__LBSI_MODE_MASK
757	| UVD_CGC_CTRL__LRBBM_MODE_MASK
758	| UVD_CGC_CTRL__WCB_MODE_MASK
759	| UVD_CGC_CTRL__VCPU_MODE_MASK
760	| UVD_CGC_CTRL__MMSCH_MODE_MASK);
761	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
762
763	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
764	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
765	| UVD_SUVD_CGC_GATE__SIT_MASK
766	| UVD_SUVD_CGC_GATE__SMP_MASK
767	| UVD_SUVD_CGC_GATE__SCM_MASK
768	| UVD_SUVD_CGC_GATE__SDB_MASK
769	| UVD_SUVD_CGC_GATE__SRE_H264_MASK
770	| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
771	| UVD_SUVD_CGC_GATE__SIT_H264_MASK
772	| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
773	| UVD_SUVD_CGC_GATE__SCM_H264_MASK
774	| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
775	| UVD_SUVD_CGC_GATE__SDB_H264_MASK
776	| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
777	| UVD_SUVD_CGC_GATE__SCLR_MASK
778	| UVD_SUVD_CGC_GATE__ENT_MASK
779	| UVD_SUVD_CGC_GATE__IME_MASK
780	| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
781	| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
782	| UVD_SUVD_CGC_GATE__SITE_MASK
783	| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
784	| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
785	| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
786	| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
787	| UVD_SUVD_CGC_GATE__IME_HEVC_MASK
788	| UVD_SUVD_CGC_GATE__EFC_MASK
789	| UVD_SUVD_CGC_GATE__SAOE_MASK
790	| UVD_SUVD_CGC_GATE__SRE_AV1_MASK
791	| UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
792	| UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
793	| UVD_SUVD_CGC_GATE__SCM_AV1_MASK
794	| UVD_SUVD_CGC_GATE__SMPA_MASK);
795	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
796
797	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
798	data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
799	| UVD_SUVD_CGC_GATE2__MPBE1_MASK
800	| UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
801	| UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
802	| UVD_SUVD_CGC_GATE2__MPC1_MASK);
803	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
804
805	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
806	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
807	| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
808	| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
809	| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
810	| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
811	| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
812	| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
813	| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
814	| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
815	| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
816	| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
817	| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
818	| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
819	| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
820	| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
821	| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
822	| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
823	| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
824	| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
825	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
826	}
827
828	static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
829	uint8_t sram_sel, int inst_idx, uint8_t indirect)
830	{
831	uint32_t reg_data = 0;
832
833	/* enable sw clock gating control */
834	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
835	reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
836	else
837	reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
838	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
839	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
840	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
841	UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
842	UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
843	UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
844	UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
845	UVD_CGC_CTRL__SYS_MODE_MASK |
846	UVD_CGC_CTRL__UDEC_MODE_MASK |
847	UVD_CGC_CTRL__MPEG2_MODE_MASK |
848	UVD_CGC_CTRL__REGS_MODE_MASK |
849	UVD_CGC_CTRL__RBC_MODE_MASK |
850	UVD_CGC_CTRL__LMI_MC_MODE_MASK |
851	UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
852	UVD_CGC_CTRL__IDCT_MODE_MASK |
853	UVD_CGC_CTRL__MPRD_MODE_MASK |
854	UVD_CGC_CTRL__MPC_MODE_MASK |
855	UVD_CGC_CTRL__LBSI_MODE_MASK |
856	UVD_CGC_CTRL__LRBBM_MODE_MASK |
857	UVD_CGC_CTRL__WCB_MODE_MASK |
858	UVD_CGC_CTRL__VCPU_MODE_MASK |
859	UVD_CGC_CTRL__MMSCH_MODE_MASK);
860	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
861	VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
862
863	/* turn off clock gating */
864	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
865	VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
866
867	/* turn on SUVD clock gating */
868	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
869	VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
870
871	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
872	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
873	VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
874	}
875
876	/**
877	* vcn_v3_0_enable_clock_gating - enable VCN clock gating
878	*
879	* @adev: amdgpu_device pointer
880	* @inst: instance number
881	*
882	* Enable clock gating for VCN block
883	*/
884	static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
885	{
886	uint32_t data;
887
888	/* enable VCN CGC */
889	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
890	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
891	data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
892	else
893	data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
894	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
895	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
896	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
897
898	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
899	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
900	| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
901	| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
902	| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
903	| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
904	| UVD_CGC_CTRL__SYS_MODE_MASK
905	| UVD_CGC_CTRL__UDEC_MODE_MASK
906	| UVD_CGC_CTRL__MPEG2_MODE_MASK
907	| UVD_CGC_CTRL__REGS_MODE_MASK
908	| UVD_CGC_CTRL__RBC_MODE_MASK
909	| UVD_CGC_CTRL__LMI_MC_MODE_MASK
910	| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
911	| UVD_CGC_CTRL__IDCT_MODE_MASK
912	| UVD_CGC_CTRL__MPRD_MODE_MASK
913	| UVD_CGC_CTRL__MPC_MODE_MASK
914	| UVD_CGC_CTRL__LBSI_MODE_MASK
915	| UVD_CGC_CTRL__LRBBM_MODE_MASK
916	| UVD_CGC_CTRL__WCB_MODE_MASK
917	| UVD_CGC_CTRL__VCPU_MODE_MASK
918	| UVD_CGC_CTRL__MMSCH_MODE_MASK);
919	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
920
921	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
922	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
923	| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
924	| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
925	| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
926	| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
927	| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
928	| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
929	| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
930	| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
931	| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
932	| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
933	| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
934	| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
935	| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
936	| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
937	| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
938	| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
939	| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
940	| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
941	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
942	}
943
944	static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
945	{
946	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
947	struct amdgpu_ring *ring;
948	uint32_t rb_bufsz, tmp;
949
950	/* disable register anti-hang mechanism */
951	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
952	~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
953	/* enable dynamic power gating mode */
954	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
955	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
956	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
957	WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
958
959	if (indirect)
960	adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
961
962	/* enable clock gating */
963	vcn_v3_0_clock_gating_dpg_mode(adev, sram_sel: 0, inst_idx, indirect);
964
965	/* enable VCPU clock */
966	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
967	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
968	tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
969	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
970	VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
971
972	/* disable master interupt */
973	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
974	VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
975
976	/* setup mmUVD_LMI_CTRL */
977	tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
978	UVD_LMI_CTRL__REQ_MODE_MASK |
979	UVD_LMI_CTRL__CRC_RESET_MASK |
980	UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
981	UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
982	UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
983	(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
984	0x00100000L);
985	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
986	VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
987
988	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
989	VCN, inst_idx, mmUVD_MPC_CNTL),
990	0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
991
992	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
993	VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
994	((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
995	(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
996	(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
997	(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
998
999	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1000	VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
1001	((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1002	(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1003	(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1004	(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1005
1006	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1007	VCN, inst_idx, mmUVD_MPC_SET_MUX),
1008	((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1009	(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1010	(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1011
1012	vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
1013
1014	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1015	VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1016	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1017	VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1018
1019	/* enable LMI MC and UMC channels */
1020	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1021	VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1022
1023	/* unblock VCPU register access */
1024	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1025	VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1026
1027	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1028	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1029	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1030	VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1031
1032	/* enable master interrupt */
1033	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1034	VCN, inst_idx, mmUVD_MASTINT_EN),
1035	UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1036
1037	/* add nop to workaround PSP size check */
1038	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1039	VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1040
1041	if (indirect)
1042	amdgpu_vcn_psp_update_sram(adev, inst_idx, ucode_id: 0);
1043
1044	ring = &adev->vcn.inst[inst_idx].ring_dec;
1045	/* force RBC into idle state */
1046	rb_bufsz = order_base_2(ring->ring_size);
1047	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1048	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1049	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1050	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1051	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1052	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1053
1054	/* Stall DPG before WPTR/RPTR reset */
1055	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1056	UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1057	~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1058	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1059
1060	/* set the write pointer delay */
1061	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1062
1063	/* set the wb address */
1064	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1065	(upper_32_bits(ring->gpu_addr) >> 2));
1066
1067	/* programm the RB_BASE for ring buffer */
1068	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1069	lower_32_bits(ring->gpu_addr));
1070	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1071	upper_32_bits(ring->gpu_addr));
1072
1073	/* Initialize the ring buffer's read and write pointers */
1074	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1075
1076	WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1077
1078	ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1079	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1080	lower_32_bits(ring->wptr));
1081
1082	/* Reset FW shared memory RBC WPTR/RPTR */
1083	fw_shared->rb.rptr = 0;
1084	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1085
1086	/*resetting done, fw can check RB ring */
1087	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1088
1089	/* Unstall DPG */
1090	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1091	0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1092
1093	return 0;
1094	}
1095
1096	static int vcn_v3_0_start(struct amdgpu_device *adev)
1097	{
1098	volatile struct amdgpu_fw_shared *fw_shared;
1099	struct amdgpu_ring *ring;
1100	uint32_t rb_bufsz, tmp;
1101	int i, j, k, r;
1102
1103	if (adev->pm.dpm_enabled)
1104	amdgpu_dpm_enable_uvd(adev, enable: true);
1105
1106	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1107	if (adev->vcn.harvest_config & (1 << i))
1108	continue;
1109
1110	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1111	r = vcn_v3_0_start_dpg_mode(adev, inst_idx: i, indirect: adev->vcn.indirect_sram);
1112	continue;
1113	}
1114
1115	/* disable VCN power gating */
1116	vcn_v3_0_disable_static_power_gating(adev, inst: i);
1117
1118	/* set VCN status busy */
1119	tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1120	WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1121
1122	/*SW clock gating */
1123	vcn_v3_0_disable_clock_gating(adev, inst: i);
1124
1125	/* enable VCPU clock */
1126	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1127	UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1128
1129	/* disable master interrupt */
1130	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1131	~UVD_MASTINT_EN__VCPU_EN_MASK);
1132
1133	/* enable LMI MC and UMC channels */
1134	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1135	~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1136
1137	tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1138	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1139	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1140	WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1141
1142	/* setup mmUVD_LMI_CTRL */
1143	tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1144	WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1145	UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1146	UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1147	UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1148	UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1149
1150	/* setup mmUVD_MPC_CNTL */
1151	tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1152	tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1153	tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1154	WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1155
1156	/* setup UVD_MPC_SET_MUXA0 */
1157	WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1158	((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1159	(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1160	(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1161	(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1162
1163	/* setup UVD_MPC_SET_MUXB0 */
1164	WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1165	((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1166	(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1167	(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1168	(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1169
1170	/* setup mmUVD_MPC_SET_MUX */
1171	WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1172	((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1173	(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1174	(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1175
1176	vcn_v3_0_mc_resume(adev, inst: i);
1177
1178	/* VCN global tiling registers */
1179	WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1180	adev->gfx.config.gb_addr_config);
1181
1182	/* unblock VCPU register access */
1183	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1184	~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1185
1186	/* release VCPU reset to boot */
1187	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1188	~UVD_VCPU_CNTL__BLK_RST_MASK);
1189
1190	for (j = 0; j < 10; ++j) {
1191	uint32_t status;
1192
1193	for (k = 0; k < 100; ++k) {
1194	status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1195	if (status & 2)
1196	break;
1197	mdelay(10);
1198	}
1199	r = 0;
1200	if (status & 2)
1201	break;
1202
1203	DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1204	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1205	UVD_VCPU_CNTL__BLK_RST_MASK,
1206	~UVD_VCPU_CNTL__BLK_RST_MASK);
1207	mdelay(10);
1208	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1209	~UVD_VCPU_CNTL__BLK_RST_MASK);
1210
1211	mdelay(10);
1212	r = -1;
1213	}
1214
1215	if (r) {
1216	DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1217	return r;
1218	}
1219
1220	/* enable master interrupt */
1221	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1222	UVD_MASTINT_EN__VCPU_EN_MASK,
1223	~UVD_MASTINT_EN__VCPU_EN_MASK);
1224
1225	/* clear the busy bit of VCN_STATUS */
1226	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1227	~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1228
1229	WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1230
1231	ring = &adev->vcn.inst[i].ring_dec;
1232	/* force RBC into idle state */
1233	rb_bufsz = order_base_2(ring->ring_size);
1234	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1235	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1236	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1237	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1238	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1239	WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1240
1241	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1242	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1243
1244	/* programm the RB_BASE for ring buffer */
1245	WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1246	lower_32_bits(ring->gpu_addr));
1247	WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1248	upper_32_bits(ring->gpu_addr));
1249
1250	/* Initialize the ring buffer's read and write pointers */
1251	WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1252
1253	WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);
1254	ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1255	WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1256	lower_32_bits(ring->wptr));
1257	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1258	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1259
1260	if (amdgpu_ip_version(adev, ip: UVD_HWIP, inst: 0) !=
1261	IP_VERSION(3, 0, 33)) {
1262	fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1263	ring = &adev->vcn.inst[i].ring_enc[0];
1264	WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1265	WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1266	WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1267	WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1268	WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1269	fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1270
1271	fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1272	ring = &adev->vcn.inst[i].ring_enc[1];
1273	WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1274	WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1275	WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1276	WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1277	WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1278	fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1279	}
1280	}
1281
1282	return 0;
1283	}
1284
1285	static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1286	{
1287	int i, j;
1288	struct amdgpu_ring *ring;
1289	uint64_t cache_addr;
1290	uint64_t rb_addr;
1291	uint64_t ctx_addr;
1292	uint32_t param, resp, expected;
1293	uint32_t offset, cache_size;
1294	uint32_t tmp, timeout;
1295
1296	struct amdgpu_mm_table *table = &adev->virt.mm_table;
1297	uint32_t *table_loc;
1298	uint32_t table_size;
1299	uint32_t size, size_dw;
1300
1301	struct mmsch_v3_0_cmd_direct_write
1302	direct_wt = { {0} };
1303	struct mmsch_v3_0_cmd_direct_read_modify_write
1304	direct_rd_mod_wt = { {0} };
1305	struct mmsch_v3_0_cmd_end end = { {0} };
1306	struct mmsch_v3_0_init_header header;
1307
1308	direct_wt.cmd_header.command_type =
1309	MMSCH_COMMAND__DIRECT_REG_WRITE;
1310	direct_rd_mod_wt.cmd_header.command_type =
1311	MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1312	end.cmd_header.command_type =
1313	MMSCH_COMMAND__END;
1314
1315	header.version = MMSCH_VERSION;
1316	header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1317	for (i = 0; i < MMSCH_V3_0_VCN_INSTANCES; i++) {
1318	header.inst[i].init_status = 0;
1319	header.inst[i].table_offset = 0;
1320	header.inst[i].table_size = 0;
1321	}
1322
1323	table_loc = (uint32_t *)table->cpu_addr;
1324	table_loc += header.total_size;
1325	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1326	if (adev->vcn.harvest_config & (1 << i))
1327	continue;
1328
1329	table_size = 0;
1330
1331	MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1332	mmUVD_STATUS),
1333	~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1334
1335	cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[i]->size + 4);
1336
1337	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1338	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1339	mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1340	adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1341	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1342	mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1343	adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1344	offset = 0;
1345	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1346	mmUVD_VCPU_CACHE_OFFSET0),
1347	0);
1348	} else {
1349	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1350	mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1351	lower_32_bits(adev->vcn.inst[i].gpu_addr));
1352	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1353	mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1354	upper_32_bits(adev->vcn.inst[i].gpu_addr));
1355	offset = cache_size;
1356	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1357	mmUVD_VCPU_CACHE_OFFSET0),
1358	AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1359	}
1360
1361	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1362	mmUVD_VCPU_CACHE_SIZE0),
1363	cache_size);
1364
1365	cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1366	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1367	mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1368	lower_32_bits(cache_addr));
1369	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1370	mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1371	upper_32_bits(cache_addr));
1372	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1373	mmUVD_VCPU_CACHE_OFFSET1),
1374	0);
1375	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1376	mmUVD_VCPU_CACHE_SIZE1),
1377	AMDGPU_VCN_STACK_SIZE);
1378
1379	cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1380	AMDGPU_VCN_STACK_SIZE;
1381	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1382	mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1383	lower_32_bits(cache_addr));
1384	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1385	mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1386	upper_32_bits(cache_addr));
1387	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1388	mmUVD_VCPU_CACHE_OFFSET2),
1389	0);
1390	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1391	mmUVD_VCPU_CACHE_SIZE2),
1392	AMDGPU_VCN_CONTEXT_SIZE);
1393
1394	for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1395	ring = &adev->vcn.inst[i].ring_enc[j];
1396	ring->wptr = 0;
1397	rb_addr = ring->gpu_addr;
1398	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1399	mmUVD_RB_BASE_LO),
1400	lower_32_bits(rb_addr));
1401	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1402	mmUVD_RB_BASE_HI),
1403	upper_32_bits(rb_addr));
1404	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1405	mmUVD_RB_SIZE),
1406	ring->ring_size / 4);
1407	}
1408
1409	ring = &adev->vcn.inst[i].ring_dec;
1410	ring->wptr = 0;
1411	rb_addr = ring->gpu_addr;
1412	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1413	mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1414	lower_32_bits(rb_addr));
1415	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1416	mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1417	upper_32_bits(rb_addr));
1418	/* force RBC into idle state */
1419	tmp = order_base_2(ring->ring_size);
1420	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1421	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1422	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1423	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1424	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1425	MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1426	mmUVD_RBC_RB_CNTL),
1427	tmp);
1428
1429	/* add end packet */
1430	MMSCH_V3_0_INSERT_END();
1431
1432	/* refine header */
1433	header.inst[i].init_status = 0;
1434	header.inst[i].table_offset = header.total_size;
1435	header.inst[i].table_size = table_size;
1436	header.total_size += table_size;
1437	}
1438
1439	/* Update init table header in memory */
1440	size = sizeof(struct mmsch_v3_0_init_header);
1441	table_loc = (uint32_t *)table->cpu_addr;
1442	memcpy((void *)table_loc, &header, size);
1443
1444	/* message MMSCH (in VCN[0]) to initialize this client
1445	* 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1446	* of memory descriptor location
1447	*/
1448	ctx_addr = table->gpu_addr;
1449	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1450	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1451
1452	/* 2, update vmid of descriptor */
1453	tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1454	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1455	/* use domain0 for MM scheduler */
1456	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1457	WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1458
1459	/* 3, notify mmsch about the size of this descriptor */
1460	size = header.total_size;
1461	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1462
1463	/* 4, set resp to zero */
1464	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1465
1466	/* 5, kick off the initialization and wait until
1467	* MMSCH_VF_MAILBOX_RESP becomes non-zero
1468	*/
1469	param = 0x10000001;
1470	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1471	tmp = 0;
1472	timeout = 1000;
1473	resp = 0;
1474	expected = param + 1;
1475	while (resp != expected) {
1476	resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1477	if (resp == expected)
1478	break;
1479
1480	udelay(10);
1481	tmp = tmp + 10;
1482	if (tmp >= timeout) {
1483	DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1484	" waiting for mmMMSCH_VF_MAILBOX_RESP "\
1485	"(expected=0x%08x, readback=0x%08x)\n",
1486	tmp, expected, resp);
1487	return -EBUSY;
1488	}
1489	}
1490
1491	return 0;
1492	}
1493
1494	static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1495	{
1496	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1497	uint32_t tmp;
1498
1499	vcn_v3_0_pause_dpg_mode(adev, inst_idx, new_state: &state);
1500
1501	/* Wait for power status to be 1 */
1502	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1503	UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1504
1505	/* wait for read ptr to be equal to write ptr */
1506	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1507	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1508
1509	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1510	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1511
1512	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1513	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1514
1515	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1516	UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1517
1518	/* disable dynamic power gating mode */
1519	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1520	~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1521
1522	return 0;
1523	}
1524
1525	static int vcn_v3_0_stop(struct amdgpu_device *adev)
1526	{
1527	uint32_t tmp;
1528	int i, r = 0;
1529
1530	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1531	if (adev->vcn.harvest_config & (1 << i))
1532	continue;
1533
1534	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1535	r = vcn_v3_0_stop_dpg_mode(adev, inst_idx: i);
1536	continue;
1537	}
1538
1539	/* wait for vcn idle */
1540	r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1541	if (r)
1542	return r;
1543
1544	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1545	UVD_LMI_STATUS__READ_CLEAN_MASK |
1546	UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1547	UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1548	r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1549	if (r)
1550	return r;
1551
1552	/* disable LMI UMC channel */
1553	tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1554	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1555	WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1556	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1557	UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1558	r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1559	if (r)
1560	return r;
1561
1562	/* block VCPU register access */
1563	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1564	UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1565	~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1566
1567	/* reset VCPU */
1568	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1569	UVD_VCPU_CNTL__BLK_RST_MASK,
1570	~UVD_VCPU_CNTL__BLK_RST_MASK);
1571
1572	/* disable VCPU clock */
1573	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1574	~(UVD_VCPU_CNTL__CLK_EN_MASK));
1575
1576	/* apply soft reset */
1577	tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1578	tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1579	WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1580	tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1581	tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1582	WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1583
1584	/* clear status */
1585	WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1586
1587	/* apply HW clock gating */
1588	vcn_v3_0_enable_clock_gating(adev, inst: i);
1589
1590	/* enable VCN power gating */
1591	vcn_v3_0_enable_static_power_gating(adev, inst: i);
1592	}
1593
1594	if (adev->pm.dpm_enabled)
1595	amdgpu_dpm_enable_uvd(adev, enable: false);
1596
1597	return 0;
1598	}
1599
1600	static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1601	int inst_idx, struct dpg_pause_state *new_state)
1602	{
1603	volatile struct amdgpu_fw_shared *fw_shared;
1604	struct amdgpu_ring *ring;
1605	uint32_t reg_data = 0;
1606	int ret_code;
1607
1608	/* pause/unpause if state is changed */
1609	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1610	DRM_DEBUG("dpg pause state changed %d -> %d",
1611	adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1612	reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1613	(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1614
1615	if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1616	ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1617	UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1618
1619	if (!ret_code) {
1620	/* pause DPG */
1621	reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1622	WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1623
1624	/* wait for ACK */
1625	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1626	UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1627	UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1628
1629	/* Stall DPG before WPTR/RPTR reset */
1630	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1631	UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1632	~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1633
1634	if (amdgpu_ip_version(adev, ip: UVD_HWIP, inst: 0) !=
1635	IP_VERSION(3, 0, 33)) {
1636	/* Restore */
1637	fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1638	fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1639	ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1640	ring->wptr = 0;
1641	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1642	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1643	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1644	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1645	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1646	fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1647
1648	fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1649	ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1650	ring->wptr = 0;
1651	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1652	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1653	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1654	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1655	WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1656	fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1657
1658	/* restore wptr/rptr with pointers saved in FW shared memory*/
1659	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);
1660	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);
1661	}
1662
1663	/* Unstall DPG */
1664	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1665	0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1666
1667	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1668	UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1669	}
1670	} else {
1671	/* unpause dpg, no need to wait */
1672	reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1673	WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1674	}
1675	adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1676	}
1677
1678	return 0;
1679	}
1680
1681	/**
1682	* vcn_v3_0_dec_ring_get_rptr - get read pointer
1683	*
1684	* @ring: amdgpu_ring pointer
1685	*
1686	* Returns the current hardware read pointer
1687	*/
1688	static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1689	{
1690	struct amdgpu_device *adev = ring->adev;
1691
1692	return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1693	}
1694
1695	/**
1696	* vcn_v3_0_dec_ring_get_wptr - get write pointer
1697	*
1698	* @ring: amdgpu_ring pointer
1699	*
1700	* Returns the current hardware write pointer
1701	*/
1702	static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1703	{
1704	struct amdgpu_device *adev = ring->adev;
1705
1706	if (ring->use_doorbell)
1707	return *ring->wptr_cpu_addr;
1708	else
1709	return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1710	}
1711
1712	/**
1713	* vcn_v3_0_dec_ring_set_wptr - set write pointer
1714	*
1715	* @ring: amdgpu_ring pointer
1716	*
1717	* Commits the write pointer to the hardware
1718	*/
1719	static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1720	{
1721	struct amdgpu_device *adev = ring->adev;
1722	volatile struct amdgpu_fw_shared *fw_shared;
1723
1724	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1725	/*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */
1726	fw_shared = adev->vcn.inst[ring->me].fw_shared.cpu_addr;
1727	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1728	WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,
1729	lower_32_bits(ring->wptr));
1730	}
1731
1732	if (ring->use_doorbell) {
1733	*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1734	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1735	} else {
1736	WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1737	}
1738	}
1739
1740	static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1741	.type = AMDGPU_RING_TYPE_VCN_DEC,
1742	.align_mask = 0x3f,
1743	.nop = VCN_DEC_SW_CMD_NO_OP,
1744	.secure_submission_supported = true,
1745	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1746	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1747	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1748	.emit_frame_size =
1749	SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1750	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1751	VCN_SW_RING_EMIT_FRAME_SIZE,
1752	.emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */
1753	.emit_ib = vcn_dec_sw_ring_emit_ib,
1754	.emit_fence = vcn_dec_sw_ring_emit_fence,
1755	.emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,
1756	.test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1757	.test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1758	.insert_nop = amdgpu_ring_insert_nop,
1759	.insert_end = vcn_dec_sw_ring_insert_end,
1760	.pad_ib = amdgpu_ring_generic_pad_ib,
1761	.begin_use = amdgpu_vcn_ring_begin_use,
1762	.end_use = amdgpu_vcn_ring_end_use,
1763	.emit_wreg = vcn_dec_sw_ring_emit_wreg,
1764	.emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,
1765	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1766	};
1767
1768	static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p,
1769	struct amdgpu_job *job)
1770	{
1771	struct drm_gpu_scheduler **scheds;
1772
1773	/* The create msg must be in the first IB submitted */
1774	if (atomic_read(v: &job->base.entity->fence_seq))
1775	return -EINVAL;
1776
1777	/* if VCN0 is harvested, we can't support AV1 */
1778	if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1779	return -EINVAL;
1780
1781	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
1782	[AMDGPU_RING_PRIO_DEFAULT].sched;
1783	drm_sched_entity_modify_sched(entity: job->base.entity, sched_list: scheds, num_sched_list: 1);
1784	return 0;
1785	}
1786
1787	static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1788	uint64_t addr)
1789	{
1790	struct ttm_operation_ctx ctx = { false, false };
1791	struct amdgpu_bo_va_mapping *map;
1792	uint32_t *msg, num_buffers;
1793	struct amdgpu_bo *bo;
1794	uint64_t start, end;
1795	unsigned int i;
1796	void *ptr;
1797	int r;
1798
1799	addr &= AMDGPU_GMC_HOLE_MASK;
1800	r = amdgpu_cs_find_mapping(parser: p, addr, bo: &bo, mapping: &map);
1801	if (r) {
1802	DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
1803	return r;
1804	}
1805
1806	start = map->start * AMDGPU_GPU_PAGE_SIZE;
1807	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1808	if (addr & 0x7) {
1809	DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1810	return -EINVAL;
1811	}
1812
1813	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1814	amdgpu_bo_placement_from_domain(abo: bo, domain: bo->allowed_domains);
1815	r = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
1816	if (r) {
1817	DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1818	return r;
1819	}
1820
1821	r = amdgpu_bo_kmap(bo, ptr: &ptr);
1822	if (r) {
1823	DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1824	return r;
1825	}
1826
1827	msg = ptr + addr - start;
1828
1829	/* Check length */
1830	if (msg[1] > end - addr) {
1831	r = -EINVAL;
1832	goto out;
1833	}
1834
1835	if (msg[3] != RDECODE_MSG_CREATE)
1836	goto out;
1837
1838	num_buffers = msg[2];
1839	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1840	uint32_t offset, size, *create;
1841
1842	if (msg[0] != RDECODE_MESSAGE_CREATE)
1843	continue;
1844
1845	offset = msg[1];
1846	size = msg[2];
1847
1848	if (offset + size > end) {
1849	r = -EINVAL;
1850	goto out;
1851	}
1852
1853	create = ptr + addr + offset - start;
1854
1855	/* H246, HEVC and VP9 can run on any instance */
1856	if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1857	continue;
1858
1859	r = vcn_v3_0_limit_sched(p, job);
1860	if (r)
1861	goto out;
1862	}
1863
1864	out:
1865	amdgpu_bo_kunmap(bo);
1866	return r;
1867	}
1868
1869	static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1870	struct amdgpu_job *job,
1871	struct amdgpu_ib *ib)
1872	{
1873	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1874	uint32_t msg_lo = 0, msg_hi = 0;
1875	unsigned i;
1876	int r;
1877
1878	/* The first instance can decode anything */
1879	if (!ring->me)
1880	return 0;
1881
1882	for (i = 0; i < ib->length_dw; i += 2) {
1883	uint32_t reg = amdgpu_ib_get_value(ib, idx: i);
1884	uint32_t val = amdgpu_ib_get_value(ib, idx: i + 1);
1885
1886	if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {
1887	msg_lo = val;
1888	} else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {
1889	msg_hi = val;
1890	} else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&
1891	val == 0) {
1892	r = vcn_v3_0_dec_msg(p, job,
1893	addr: ((u64)msg_hi) << 32 | msg_lo);
1894	if (r)
1895	return r;
1896	}
1897	}
1898	return 0;
1899	}
1900
1901	static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1902	.type = AMDGPU_RING_TYPE_VCN_DEC,
1903	.align_mask = 0xf,
1904	.secure_submission_supported = true,
1905	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1906	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1907	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1908	.patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,
1909	.emit_frame_size =
1910	SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1911	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1912	8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1913	14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1914	6,
1915	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1916	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
1917	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
1918	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1919	.test_ring = vcn_v2_0_dec_ring_test_ring,
1920	.test_ib = amdgpu_vcn_dec_ring_test_ib,
1921	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
1922	.insert_start = vcn_v2_0_dec_ring_insert_start,
1923	.insert_end = vcn_v2_0_dec_ring_insert_end,
1924	.pad_ib = amdgpu_ring_generic_pad_ib,
1925	.begin_use = amdgpu_vcn_ring_begin_use,
1926	.end_use = amdgpu_vcn_ring_end_use,
1927	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1928	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1929	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1930	};
1931
1932	/**
1933	* vcn_v3_0_enc_ring_get_rptr - get enc read pointer
1934	*
1935	* @ring: amdgpu_ring pointer
1936	*
1937	* Returns the current hardware enc read pointer
1938	*/
1939	static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1940	{
1941	struct amdgpu_device *adev = ring->adev;
1942
1943	if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1944	return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1945	else
1946	return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1947	}
1948
1949	/**
1950	* vcn_v3_0_enc_ring_get_wptr - get enc write pointer
1951	*
1952	* @ring: amdgpu_ring pointer
1953	*
1954	* Returns the current hardware enc write pointer
1955	*/
1956	static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1957	{
1958	struct amdgpu_device *adev = ring->adev;
1959
1960	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1961	if (ring->use_doorbell)
1962	return *ring->wptr_cpu_addr;
1963	else
1964	return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1965	} else {
1966	if (ring->use_doorbell)
1967	return *ring->wptr_cpu_addr;
1968	else
1969	return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1970	}
1971	}
1972
1973	/**
1974	* vcn_v3_0_enc_ring_set_wptr - set enc write pointer
1975	*
1976	* @ring: amdgpu_ring pointer
1977	*
1978	* Commits the enc write pointer to the hardware
1979	*/
1980	static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1981	{
1982	struct amdgpu_device *adev = ring->adev;
1983
1984	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1985	if (ring->use_doorbell) {
1986	*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1987	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1988	} else {
1989	WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1990	}
1991	} else {
1992	if (ring->use_doorbell) {
1993	*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1994	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1995	} else {
1996	WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1997	}
1998	}
1999	}
2000
2001	static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
2002	.type = AMDGPU_RING_TYPE_VCN_ENC,
2003	.align_mask = 0x3f,
2004	.nop = VCN_ENC_CMD_NO_OP,
2005	.get_rptr = vcn_v3_0_enc_ring_get_rptr,
2006	.get_wptr = vcn_v3_0_enc_ring_get_wptr,
2007	.set_wptr = vcn_v3_0_enc_ring_set_wptr,
2008	.emit_frame_size =
2009	SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2010	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2011	4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2012	5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2013	1, /* vcn_v2_0_enc_ring_insert_end */
2014	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2015	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
2016	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
2017	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2018	.test_ring = amdgpu_vcn_enc_ring_test_ring,
2019	.test_ib = amdgpu_vcn_enc_ring_test_ib,
2020	.insert_nop = amdgpu_ring_insert_nop,
2021	.insert_end = vcn_v2_0_enc_ring_insert_end,
2022	.pad_ib = amdgpu_ring_generic_pad_ib,
2023	.begin_use = amdgpu_vcn_ring_begin_use,
2024	.end_use = amdgpu_vcn_ring_end_use,
2025	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2026	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2027	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2028	};
2029
2030	static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2031	{
2032	int i;
2033
2034	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2035	if (adev->vcn.harvest_config & (1 << i))
2036	continue;
2037
2038	if (!DEC_SW_RING_ENABLED)
2039	adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
2040	else
2041	adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
2042	adev->vcn.inst[i].ring_dec.me = i;
2043	DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
2044	DEC_SW_RING_ENABLED?"(Software Ring)":"");
2045	}
2046	}
2047
2048	static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2049	{
2050	int i, j;
2051
2052	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2053	if (adev->vcn.harvest_config & (1 << i))
2054	continue;
2055
2056	for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
2057	adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
2058	adev->vcn.inst[i].ring_enc[j].me = i;
2059	}
2060	if (adev->vcn.num_enc_rings > 0)
2061	DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
2062	}
2063	}
2064
2065	static bool vcn_v3_0_is_idle(void *handle)
2066	{
2067	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2068	int i, ret = 1;
2069
2070	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2071	if (adev->vcn.harvest_config & (1 << i))
2072	continue;
2073
2074	ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
2075	}
2076
2077	return ret;
2078	}
2079
2080	static int vcn_v3_0_wait_for_idle(void *handle)
2081	{
2082	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2083	int i, ret = 0;
2084
2085	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2086	if (adev->vcn.harvest_config & (1 << i))
2087	continue;
2088
2089	ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2090	UVD_STATUS__IDLE);
2091	if (ret)
2092	return ret;
2093	}
2094
2095	return ret;
2096	}
2097
2098	static int vcn_v3_0_set_clockgating_state(void *handle,
2099	enum amd_clockgating_state state)
2100	{
2101	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2102	bool enable = state == AMD_CG_STATE_GATE;
2103	int i;
2104
2105	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2106	if (adev->vcn.harvest_config & (1 << i))
2107	continue;
2108
2109	if (enable) {
2110	if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2111	return -EBUSY;
2112	vcn_v3_0_enable_clock_gating(adev, inst: i);
2113	} else {
2114	vcn_v3_0_disable_clock_gating(adev, inst: i);
2115	}
2116	}
2117
2118	return 0;
2119	}
2120
2121	static int vcn_v3_0_set_powergating_state(void *handle,
2122	enum amd_powergating_state state)
2123	{
2124	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2125	int ret;
2126
2127	/* for SRIOV, guest should not control VCN Power-gating
2128	* MMSCH FW should control Power-gating and clock-gating
2129	* guest should avoid touching CGC and PG
2130	*/
2131	if (amdgpu_sriov_vf(adev)) {
2132	adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2133	return 0;
2134	}
2135
2136	if (state == adev->vcn.cur_state)
2137	return 0;
2138
2139	if (state == AMD_PG_STATE_GATE)
2140	ret = vcn_v3_0_stop(adev);
2141	else
2142	ret = vcn_v3_0_start(adev);
2143
2144	if (!ret)
2145	adev->vcn.cur_state = state;
2146
2147	return ret;
2148	}
2149
2150	static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2151	struct amdgpu_irq_src *source,
2152	unsigned type,
2153	enum amdgpu_interrupt_state state)
2154	{
2155	return 0;
2156	}
2157
2158	static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2159	struct amdgpu_irq_src *source,
2160	struct amdgpu_iv_entry *entry)
2161	{
2162	uint32_t ip_instance;
2163
2164	switch (entry->client_id) {
2165	case SOC15_IH_CLIENTID_VCN:
2166	ip_instance = 0;
2167	break;
2168	case SOC15_IH_CLIENTID_VCN1:
2169	ip_instance = 1;
2170	break;
2171	default:
2172	DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2173	return 0;
2174	}
2175
2176	DRM_DEBUG("IH: VCN TRAP\n");
2177
2178	switch (entry->src_id) {
2179	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2180	amdgpu_fence_process(ring: &adev->vcn.inst[ip_instance].ring_dec);
2181	break;
2182	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2183	amdgpu_fence_process(ring: &adev->vcn.inst[ip_instance].ring_enc[0]);
2184	break;
2185	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2186	amdgpu_fence_process(ring: &adev->vcn.inst[ip_instance].ring_enc[1]);
2187	break;
2188	default:
2189	DRM_ERROR("Unhandled interrupt: %d %d\n",
2190	entry->src_id, entry->src_data[0]);
2191	break;
2192	}
2193
2194	return 0;
2195	}
2196
2197	static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2198	.set = vcn_v3_0_set_interrupt_state,
2199	.process = vcn_v3_0_process_interrupt,
2200	};
2201
2202	static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2203	{
2204	int i;
2205
2206	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2207	if (adev->vcn.harvest_config & (1 << i))
2208	continue;
2209
2210	adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2211	adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2212	}
2213	}
2214
2215	static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2216	.name = "vcn_v3_0",
2217	.early_init = vcn_v3_0_early_init,
2218	.late_init = NULL,
2219	.sw_init = vcn_v3_0_sw_init,
2220	.sw_fini = vcn_v3_0_sw_fini,
2221	.hw_init = vcn_v3_0_hw_init,
2222	.hw_fini = vcn_v3_0_hw_fini,
2223	.suspend = vcn_v3_0_suspend,
2224	.resume = vcn_v3_0_resume,
2225	.is_idle = vcn_v3_0_is_idle,
2226	.wait_for_idle = vcn_v3_0_wait_for_idle,
2227	.check_soft_reset = NULL,
2228	.pre_soft_reset = NULL,
2229	.soft_reset = NULL,
2230	.post_soft_reset = NULL,
2231	.set_clockgating_state = vcn_v3_0_set_clockgating_state,
2232	.set_powergating_state = vcn_v3_0_set_powergating_state,
2233	};
2234
2235	const struct amdgpu_ip_block_version vcn_v3_0_ip_block = {
2236	.type = AMD_IP_BLOCK_TYPE_VCN,
2237	.major = 3,
2238	.minor = 0,
2239	.rev = 0,
2240	.funcs = &vcn_v3_0_ip_funcs,
2241	};
2242