1	/*
2	* Copyright 2016 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/delay.h>
25	#include <linux/kernel.h>
26	#include <linux/firmware.h>
27	#include <linux/module.h>
28	#include <linux/pci.h>
29
30	#include "amdgpu.h"
31	#include "amdgpu_gfx.h"
32	#include "soc15.h"
33	#include "soc15d.h"
34	#include "amdgpu_atomfirmware.h"
35	#include "amdgpu_pm.h"
36
37	#include "gc/gc_9_0_offset.h"
38	#include "gc/gc_9_0_sh_mask.h"
39
40	#include "vega10_enum.h"
41
42	#include "soc15_common.h"
43	#include "clearstate_gfx9.h"
44	#include "v9_structs.h"
45
46	#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48	#include "amdgpu_ras.h"
49
50	#include "amdgpu_ring_mux.h"
51	#include "gfx_v9_4.h"
52	#include "gfx_v9_0.h"
53	#include "gfx_v9_4_2.h"
54
55	#include "asic_reg/pwr/pwr_10_0_offset.h"
56	#include "asic_reg/pwr/pwr_10_0_sh_mask.h"
57	#include "asic_reg/gc/gc_9_0_default.h"
58
59	#define GFX9_NUM_GFX_RINGS 1
60	#define GFX9_NUM_SW_GFX_RINGS 2
61	#define GFX9_MEC_HPD_SIZE 4096
62	#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
63	#define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
64
65	#define mmGCEA_PROBE_MAP 0x070c
66	#define mmGCEA_PROBE_MAP_BASE_IDX 0
67
68	MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
69	MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
70	MODULE_FIRMWARE("amdgpu/vega10_me.bin");
71	MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
72	MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
73	MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
74
75	MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
76	MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
77	MODULE_FIRMWARE("amdgpu/vega12_me.bin");
78	MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
79	MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
80	MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
81
82	MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
83	MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
84	MODULE_FIRMWARE("amdgpu/vega20_me.bin");
85	MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
86	MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
87	MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
88
89	MODULE_FIRMWARE("amdgpu/raven_ce.bin");
90	MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
91	MODULE_FIRMWARE("amdgpu/raven_me.bin");
92	MODULE_FIRMWARE("amdgpu/raven_mec.bin");
93	MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
94	MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
95
96	MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
97	MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
98	MODULE_FIRMWARE("amdgpu/picasso_me.bin");
99	MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
100	MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
101	MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
102	MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
103
104	MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
105	MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
106	MODULE_FIRMWARE("amdgpu/raven2_me.bin");
107	MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
108	MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
109	MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
110	MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
111
112	MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
113	MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
114
115	MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
116	MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
117	MODULE_FIRMWARE("amdgpu/renoir_me.bin");
118	MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
119	MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
120
121	MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
122	MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
123	MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
124	MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
125	MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
126	MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
127
128	MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
129	MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
130	MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
131	MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
132	MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
133
134	#define mmTCP_CHAN_STEER_0_ARCT 0x0b03
135	#define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
136	#define mmTCP_CHAN_STEER_1_ARCT 0x0b04
137	#define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
138	#define mmTCP_CHAN_STEER_2_ARCT 0x0b09
139	#define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
140	#define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
141	#define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
142	#define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
143	#define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
144	#define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
145	#define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
146
147	#define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
148	#define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
149	#define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
150	#define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
151
152	enum ta_ras_gfx_subblock {
153	/*CPC*/
154	TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
155	TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
156	TA_RAS_BLOCK__GFX_CPC_UCODE,
157	TA_RAS_BLOCK__GFX_DC_STATE_ME1,
158	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
159	TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
160	TA_RAS_BLOCK__GFX_DC_STATE_ME2,
161	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
162	TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
163	TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
164	/* CPF*/
165	TA_RAS_BLOCK__GFX_CPF_INDEX_START,
166	TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
167	TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
168	TA_RAS_BLOCK__GFX_CPF_TAG,
169	TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
170	/* CPG*/
171	TA_RAS_BLOCK__GFX_CPG_INDEX_START,
172	TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
173	TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
174	TA_RAS_BLOCK__GFX_CPG_TAG,
175	TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
176	/* GDS*/
177	TA_RAS_BLOCK__GFX_GDS_INDEX_START,
178	TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
179	TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
180	TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
181	TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
182	TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
183	TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
184	/* SPI*/
185	TA_RAS_BLOCK__GFX_SPI_SR_MEM,
186	/* SQ*/
187	TA_RAS_BLOCK__GFX_SQ_INDEX_START,
188	TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
189	TA_RAS_BLOCK__GFX_SQ_LDS_D,
190	TA_RAS_BLOCK__GFX_SQ_LDS_I,
191	TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
192	TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
193	/* SQC (3 ranges)*/
194	TA_RAS_BLOCK__GFX_SQC_INDEX_START,
195	/* SQC range 0*/
196	TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
197	TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
198	TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
199	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
200	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
201	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
202	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
203	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
204	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
205	TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
206	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
207	/* SQC range 1*/
208	TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
209	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
210	TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
211	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
212	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
213	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
214	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
215	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
216	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
217	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
218	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
219	TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
220	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
221	/* SQC range 2*/
222	TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
223	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
224	TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
225	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
226	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
227	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
228	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
229	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
230	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
231	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
232	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
233	TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
234	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
235	TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
236	/* TA*/
237	TA_RAS_BLOCK__GFX_TA_INDEX_START,
238	TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
239	TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
240	TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
241	TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
242	TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
243	TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
244	/* TCA*/
245	TA_RAS_BLOCK__GFX_TCA_INDEX_START,
246	TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
247	TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
248	TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
249	/* TCC (5 sub-ranges)*/
250	TA_RAS_BLOCK__GFX_TCC_INDEX_START,
251	/* TCC range 0*/
252	TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
253	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
254	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
255	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
256	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
257	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
258	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
259	TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
260	TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
261	TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
262	/* TCC range 1*/
263	TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
264	TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
265	TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
266	TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
267	TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
268	/* TCC range 2*/
269	TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
270	TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
271	TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
272	TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
273	TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
274	TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
275	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
276	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
277	TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
278	TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
279	TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
280	/* TCC range 3*/
281	TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
282	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
283	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
284	TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
285	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
286	/* TCC range 4*/
287	TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
288	TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
289	TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
290	TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
291	TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
292	TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
293	TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
294	/* TCI*/
295	TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
296	/* TCP*/
297	TA_RAS_BLOCK__GFX_TCP_INDEX_START,
298	TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
299	TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
300	TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
301	TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
302	TA_RAS_BLOCK__GFX_TCP_DB_RAM,
303	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
304	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
305	TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
306	/* TD*/
307	TA_RAS_BLOCK__GFX_TD_INDEX_START,
308	TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
309	TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
310	TA_RAS_BLOCK__GFX_TD_CS_FIFO,
311	TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
312	/* EA (3 sub-ranges)*/
313	TA_RAS_BLOCK__GFX_EA_INDEX_START,
314	/* EA range 0*/
315	TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
316	TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
317	TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
318	TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
319	TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
320	TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
321	TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
322	TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
323	TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
324	TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
325	/* EA range 1*/
326	TA_RAS_BLOCK__GFX_EA_INDEX1_START,
327	TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
328	TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
329	TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
330	TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
331	TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
332	TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
333	TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
334	TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
335	/* EA range 2*/
336	TA_RAS_BLOCK__GFX_EA_INDEX2_START,
337	TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
338	TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
339	TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
340	TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
341	TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
342	TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
343	/* UTC VM L2 bank*/
344	TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
345	/* UTC VM walker*/
346	TA_RAS_BLOCK__UTC_VML2_WALKER,
347	/* UTC ATC L2 2MB cache*/
348	TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
349	/* UTC ATC L2 4KB cache*/
350	TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
351	TA_RAS_BLOCK__GFX_MAX
352	};
353
354	struct ras_gfx_subblock {
355	unsigned char *name;
356	int ta_subblock;
357	int hw_supported_error_type;
358	int sw_supported_error_type;
359	};
360
361	#define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
362	[AMDGPU_RAS_BLOCK__##subblock] = { \
363	#subblock, \
364	TA_RAS_BLOCK__##subblock, \
365	((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
366	(((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
367	}
368
369	static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
370	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
371	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
372	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
373	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
374	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
375	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
376	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
377	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
378	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
379	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
380	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
381	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
382	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
383	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
384	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
385	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
386	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
387	0),
388	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
389	0),
390	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
391	AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
392	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
393	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
394	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
395	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
396	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
397	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
398	0, 0),
399	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
400	0),
401	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
402	0, 0),
403	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
404	0),
405	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
406	0, 0),
407	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
408	0),
409	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
410	1),
411	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
412	0, 0, 0),
413	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
414	0),
415	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
416	0),
417	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
418	0),
419	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
420	0),
421	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
422	0),
423	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
424	0, 0),
425	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
426	0),
427	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
428	0),
429	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
430	0, 0, 0),
431	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
432	0),
433	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
434	0),
435	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
436	0),
437	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
438	0),
439	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
440	0),
441	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
442	0, 0),
443	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
444	0),
445	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
446	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
447	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
448	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
449	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
450	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
451	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
452	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
453	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
454	1),
455	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
456	1),
457	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
458	1),
459	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
460	0),
461	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
462	0),
463	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
464	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
465	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
466	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
467	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
468	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
469	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
470	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
471	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
472	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
473	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
474	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
475	0),
476	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
477	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
478	0),
479	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
480	0, 0),
481	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
482	0),
483	AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
484	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
485	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
486	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
487	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
488	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
489	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
490	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
491	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
492	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
493	AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
494	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
495	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
496	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
497	AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
498	AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
499	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
500	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
501	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
502	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
503	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
504	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
505	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
506	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
507	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
508	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
509	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
510	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
511	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
512	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
513	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
514	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
515	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
516	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
517	};
518
519	static const struct soc15_reg_golden golden_settings_gc_9_0[] =
520	{
521	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
522	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
523	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
524	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
525	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
526	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
527	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
528	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
529	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
530	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
531	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
532	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
533	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
534	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
535	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
536	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
537	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
538	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
539	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
540	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
541	};
542
543	static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
544	{
545	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
546	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
547	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
548	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
549	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
550	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
551	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
552	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
553	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
554	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
555	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
556	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
557	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
558	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
559	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
560	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
561	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
562	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
563	};
564
565	static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
566	{
567	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
568	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
569	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
570	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
571	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
572	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
573	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
574	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
575	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
576	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
577	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
578	};
579
580	static const struct soc15_reg_golden golden_settings_gc_9_1[] =
581	{
582	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
583	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
584	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
585	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
586	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
587	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
588	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
589	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
590	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
591	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
592	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
593	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
594	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
595	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
596	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
597	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
598	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
599	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
600	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
601	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
602	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
603	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
604	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
605	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
606	};
607
608	static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
609	{
610	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
611	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
612	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
613	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
614	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
615	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
616	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
617	};
618
619	static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
620	{
621	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
622	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
623	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
624	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
625	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
626	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
627	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
628	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
629	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
630	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
631	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
632	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
633	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
634	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
635	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
636	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
637	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
638	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
639	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
640	};
641
642	static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
643	{
644	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
645	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
646	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
647	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
648	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
649	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
650	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
651	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
652	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
653	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
654	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
655	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
656	};
657
658	static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
659	{
660	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
661	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
662	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
663	};
664
665	static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
666	{
667	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
668	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
669	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
670	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
671	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
672	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
673	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
674	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
675	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
676	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
677	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
678	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
679	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
680	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
681	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
682	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
683	};
684
685	static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
686	{
687	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
688	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
689	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
690	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
691	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
692	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
693	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
694	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
695	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
696	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
697	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
698	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
699	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
700	};
701
702	static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
703	{
704	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
705	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
706	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
707	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
708	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
709	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
710	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
711	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
712	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
713	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
714	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
715	};
716
717	static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
718	{SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
719	{SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
720	};
721
722	static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
723	{
724	mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
725	mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
726	mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
727	mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
728	mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
729	mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
730	mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
731	mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
732	};
733
734	static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
735	{
736	mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
737	mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
738	mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
739	mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
740	mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
741	mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
742	mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
743	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
744	};
745
746	#define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
747	#define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
748	#define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
749	#define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
750
751	static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
752	static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
753	static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
754	static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
755	static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
756	struct amdgpu_cu_info *cu_info);
757	static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
758	static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
759	static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
760	static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
761	void *ras_error_status);
762	static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
763	void *inject_if, uint32_t instance_mask);
764	static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
765	static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
766	unsigned int vmid);
767
768	static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
769	uint64_t queue_mask)
770	{
771	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
772	amdgpu_ring_write(ring: kiq_ring,
773	PACKET3_SET_RESOURCES_VMID_MASK(0) |
774	/* vmid_mask:0* queue_type:0 (KIQ) */
775	PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
776	amdgpu_ring_write(ring: kiq_ring,
777	lower_32_bits(queue_mask)); /* queue mask lo */
778	amdgpu_ring_write(ring: kiq_ring,
779	upper_32_bits(queue_mask)); /* queue mask hi */
780	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gws mask lo */
781	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gws mask hi */
782	amdgpu_ring_write(ring: kiq_ring, v: 0); /* oac mask */
783	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gds heap base:0, gds heap size:0 */
784	}
785
786	static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
787	struct amdgpu_ring *ring)
788	{
789	uint64_t mqd_addr = amdgpu_bo_gpu_offset(bo: ring->mqd_obj);
790	uint64_t wptr_addr = ring->wptr_gpu_addr;
791	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
792
793	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
794	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
795	amdgpu_ring_write(ring: kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
796	PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
797	PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
798	PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
799	PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
800	PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
801	/*queue_type: normal compute queue */
802	PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
803	/* alloc format: all_on_one_pipe */
804	PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
805	PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
806	/* num_queues: must be 1 */
807	PACKET3_MAP_QUEUES_NUM_QUEUES(1));
808	amdgpu_ring_write(ring: kiq_ring,
809	PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
810	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(mqd_addr));
811	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(mqd_addr));
812	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(wptr_addr));
813	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(wptr_addr));
814	}
815
816	static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
817	struct amdgpu_ring *ring,
818	enum amdgpu_unmap_queues_action action,
819	u64 gpu_addr, u64 seq)
820	{
821	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
822
823	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
824	amdgpu_ring_write(ring: kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
825	PACKET3_UNMAP_QUEUES_ACTION(action) |
826	PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
827	PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
828	PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
829	amdgpu_ring_write(ring: kiq_ring,
830	PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
831
832	if (action == PREEMPT_QUEUES_NO_UNMAP) {
833	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
834	amdgpu_ring_write(ring: kiq_ring, v: 0);
835	amdgpu_ring_write(ring: kiq_ring, v: 0);
836
837	} else {
838	amdgpu_ring_write(ring: kiq_ring, v: 0);
839	amdgpu_ring_write(ring: kiq_ring, v: 0);
840	amdgpu_ring_write(ring: kiq_ring, v: 0);
841	}
842	}
843
844	static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
845	struct amdgpu_ring *ring,
846	u64 addr,
847	u64 seq)
848	{
849	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
850
851	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
852	amdgpu_ring_write(ring: kiq_ring,
853	PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
854	PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
855	PACKET3_QUERY_STATUS_COMMAND(2));
856	/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
857	amdgpu_ring_write(ring: kiq_ring,
858	PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
859	PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
860	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(addr));
861	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(addr));
862	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(seq));
863	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(seq));
864	}
865
866	static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
867	uint16_t pasid, uint32_t flush_type,
868	bool all_hub)
869	{
870	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
871	amdgpu_ring_write(ring: kiq_ring,
872	PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
873	PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
874	PACKET3_INVALIDATE_TLBS_PASID(pasid) |
875	PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
876	}
877
878	static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
879	.kiq_set_resources = gfx_v9_0_kiq_set_resources,
880	.kiq_map_queues = gfx_v9_0_kiq_map_queues,
881	.kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
882	.kiq_query_status = gfx_v9_0_kiq_query_status,
883	.kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
884	.set_resources_size = 8,
885	.map_queues_size = 7,
886	.unmap_queues_size = 6,
887	.query_status_size = 7,
888	.invalidate_tlbs_size = 2,
889	};
890
891	static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
892	{
893	adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
894	}
895
896	static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
897	{
898	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
899	case IP_VERSION(9, 0, 1):
900	soc15_program_register_sequence(adev,
901	registers: golden_settings_gc_9_0,
902	ARRAY_SIZE(golden_settings_gc_9_0));
903	soc15_program_register_sequence(adev,
904	registers: golden_settings_gc_9_0_vg10,
905	ARRAY_SIZE(golden_settings_gc_9_0_vg10));
906	break;
907	case IP_VERSION(9, 2, 1):
908	soc15_program_register_sequence(adev,
909	registers: golden_settings_gc_9_2_1,
910	ARRAY_SIZE(golden_settings_gc_9_2_1));
911	soc15_program_register_sequence(adev,
912	registers: golden_settings_gc_9_2_1_vg12,
913	ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
914	break;
915	case IP_VERSION(9, 4, 0):
916	soc15_program_register_sequence(adev,
917	registers: golden_settings_gc_9_0,
918	ARRAY_SIZE(golden_settings_gc_9_0));
919	soc15_program_register_sequence(adev,
920	registers: golden_settings_gc_9_0_vg20,
921	ARRAY_SIZE(golden_settings_gc_9_0_vg20));
922	break;
923	case IP_VERSION(9, 4, 1):
924	soc15_program_register_sequence(adev,
925	registers: golden_settings_gc_9_4_1_arct,
926	ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
927	break;
928	case IP_VERSION(9, 2, 2):
929	case IP_VERSION(9, 1, 0):
930	soc15_program_register_sequence(adev, registers: golden_settings_gc_9_1,
931	ARRAY_SIZE(golden_settings_gc_9_1));
932	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
933	soc15_program_register_sequence(adev,
934	registers: golden_settings_gc_9_1_rv2,
935	ARRAY_SIZE(golden_settings_gc_9_1_rv2));
936	else
937	soc15_program_register_sequence(adev,
938	registers: golden_settings_gc_9_1_rv1,
939	ARRAY_SIZE(golden_settings_gc_9_1_rv1));
940	break;
941	case IP_VERSION(9, 3, 0):
942	soc15_program_register_sequence(adev,
943	registers: golden_settings_gc_9_1_rn,
944	ARRAY_SIZE(golden_settings_gc_9_1_rn));
945	return; /* for renoir, don't need common goldensetting */
946	case IP_VERSION(9, 4, 2):
947	gfx_v9_4_2_init_golden_registers(adev,
948	die_id: adev->smuio.funcs->get_die_id(adev));
949	break;
950	default:
951	break;
952	}
953
954	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1)) &&
955	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 2)))
956	soc15_program_register_sequence(adev, registers: golden_settings_gc_9_x_common,
957	array_size: (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
958	}
959
960	static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
961	bool wc, uint32_t reg, uint32_t val)
962	{
963	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
964	amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
965	WRITE_DATA_DST_SEL(0) |
966	(wc ? WR_CONFIRM : 0));
967	amdgpu_ring_write(ring, v: reg);
968	amdgpu_ring_write(ring, v: 0);
969	amdgpu_ring_write(ring, v: val);
970	}
971
972	static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
973	int mem_space, int opt, uint32_t addr0,
974	uint32_t addr1, uint32_t ref, uint32_t mask,
975	uint32_t inv)
976	{
977	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
978	amdgpu_ring_write(ring,
979	/* memory (1) or register (0) */
980	v: (WAIT_REG_MEM_MEM_SPACE(mem_space) |
981	WAIT_REG_MEM_OPERATION(opt) | /* wait */
982	WAIT_REG_MEM_FUNCTION(3) | /* equal */
983	WAIT_REG_MEM_ENGINE(eng_sel)));
984
985	if (mem_space)
986	BUG_ON(addr0 & 0x3); /* Dword align */
987	amdgpu_ring_write(ring, v: addr0);
988	amdgpu_ring_write(ring, v: addr1);
989	amdgpu_ring_write(ring, v: ref);
990	amdgpu_ring_write(ring, v: mask);
991	amdgpu_ring_write(ring, v: inv); /* poll interval */
992	}
993
994	static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
995	{
996	struct amdgpu_device *adev = ring->adev;
997	uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
998	uint32_t tmp = 0;
999	unsigned i;
1000	int r;
1001
1002	WREG32(scratch, 0xCAFEDEAD);
1003	r = amdgpu_ring_alloc(ring, ndw: 3);
1004	if (r)
1005	return r;
1006
1007	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1008	amdgpu_ring_write(ring, v: scratch - PACKET3_SET_UCONFIG_REG_START);
1009	amdgpu_ring_write(ring, v: 0xDEADBEEF);
1010	amdgpu_ring_commit(ring);
1011
1012	for (i = 0; i < adev->usec_timeout; i++) {
1013	tmp = RREG32(scratch);
1014	if (tmp == 0xDEADBEEF)
1015	break;
1016	udelay(1);
1017	}
1018
1019	if (i >= adev->usec_timeout)
1020	r = -ETIMEDOUT;
1021	return r;
1022	}
1023
1024	static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1025	{
1026	struct amdgpu_device *adev = ring->adev;
1027	struct amdgpu_ib ib;
1028	struct dma_fence *f = NULL;
1029
1030	unsigned index;
1031	uint64_t gpu_addr;
1032	uint32_t tmp;
1033	long r;
1034
1035	r = amdgpu_device_wb_get(adev, wb: &index);
1036	if (r)
1037	return r;
1038
1039	gpu_addr = adev->wb.gpu_addr + (index * 4);
1040	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1041	memset(&ib, 0, sizeof(ib));
1042
1043	r = amdgpu_ib_get(adev, NULL, size: 20, pool: AMDGPU_IB_POOL_DIRECT, ib: &ib);
1044	if (r)
1045	goto err1;
1046
1047	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1048	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1049	ib.ptr[2] = lower_32_bits(gpu_addr);
1050	ib.ptr[3] = upper_32_bits(gpu_addr);
1051	ib.ptr[4] = 0xDEADBEEF;
1052	ib.length_dw = 5;
1053
1054	r = amdgpu_ib_schedule(ring, num_ibs: 1, ibs: &ib, NULL, f: &f);
1055	if (r)
1056	goto err2;
1057
1058	r = dma_fence_wait_timeout(f, intr: false, timeout);
1059	if (r == 0) {
1060	r = -ETIMEDOUT;
1061	goto err2;
1062	} else if (r < 0) {
1063	goto err2;
1064	}
1065
1066	tmp = adev->wb.wb[index];
1067	if (tmp == 0xDEADBEEF)
1068	r = 0;
1069	else
1070	r = -EINVAL;
1071
1072	err2:
1073	amdgpu_ib_free(adev, ib: &ib, NULL);
1074	dma_fence_put(fence: f);
1075	err1:
1076	amdgpu_device_wb_free(adev, wb: index);
1077	return r;
1078	}
1079
1080
1081	static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1082	{
1083	amdgpu_ucode_release(fw: &adev->gfx.pfp_fw);
1084	amdgpu_ucode_release(fw: &adev->gfx.me_fw);
1085	amdgpu_ucode_release(fw: &adev->gfx.ce_fw);
1086	amdgpu_ucode_release(fw: &adev->gfx.rlc_fw);
1087	amdgpu_ucode_release(fw: &adev->gfx.mec_fw);
1088	amdgpu_ucode_release(fw: &adev->gfx.mec2_fw);
1089
1090	kfree(objp: adev->gfx.rlc.register_list_format);
1091	}
1092
1093	static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1094	{
1095	adev->gfx.me_fw_write_wait = false;
1096	adev->gfx.mec_fw_write_wait = false;
1097
1098	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1)) &&
1099	((adev->gfx.mec_fw_version < 0x000001a5) ||
1100	(adev->gfx.mec_feature_version < 46) ||
1101	(adev->gfx.pfp_fw_version < 0x000000b7) ||
1102	(adev->gfx.pfp_feature_version < 46)))
1103	DRM_WARN_ONCE("CP firmware version too old, please update!");
1104
1105	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1106	case IP_VERSION(9, 0, 1):
1107	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1108	(adev->gfx.me_feature_version >= 42) &&
1109	(adev->gfx.pfp_fw_version >= 0x000000b1) &&
1110	(adev->gfx.pfp_feature_version >= 42))
1111	adev->gfx.me_fw_write_wait = true;
1112
1113	if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1114	(adev->gfx.mec_feature_version >= 42))
1115	adev->gfx.mec_fw_write_wait = true;
1116	break;
1117	case IP_VERSION(9, 2, 1):
1118	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1119	(adev->gfx.me_feature_version >= 44) &&
1120	(adev->gfx.pfp_fw_version >= 0x000000b2) &&
1121	(adev->gfx.pfp_feature_version >= 44))
1122	adev->gfx.me_fw_write_wait = true;
1123
1124	if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1125	(adev->gfx.mec_feature_version >= 44))
1126	adev->gfx.mec_fw_write_wait = true;
1127	break;
1128	case IP_VERSION(9, 4, 0):
1129	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1130	(adev->gfx.me_feature_version >= 44) &&
1131	(adev->gfx.pfp_fw_version >= 0x000000b2) &&
1132	(adev->gfx.pfp_feature_version >= 44))
1133	adev->gfx.me_fw_write_wait = true;
1134
1135	if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1136	(adev->gfx.mec_feature_version >= 44))
1137	adev->gfx.mec_fw_write_wait = true;
1138	break;
1139	case IP_VERSION(9, 1, 0):
1140	case IP_VERSION(9, 2, 2):
1141	if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1142	(adev->gfx.me_feature_version >= 42) &&
1143	(adev->gfx.pfp_fw_version >= 0x000000b1) &&
1144	(adev->gfx.pfp_feature_version >= 42))
1145	adev->gfx.me_fw_write_wait = true;
1146
1147	if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1148	(adev->gfx.mec_feature_version >= 42))
1149	adev->gfx.mec_fw_write_wait = true;
1150	break;
1151	default:
1152	adev->gfx.me_fw_write_wait = true;
1153	adev->gfx.mec_fw_write_wait = true;
1154	break;
1155	}
1156	}
1157
1158	struct amdgpu_gfxoff_quirk {
1159	u16 chip_vendor;
1160	u16 chip_device;
1161	u16 subsys_vendor;
1162	u16 subsys_device;
1163	u8 revision;
1164	};
1165
1166	static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1167	/* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1168	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1169	/* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1170	{ 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1171	/* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1172	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1173	/* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1174	{ 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1175	{ 0, 0, 0, 0, 0 },
1176	};
1177
1178	static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1179	{
1180	const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1181
1182	while (p && p->chip_device != 0) {
1183	if (pdev->vendor == p->chip_vendor &&
1184	pdev->device == p->chip_device &&
1185	pdev->subsystem_vendor == p->subsys_vendor &&
1186	pdev->subsystem_device == p->subsys_device &&
1187	pdev->revision == p->revision) {
1188	return true;
1189	}
1190	++p;
1191	}
1192	return false;
1193	}
1194
1195	static bool is_raven_kicker(struct amdgpu_device *adev)
1196	{
1197	if (adev->pm.fw_version >= 0x41e2b)
1198	return true;
1199	else
1200	return false;
1201	}
1202
1203	static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1204	{
1205	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 3, 0)) &&
1206	(adev->gfx.me_fw_version >= 0x000000a5) &&
1207	(adev->gfx.me_feature_version >= 52))
1208	return true;
1209	else
1210	return false;
1211	}
1212
1213	static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1214	{
1215	if (gfx_v9_0_should_disable_gfxoff(pdev: adev->pdev))
1216	adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1217
1218	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1219	case IP_VERSION(9, 0, 1):
1220	case IP_VERSION(9, 2, 1):
1221	case IP_VERSION(9, 4, 0):
1222	break;
1223	case IP_VERSION(9, 2, 2):
1224	case IP_VERSION(9, 1, 0):
1225	if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1226	(adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1227	((!is_raven_kicker(adev) &&
1228	adev->gfx.rlc_fw_version < 531) ||
1229	(adev->gfx.rlc_feature_version < 1) ||
1230	!adev->gfx.rlc.is_rlc_v2_1))
1231	adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1232
1233	if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1234	adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1235	AMD_PG_SUPPORT_CP |
1236	AMD_PG_SUPPORT_RLC_SMU_HS;
1237	break;
1238	case IP_VERSION(9, 3, 0):
1239	if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1240	adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1241	AMD_PG_SUPPORT_CP |
1242	AMD_PG_SUPPORT_RLC_SMU_HS;
1243	break;
1244	default:
1245	break;
1246	}
1247	}
1248
1249	static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1250	char *chip_name)
1251	{
1252	char fw_name[30];
1253	int err;
1254
1255	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_pfp.bin", chip_name);
1256	err = amdgpu_ucode_request(adev, fw: &adev->gfx.pfp_fw, fw_name);
1257	if (err)
1258	goto out;
1259	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_PFP);
1260
1261	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_me.bin", chip_name);
1262	err = amdgpu_ucode_request(adev, fw: &adev->gfx.me_fw, fw_name);
1263	if (err)
1264	goto out;
1265	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_ME);
1266
1267	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_ce.bin", chip_name);
1268	err = amdgpu_ucode_request(adev, fw: &adev->gfx.ce_fw, fw_name);
1269	if (err)
1270	goto out;
1271	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_CE);
1272
1273	out:
1274	if (err) {
1275	amdgpu_ucode_release(fw: &adev->gfx.pfp_fw);
1276	amdgpu_ucode_release(fw: &adev->gfx.me_fw);
1277	amdgpu_ucode_release(fw: &adev->gfx.ce_fw);
1278	}
1279	return err;
1280	}
1281
1282	static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1283	char *chip_name)
1284	{
1285	char fw_name[30];
1286	int err;
1287	const struct rlc_firmware_header_v2_0 *rlc_hdr;
1288	uint16_t version_major;
1289	uint16_t version_minor;
1290	uint32_t smu_version;
1291
1292	/*
1293	* For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1294	* instead of picasso_rlc.bin.
1295	* Judgment method:
1296	* PCO AM4: revision >= 0xC8 && revision <= 0xCF
1297	* or revision >= 0xD8 && revision <= 0xDF
1298	* otherwise is PCO FP5
1299	*/
1300	if (!strcmp(chip_name, "picasso") &&
1301	(((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1302	((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1303	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_rlc_am4.bin", chip_name);
1304	else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, smu_version: &smu_version) == 0) &&
1305	(smu_version >= 0x41e2b))
1306	/**
1307	*SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1308	*/
1309	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_kicker_rlc.bin", chip_name);
1310	else
1311	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_rlc.bin", chip_name);
1312	err = amdgpu_ucode_request(adev, fw: &adev->gfx.rlc_fw, fw_name);
1313	if (err)
1314	goto out;
1315	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1316
1317	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1318	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1319	err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1320	out:
1321	if (err)
1322	amdgpu_ucode_release(fw: &adev->gfx.rlc_fw);
1323
1324	return err;
1325	}
1326
1327	static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1328	{
1329	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2) ||
1330	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
1331	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 3, 0))
1332	return false;
1333
1334	return true;
1335	}
1336
1337	static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1338	char *chip_name)
1339	{
1340	char fw_name[30];
1341	int err;
1342
1343	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1344	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_sjt_mec.bin", chip_name);
1345	else
1346	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mec.bin", chip_name);
1347
1348	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec_fw, fw_name);
1349	if (err)
1350	goto out;
1351	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC1);
1352	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC1_JT);
1353
1354	if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1355	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1356	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_sjt_mec2.bin", chip_name);
1357	else
1358	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mec2.bin", chip_name);
1359
1360	/* ignore failures to load */
1361	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec2_fw, fw_name);
1362	if (!err) {
1363	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC2);
1364	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC2_JT);
1365	} else {
1366	err = 0;
1367	amdgpu_ucode_release(fw: &adev->gfx.mec2_fw);
1368	}
1369	} else {
1370	adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1371	adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1372	}
1373
1374	gfx_v9_0_check_if_need_gfxoff(adev);
1375	gfx_v9_0_check_fw_write_wait(adev);
1376
1377	out:
1378	if (err)
1379	amdgpu_ucode_release(fw: &adev->gfx.mec_fw);
1380	return err;
1381	}
1382
1383	static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1384	{
1385	char ucode_prefix[30];
1386	int r;
1387
1388	DRM_DEBUG("\n");
1389	amdgpu_ucode_ip_version_decode(adev, block_type: GC_HWIP, ucode_prefix, len: sizeof(ucode_prefix));
1390
1391	/* No CPG in Arcturus */
1392	if (adev->gfx.num_gfx_rings) {
1393	r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name: ucode_prefix);
1394	if (r)
1395	return r;
1396	}
1397
1398	r = gfx_v9_0_init_rlc_microcode(adev, chip_name: ucode_prefix);
1399	if (r)
1400	return r;
1401
1402	r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name: ucode_prefix);
1403	if (r)
1404	return r;
1405
1406	return r;
1407	}
1408
1409	static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1410	{
1411	u32 count = 0;
1412	const struct cs_section_def *sect = NULL;
1413	const struct cs_extent_def *ext = NULL;
1414
1415	/* begin clear state */
1416	count += 2;
1417	/* context control state */
1418	count += 3;
1419
1420	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1421	for (ext = sect->section; ext->extent != NULL; ++ext) {
1422	if (sect->id == SECT_CONTEXT)
1423	count += 2 + ext->reg_count;
1424	else
1425	return 0;
1426	}
1427	}
1428
1429	/* end clear state */
1430	count += 2;
1431	/* clear state */
1432	count += 2;
1433
1434	return count;
1435	}
1436
1437	static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1438	volatile u32 *buffer)
1439	{
1440	u32 count = 0, i;
1441	const struct cs_section_def *sect = NULL;
1442	const struct cs_extent_def *ext = NULL;
1443
1444	if (adev->gfx.rlc.cs_data == NULL)
1445	return;
1446	if (buffer == NULL)
1447	return;
1448
1449	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1450	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1451
1452	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1453	buffer[count++] = cpu_to_le32(0x80000000);
1454	buffer[count++] = cpu_to_le32(0x80000000);
1455
1456	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1457	for (ext = sect->section; ext->extent != NULL; ++ext) {
1458	if (sect->id == SECT_CONTEXT) {
1459	buffer[count++] =
1460	cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1461	buffer[count++] = cpu_to_le32(ext->reg_index -
1462	PACKET3_SET_CONTEXT_REG_START);
1463	for (i = 0; i < ext->reg_count; i++)
1464	buffer[count++] = cpu_to_le32(ext->extent[i]);
1465	} else {
1466	return;
1467	}
1468	}
1469	}
1470
1471	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1472	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1473
1474	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1475	buffer[count++] = cpu_to_le32(0);
1476	}
1477
1478	static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1479	{
1480	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1481	uint32_t pg_always_on_cu_num = 2;
1482	uint32_t always_on_cu_num;
1483	uint32_t i, j, k;
1484	uint32_t mask, cu_bitmap, counter;
1485
1486	if (adev->flags & AMD_IS_APU)
1487	always_on_cu_num = 4;
1488	else if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 2, 1))
1489	always_on_cu_num = 8;
1490	else
1491	always_on_cu_num = 12;
1492
1493	mutex_lock(&adev->grbm_idx_mutex);
1494	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1495	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1496	mask = 1;
1497	cu_bitmap = 0;
1498	counter = 0;
1499	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1500
1501	for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1502	if (cu_info->bitmap[0][i][j] & mask) {
1503	if (counter == pg_always_on_cu_num)
1504	WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1505	if (counter < always_on_cu_num)
1506	cu_bitmap |= mask;
1507	else
1508	break;
1509	counter++;
1510	}
1511	mask <<= 1;
1512	}
1513
1514	WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1515	cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1516	}
1517	}
1518	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1519	mutex_unlock(lock: &adev->grbm_idx_mutex);
1520	}
1521
1522	static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1523	{
1524	uint32_t data;
1525
1526	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1527	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1528	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1529	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1530	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1531
1532	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1533	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1534
1535	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1536	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1537
1538	mutex_lock(&adev->grbm_idx_mutex);
1539	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1540	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1541	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1542
1543	/* set mmRLC_LB_PARAMS = 0x003F_1006 */
1544	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1545	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1546	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1547	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1548
1549	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1550	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1551	data &= 0x0000FFFF;
1552	data |= 0x00C00000;
1553	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1554
1555	/*
1556	* RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1557	* programmed in gfx_v9_0_init_always_on_cu_mask()
1558	*/
1559
1560	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1561	* but used for RLC_LB_CNTL configuration */
1562	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1563	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1564	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1565	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1566	mutex_unlock(lock: &adev->grbm_idx_mutex);
1567
1568	gfx_v9_0_init_always_on_cu_mask(adev);
1569	}
1570
1571	static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1572	{
1573	uint32_t data;
1574
1575	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1576	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1577	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1578	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1579	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1580
1581	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1582	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1583
1584	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1585	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1586
1587	mutex_lock(&adev->grbm_idx_mutex);
1588	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1589	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1590	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1591
1592	/* set mmRLC_LB_PARAMS = 0x003F_1006 */
1593	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1594	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1595	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1596	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1597
1598	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1599	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1600	data &= 0x0000FFFF;
1601	data |= 0x00C00000;
1602	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1603
1604	/*
1605	* RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1606	* programmed in gfx_v9_0_init_always_on_cu_mask()
1607	*/
1608
1609	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1610	* but used for RLC_LB_CNTL configuration */
1611	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1612	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1613	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1614	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1615	mutex_unlock(lock: &adev->grbm_idx_mutex);
1616
1617	gfx_v9_0_init_always_on_cu_mask(adev);
1618	}
1619
1620	static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1621	{
1622	WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1623	}
1624
1625	static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1626	{
1627	if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1628	return 5;
1629	else
1630	return 4;
1631	}
1632
1633	static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1634	{
1635	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1636
1637	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1638	reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1639	reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1640	reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1641	reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1642	reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1643	reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1644	reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1645	adev->gfx.rlc.rlcg_reg_access_supported = true;
1646	}
1647
1648	static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1649	{
1650	const struct cs_section_def *cs_data;
1651	int r;
1652
1653	adev->gfx.rlc.cs_data = gfx9_cs_data;
1654
1655	cs_data = adev->gfx.rlc.cs_data;
1656
1657	if (cs_data) {
1658	/* init clear state block */
1659	r = amdgpu_gfx_rlc_init_csb(adev);
1660	if (r)
1661	return r;
1662	}
1663
1664	if (adev->flags & AMD_IS_APU) {
1665	/* TODO: double check the cp_table_size for RV */
1666	adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1667	r = amdgpu_gfx_rlc_init_cpt(adev);
1668	if (r)
1669	return r;
1670	}
1671
1672	return 0;
1673	}
1674
1675	static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1676	{
1677	amdgpu_bo_free_kernel(bo: &adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1678	amdgpu_bo_free_kernel(bo: &adev->gfx.mec.mec_fw_obj, NULL, NULL);
1679	}
1680
1681	static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1682	{
1683	int r;
1684	u32 *hpd;
1685	const __le32 *fw_data;
1686	unsigned fw_size;
1687	u32 *fw;
1688	size_t mec_hpd_size;
1689
1690	const struct gfx_firmware_header_v1_0 *mec_hdr;
1691
1692	bitmap_zero(dst: adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1693
1694	/* take ownership of the relevant compute queues */
1695	amdgpu_gfx_compute_queue_acquire(adev);
1696	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1697	if (mec_hpd_size) {
1698	r = amdgpu_bo_create_reserved(adev, size: mec_hpd_size, PAGE_SIZE,
1699	AMDGPU_GEM_DOMAIN_VRAM |
1700	AMDGPU_GEM_DOMAIN_GTT,
1701	bo_ptr: &adev->gfx.mec.hpd_eop_obj,
1702	gpu_addr: &adev->gfx.mec.hpd_eop_gpu_addr,
1703	cpu_addr: (void **)&hpd);
1704	if (r) {
1705	dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1706	gfx_v9_0_mec_fini(adev);
1707	return r;
1708	}
1709
1710	memset(hpd, 0, mec_hpd_size);
1711
1712	amdgpu_bo_kunmap(bo: adev->gfx.mec.hpd_eop_obj);
1713	amdgpu_bo_unreserve(bo: adev->gfx.mec.hpd_eop_obj);
1714	}
1715
1716	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1717
1718	fw_data = (const __le32 *)
1719	(adev->gfx.mec_fw->data +
1720	le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1721	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1722
1723	r = amdgpu_bo_create_reserved(adev, size: mec_hdr->header.ucode_size_bytes,
1724	PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1725	bo_ptr: &adev->gfx.mec.mec_fw_obj,
1726	gpu_addr: &adev->gfx.mec.mec_fw_gpu_addr,
1727	cpu_addr: (void **)&fw);
1728	if (r) {
1729	dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1730	gfx_v9_0_mec_fini(adev);
1731	return r;
1732	}
1733
1734	memcpy(fw, fw_data, fw_size);
1735
1736	amdgpu_bo_kunmap(bo: adev->gfx.mec.mec_fw_obj);
1737	amdgpu_bo_unreserve(bo: adev->gfx.mec.mec_fw_obj);
1738
1739	return 0;
1740	}
1741
1742	static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1743	{
1744	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1745	(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1746	(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1747	(address << SQ_IND_INDEX__INDEX__SHIFT) |
1748	(SQ_IND_INDEX__FORCE_READ_MASK));
1749	return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1750	}
1751
1752	static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1753	uint32_t wave, uint32_t thread,
1754	uint32_t regno, uint32_t num, uint32_t *out)
1755	{
1756	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1757	(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1758	(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1759	(regno << SQ_IND_INDEX__INDEX__SHIFT) |
1760	(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1761	(SQ_IND_INDEX__FORCE_READ_MASK) |
1762	(SQ_IND_INDEX__AUTO_INCR_MASK));
1763	while (num--)
1764	*(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1765	}
1766
1767	static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1768	{
1769	/* type 1 wave data */
1770	dst[(*no_fields)++] = 1;
1771	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1772	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1773	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1774	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1775	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1776	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1777	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1778	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1779	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1780	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1781	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1782	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1783	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1784	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1785	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1786	}
1787
1788	static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1789	uint32_t wave, uint32_t start,
1790	uint32_t size, uint32_t *dst)
1791	{
1792	wave_read_regs(
1793	adev, simd, wave, thread: 0,
1794	regno: start + SQIND_WAVE_SGPRS_OFFSET, num: size, out: dst);
1795	}
1796
1797	static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1798	uint32_t wave, uint32_t thread,
1799	uint32_t start, uint32_t size,
1800	uint32_t *dst)
1801	{
1802	wave_read_regs(
1803	adev, simd, wave, thread,
1804	regno: start + SQIND_WAVE_VGPRS_OFFSET, num: size, out: dst);
1805	}
1806
1807	static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1808	u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1809	{
1810	soc15_grbm_select(adev, me, pipe, queue: q, vmid: vm, xcc_id: 0);
1811	}
1812
1813	static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
1814	.get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
1815	.select_se_sh = &gfx_v9_0_select_se_sh,
1816	.read_wave_data = &gfx_v9_0_read_wave_data,
1817	.read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
1818	.read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
1819	.select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
1820	};
1821
1822	const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = {
1823	.ras_error_inject = &gfx_v9_0_ras_error_inject,
1824	.query_ras_error_count = &gfx_v9_0_query_ras_error_count,
1825	.reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
1826	};
1827
1828	static struct amdgpu_gfx_ras gfx_v9_0_ras = {
1829	.ras_block = {
1830	.hw_ops = &gfx_v9_0_ras_ops,
1831	},
1832	};
1833
1834	static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
1835	{
1836	u32 gb_addr_config;
1837	int err;
1838
1839	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1840	case IP_VERSION(9, 0, 1):
1841	adev->gfx.config.max_hw_contexts = 8;
1842	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1843	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1844	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1845	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1846	gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
1847	break;
1848	case IP_VERSION(9, 2, 1):
1849	adev->gfx.config.max_hw_contexts = 8;
1850	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1851	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1852	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1853	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1854	gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
1855	DRM_INFO("fix gfx.config for vega12\n");
1856	break;
1857	case IP_VERSION(9, 4, 0):
1858	adev->gfx.ras = &gfx_v9_0_ras;
1859	adev->gfx.config.max_hw_contexts = 8;
1860	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1861	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1862	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1863	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1864	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1865	gb_addr_config &= ~0xf3e777ff;
1866	gb_addr_config |= 0x22014042;
1867	/* check vbios table if gpu info is not available */
1868	err = amdgpu_atomfirmware_get_gfx_info(adev);
1869	if (err)
1870	return err;
1871	break;
1872	case IP_VERSION(9, 2, 2):
1873	case IP_VERSION(9, 1, 0):
1874	adev->gfx.config.max_hw_contexts = 8;
1875	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1876	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1877	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1878	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1879	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1880	gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
1881	else
1882	gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
1883	break;
1884	case IP_VERSION(9, 4, 1):
1885	adev->gfx.ras = &gfx_v9_4_ras;
1886	adev->gfx.config.max_hw_contexts = 8;
1887	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1888	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1889	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1890	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1891	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1892	gb_addr_config &= ~0xf3e777ff;
1893	gb_addr_config |= 0x22014042;
1894	break;
1895	case IP_VERSION(9, 3, 0):
1896	adev->gfx.config.max_hw_contexts = 8;
1897	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1898	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1899	adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1900	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1901	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1902	gb_addr_config &= ~0xf3e777ff;
1903	gb_addr_config |= 0x22010042;
1904	break;
1905	case IP_VERSION(9, 4, 2):
1906	adev->gfx.ras = &gfx_v9_4_2_ras;
1907	adev->gfx.config.max_hw_contexts = 8;
1908	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1909	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1910	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1911	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1912	gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1913	gb_addr_config &= ~0xf3e777ff;
1914	gb_addr_config |= 0x22014042;
1915	/* check vbios table if gpu info is not available */
1916	err = amdgpu_atomfirmware_get_gfx_info(adev);
1917	if (err)
1918	return err;
1919	break;
1920	default:
1921	BUG();
1922	break;
1923	}
1924
1925	adev->gfx.config.gb_addr_config = gb_addr_config;
1926
1927	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
1928	REG_GET_FIELD(
1929	adev->gfx.config.gb_addr_config,
1930	GB_ADDR_CONFIG,
1931	NUM_PIPES);
1932
1933	adev->gfx.config.max_tile_pipes =
1934	adev->gfx.config.gb_addr_config_fields.num_pipes;
1935
1936	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
1937	REG_GET_FIELD(
1938	adev->gfx.config.gb_addr_config,
1939	GB_ADDR_CONFIG,
1940	NUM_BANKS);
1941	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
1942	REG_GET_FIELD(
1943	adev->gfx.config.gb_addr_config,
1944	GB_ADDR_CONFIG,
1945	MAX_COMPRESSED_FRAGS);
1946	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
1947	REG_GET_FIELD(
1948	adev->gfx.config.gb_addr_config,
1949	GB_ADDR_CONFIG,
1950	NUM_RB_PER_SE);
1951	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
1952	REG_GET_FIELD(
1953	adev->gfx.config.gb_addr_config,
1954	GB_ADDR_CONFIG,
1955	NUM_SHADER_ENGINES);
1956	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
1957	REG_GET_FIELD(
1958	adev->gfx.config.gb_addr_config,
1959	GB_ADDR_CONFIG,
1960	PIPE_INTERLEAVE_SIZE));
1961
1962	return 0;
1963	}
1964
1965	static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1966	int mec, int pipe, int queue)
1967	{
1968	unsigned irq_type;
1969	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1970	unsigned int hw_prio;
1971
1972	ring = &adev->gfx.compute_ring[ring_id];
1973
1974	/* mec0 is me1 */
1975	ring->me = mec + 1;
1976	ring->pipe = pipe;
1977	ring->queue = queue;
1978
1979	ring->ring_obj = NULL;
1980	ring->use_doorbell = true;
1981	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1982	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1983	+ (ring_id * GFX9_MEC_HPD_SIZE);
1984	ring->vm_hub = AMDGPU_GFXHUB(0);
1985	sprintf(buf: ring->name, fmt: "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1986
1987	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1988	+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1989	+ ring->pipe;
1990	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1991	AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
1992	/* type-2 packets are deprecated on MEC, use type-3 instead */
1993	return amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq, irq_type,
1994	hw_prio, NULL);
1995	}
1996
1997	static int gfx_v9_0_sw_init(void *handle)
1998	{
1999	int i, j, k, r, ring_id;
2000	int xcc_id = 0;
2001	struct amdgpu_ring *ring;
2002	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2003	unsigned int hw_prio;
2004
2005	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2006	case IP_VERSION(9, 0, 1):
2007	case IP_VERSION(9, 2, 1):
2008	case IP_VERSION(9, 4, 0):
2009	case IP_VERSION(9, 2, 2):
2010	case IP_VERSION(9, 1, 0):
2011	case IP_VERSION(9, 4, 1):
2012	case IP_VERSION(9, 3, 0):
2013	case IP_VERSION(9, 4, 2):
2014	adev->gfx.mec.num_mec = 2;
2015	break;
2016	default:
2017	adev->gfx.mec.num_mec = 1;
2018	break;
2019	}
2020
2021	adev->gfx.mec.num_pipe_per_mec = 4;
2022	adev->gfx.mec.num_queue_per_pipe = 8;
2023
2024	/* EOP Event */
2025	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, source: &adev->gfx.eop_irq);
2026	if (r)
2027	return r;
2028
2029	/* Privileged reg */
2030	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2031	source: &adev->gfx.priv_reg_irq);
2032	if (r)
2033	return r;
2034
2035	/* Privileged inst */
2036	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2037	source: &adev->gfx.priv_inst_irq);
2038	if (r)
2039	return r;
2040
2041	/* ECC error */
2042	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2043	source: &adev->gfx.cp_ecc_error_irq);
2044	if (r)
2045	return r;
2046
2047	/* FUE error */
2048	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2049	source: &adev->gfx.cp_ecc_error_irq);
2050	if (r)
2051	return r;
2052
2053	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2054
2055	if (adev->gfx.rlc.funcs) {
2056	if (adev->gfx.rlc.funcs->init) {
2057	r = adev->gfx.rlc.funcs->init(adev);
2058	if (r) {
2059	dev_err(adev->dev, "Failed to init rlc BOs!\n");
2060	return r;
2061	}
2062	}
2063	}
2064
2065	r = gfx_v9_0_mec_init(adev);
2066	if (r) {
2067	DRM_ERROR("Failed to init MEC BOs!\n");
2068	return r;
2069	}
2070
2071	/* set up the gfx ring */
2072	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2073	ring = &adev->gfx.gfx_ring[i];
2074	ring->ring_obj = NULL;
2075	if (!i)
2076	sprintf(buf: ring->name, fmt: "gfx");
2077	else
2078	sprintf(buf: ring->name, fmt: "gfx_%d", i);
2079	ring->use_doorbell = true;
2080	ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2081
2082	/* disable scheduler on the real ring */
2083	ring->no_scheduler = adev->gfx.mcbp;
2084	ring->vm_hub = AMDGPU_GFXHUB(0);
2085	r = amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq,
2086	irq_type: AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2087	hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
2088	if (r)
2089	return r;
2090	}
2091
2092	/* set up the software rings */
2093	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2094	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2095	ring = &adev->gfx.sw_gfx_ring[i];
2096	ring->ring_obj = NULL;
2097	sprintf(buf: ring->name, fmt: amdgpu_sw_ring_name(idx: i));
2098	ring->use_doorbell = true;
2099	ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2100	ring->is_sw_ring = true;
2101	hw_prio = amdgpu_sw_ring_priority(idx: i);
2102	ring->vm_hub = AMDGPU_GFXHUB(0);
2103	r = amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq,
2104	irq_type: AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2105	NULL);
2106	if (r)
2107	return r;
2108	ring->wptr = 0;
2109	}
2110
2111	/* init the muxer and add software rings */
2112	r = amdgpu_ring_mux_init(mux: &adev->gfx.muxer, ring: &adev->gfx.gfx_ring[0],
2113	GFX9_NUM_SW_GFX_RINGS);
2114	if (r) {
2115	DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2116	return r;
2117	}
2118	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2119	r = amdgpu_ring_mux_add_sw_ring(mux: &adev->gfx.muxer,
2120	ring: &adev->gfx.sw_gfx_ring[i]);
2121	if (r) {
2122	DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2123	return r;
2124	}
2125	}
2126	}
2127
2128	/* set up the compute queues - allocate horizontally across pipes */
2129	ring_id = 0;
2130	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2131	for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2132	for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2133	if (!amdgpu_gfx_is_mec_queue_enabled(adev, xcc_id: 0, mec: i,
2134	pipe: k, queue: j))
2135	continue;
2136
2137	r = gfx_v9_0_compute_ring_init(adev,
2138	ring_id,
2139	mec: i, pipe: k, queue: j);
2140	if (r)
2141	return r;
2142
2143	ring_id++;
2144	}
2145	}
2146	}
2147
2148	r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, xcc_id: 0);
2149	if (r) {
2150	DRM_ERROR("Failed to init KIQ BOs!\n");
2151	return r;
2152	}
2153
2154	r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2155	if (r)
2156	return r;
2157
2158	/* create MQD for all compute queues as wel as KIQ for SRIOV case */
2159	r = amdgpu_gfx_mqd_sw_init(adev, mqd_size: sizeof(struct v9_mqd_allocation), xcc_id: 0);
2160	if (r)
2161	return r;
2162
2163	adev->gfx.ce_ram_size = 0x8000;
2164
2165	r = gfx_v9_0_gpu_early_init(adev);
2166	if (r)
2167	return r;
2168
2169	if (amdgpu_gfx_ras_sw_init(adev)) {
2170	dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2171	return -EINVAL;
2172	}
2173
2174	return 0;
2175	}
2176
2177
2178	static int gfx_v9_0_sw_fini(void *handle)
2179	{
2180	int i;
2181	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2182
2183	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2184	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2185	amdgpu_ring_fini(ring: &adev->gfx.sw_gfx_ring[i]);
2186	amdgpu_ring_mux_fini(mux: &adev->gfx.muxer);
2187	}
2188
2189	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2190	amdgpu_ring_fini(ring: &adev->gfx.gfx_ring[i]);
2191	for (i = 0; i < adev->gfx.num_compute_rings; i++)
2192	amdgpu_ring_fini(ring: &adev->gfx.compute_ring[i]);
2193
2194	amdgpu_gfx_mqd_sw_fini(adev, xcc_id: 0);
2195	amdgpu_gfx_kiq_free_ring(ring: &adev->gfx.kiq[0].ring);
2196	amdgpu_gfx_kiq_fini(adev, xcc_id: 0);
2197
2198	gfx_v9_0_mec_fini(adev);
2199	amdgpu_bo_free_kernel(bo: &adev->gfx.rlc.clear_state_obj,
2200	gpu_addr: &adev->gfx.rlc.clear_state_gpu_addr,
2201	cpu_addr: (void **)&adev->gfx.rlc.cs_ptr);
2202	if (adev->flags & AMD_IS_APU) {
2203	amdgpu_bo_free_kernel(bo: &adev->gfx.rlc.cp_table_obj,
2204	gpu_addr: &adev->gfx.rlc.cp_table_gpu_addr,
2205	cpu_addr: (void **)&adev->gfx.rlc.cp_table_ptr);
2206	}
2207	gfx_v9_0_free_microcode(adev);
2208
2209	return 0;
2210	}
2211
2212
2213	static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2214	{
2215	/* TODO */
2216	}
2217
2218	void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2219	u32 instance, int xcc_id)
2220	{
2221	u32 data;
2222
2223	if (instance == 0xffffffff)
2224	data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2225	else
2226	data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2227
2228	if (se_num == 0xffffffff)
2229	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2230	else
2231	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2232
2233	if (sh_num == 0xffffffff)
2234	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2235	else
2236	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2237
2238	WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2239	}
2240
2241	static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2242	{
2243	u32 data, mask;
2244
2245	data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2246	data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2247
2248	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2249	data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2250
2251	mask = amdgpu_gfx_create_bitmask(bit_width: adev->gfx.config.max_backends_per_se /
2252	adev->gfx.config.max_sh_per_se);
2253
2254	return (~data) & mask;
2255	}
2256
2257	static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2258	{
2259	int i, j;
2260	u32 data;
2261	u32 active_rbs = 0;
2262	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2263	adev->gfx.config.max_sh_per_se;
2264
2265	mutex_lock(&adev->grbm_idx_mutex);
2266	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2267	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2268	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2269	data = gfx_v9_0_get_rb_active_bitmap(adev);
2270	active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2271	rb_bitmap_width_per_sh);
2272	}
2273	}
2274	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2275	mutex_unlock(lock: &adev->grbm_idx_mutex);
2276
2277	adev->gfx.config.backend_enable_mask = active_rbs;
2278	adev->gfx.config.num_rbs = hweight32(active_rbs);
2279	}
2280
2281	static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2282	uint32_t first_vmid,
2283	uint32_t last_vmid)
2284	{
2285	uint32_t data;
2286	uint32_t trap_config_vmid_mask = 0;
2287	int i;
2288
2289	/* Calculate trap config vmid mask */
2290	for (i = first_vmid; i < last_vmid; i++)
2291	trap_config_vmid_mask |= (1 << i);
2292
2293	data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2294	VMID_SEL, trap_config_vmid_mask);
2295	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2296	TRAP_EN, 1);
2297	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2298	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2299
2300	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2301	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2302	}
2303
2304	#define DEFAULT_SH_MEM_BASES (0x6000)
2305	static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2306	{
2307	int i;
2308	uint32_t sh_mem_config;
2309	uint32_t sh_mem_bases;
2310
2311	/*
2312	* Configure apertures:
2313	* LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2314	* Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2315	* GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2316	*/
2317	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2318
2319	sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2320	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2321	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2322
2323	mutex_lock(&adev->srbm_mutex);
2324	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2325	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: i, xcc_id: 0);
2326	/* CP and shaders */
2327	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2328	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2329	}
2330	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
2331	mutex_unlock(lock: &adev->srbm_mutex);
2332
2333	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
2334	access. These should be enabled by FW for target VMIDs. */
2335	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2336	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2337	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2338	WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2339	WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2340	}
2341	}
2342
2343	static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2344	{
2345	int vmid;
2346
2347	/*
2348	* Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2349	* access. Compute VMIDs should be enabled by FW for target VMIDs,
2350	* the driver can enable them for graphics. VMID0 should maintain
2351	* access so that HWS firmware can save/restore entries.
2352	*/
2353	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2354	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2355	WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2356	WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2357	WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2358	}
2359	}
2360
2361	static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2362	{
2363	uint32_t tmp;
2364
2365	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2366	case IP_VERSION(9, 4, 1):
2367	tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2368	tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2369	!READ_ONCE(adev->barrier_has_auto_waitcnt));
2370	WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2371	break;
2372	default:
2373	break;
2374	}
2375	}
2376
2377	static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2378	{
2379	u32 tmp;
2380	int i;
2381
2382	WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2383
2384	gfx_v9_0_tiling_mode_table_init(adev);
2385
2386	if (adev->gfx.num_gfx_rings)
2387	gfx_v9_0_setup_rb(adev);
2388	gfx_v9_0_get_cu_info(adev, cu_info: &adev->gfx.cu_info);
2389	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2390
2391	/* XXX SH_MEM regs */
2392	/* where to put LDS, scratch, GPUVM in FSA64 space */
2393	mutex_lock(&adev->srbm_mutex);
2394	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2395	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: i, xcc_id: 0);
2396	/* CP and shaders */
2397	if (i == 0) {
2398	tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2399	SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2400	tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2401	!!adev->gmc.noretry);
2402	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2403	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2404	} else {
2405	tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2406	SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2407	tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2408	!!adev->gmc.noretry);
2409	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2410	tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2411	(adev->gmc.private_aperture_start >> 48));
2412	tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2413	(adev->gmc.shared_aperture_start >> 48));
2414	WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2415	}
2416	}
2417	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
2418
2419	mutex_unlock(lock: &adev->srbm_mutex);
2420
2421	gfx_v9_0_init_compute_vmid(adev);
2422	gfx_v9_0_init_gds_vmid(adev);
2423	gfx_v9_0_init_sq_config(adev);
2424	}
2425
2426	static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2427	{
2428	u32 i, j, k;
2429	u32 mask;
2430
2431	mutex_lock(&adev->grbm_idx_mutex);
2432	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2433	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2434	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2435	for (k = 0; k < adev->usec_timeout; k++) {
2436	if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2437	break;
2438	udelay(1);
2439	}
2440	if (k == adev->usec_timeout) {
2441	amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2442	0xffffffff, 0xffffffff, 0);
2443	mutex_unlock(lock: &adev->grbm_idx_mutex);
2444	DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2445	i, j);
2446	return;
2447	}
2448	}
2449	}
2450	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2451	mutex_unlock(lock: &adev->grbm_idx_mutex);
2452
2453	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2454	RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2455	RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2456	RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2457	for (k = 0; k < adev->usec_timeout; k++) {
2458	if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2459	break;
2460	udelay(1);
2461	}
2462	}
2463
2464	static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2465	bool enable)
2466	{
2467	u32 tmp;
2468
2469	/* These interrupts should be enabled to drive DS clock */
2470
2471	tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2472
2473	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2474	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2475	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2476	if(adev->gfx.num_gfx_rings)
2477	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2478
2479	WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2480	}
2481
2482	static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2483	{
2484	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2485	/* csib */
2486	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2487	adev->gfx.rlc.clear_state_gpu_addr >> 32);
2488	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2489	adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2490	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2491	adev->gfx.rlc.clear_state_size);
2492	}
2493
2494	static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2495	int indirect_offset,
2496	int list_size,
2497	int *unique_indirect_regs,
2498	int unique_indirect_reg_count,
2499	int *indirect_start_offsets,
2500	int *indirect_start_offsets_count,
2501	int max_start_offsets_count)
2502	{
2503	int idx;
2504
2505	for (; indirect_offset < list_size; indirect_offset++) {
2506	WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2507	indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2508	*indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2509
2510	while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2511	indirect_offset += 2;
2512
2513	/* look for the matching indice */
2514	for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2515	if (unique_indirect_regs[idx] ==
2516	register_list_format[indirect_offset] ||
2517	!unique_indirect_regs[idx])
2518	break;
2519	}
2520
2521	BUG_ON(idx >= unique_indirect_reg_count);
2522
2523	if (!unique_indirect_regs[idx])
2524	unique_indirect_regs[idx] = register_list_format[indirect_offset];
2525
2526	indirect_offset++;
2527	}
2528	}
2529	}
2530
2531	static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2532	{
2533	int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2534	int unique_indirect_reg_count = 0;
2535
2536	int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2537	int indirect_start_offsets_count = 0;
2538
2539	int list_size = 0;
2540	int i = 0, j = 0;
2541	u32 tmp = 0;
2542
2543	u32 *register_list_format =
2544	kmemdup(p: adev->gfx.rlc.register_list_format,
2545	size: adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2546	if (!register_list_format)
2547	return -ENOMEM;
2548
2549	/* setup unique_indirect_regs array and indirect_start_offsets array */
2550	unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2551	gfx_v9_1_parse_ind_reg_list(register_list_format,
2552	indirect_offset: adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2553	list_size: adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2554	unique_indirect_regs,
2555	unique_indirect_reg_count,
2556	indirect_start_offsets,
2557	indirect_start_offsets_count: &indirect_start_offsets_count,
2558	ARRAY_SIZE(indirect_start_offsets));
2559
2560	/* enable auto inc in case it is disabled */
2561	tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2562	tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2563	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2564
2565	/* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2566	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2567	RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2568	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2569	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2570	adev->gfx.rlc.register_restore[i]);
2571
2572	/* load indirect register */
2573	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2574	adev->gfx.rlc.reg_list_format_start);
2575
2576	/* direct register portion */
2577	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2578	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2579	register_list_format[i]);
2580
2581	/* indirect register portion */
2582	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2583	if (register_list_format[i] == 0xFFFFFFFF) {
2584	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2585	continue;
2586	}
2587
2588	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2589	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2590
2591	for (j = 0; j < unique_indirect_reg_count; j++) {
2592	if (register_list_format[i] == unique_indirect_regs[j]) {
2593	WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2594	break;
2595	}
2596	}
2597
2598	BUG_ON(j >= unique_indirect_reg_count);
2599
2600	i++;
2601	}
2602
2603	/* set save/restore list size */
2604	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2605	list_size = list_size >> 1;
2606	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2607	adev->gfx.rlc.reg_restore_list_size);
2608	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2609
2610	/* write the starting offsets to RLC scratch ram */
2611	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2612	adev->gfx.rlc.starting_offsets_start);
2613	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2614	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2615	indirect_start_offsets[i]);
2616
2617	/* load unique indirect regs*/
2618	for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2619	if (unique_indirect_regs[i] != 0) {
2620	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2621	+ GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2622	unique_indirect_regs[i] & 0x3FFFF);
2623
2624	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2625	+ GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2626	unique_indirect_regs[i] >> 20);
2627	}
2628	}
2629
2630	kfree(objp: register_list_format);
2631	return 0;
2632	}
2633
2634	static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2635	{
2636	WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2637	}
2638
2639	static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2640	bool enable)
2641	{
2642	uint32_t data = 0;
2643	uint32_t default_data = 0;
2644
2645	default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2646	if (enable) {
2647	/* enable GFXIP control over CGPG */
2648	data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2649	if(default_data != data)
2650	WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2651
2652	/* update status */
2653	data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2654	data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2655	if(default_data != data)
2656	WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2657	} else {
2658	/* restore GFXIP control over GCPG */
2659	data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2660	if(default_data != data)
2661	WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2662	}
2663	}
2664
2665	static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2666	{
2667	uint32_t data = 0;
2668
2669	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2670	AMD_PG_SUPPORT_GFX_SMG |
2671	AMD_PG_SUPPORT_GFX_DMG)) {
2672	/* init IDLE_POLL_COUNT = 60 */
2673	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2674	data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2675	data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2676	WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2677
2678	/* init RLC PG Delay */
2679	data = 0;
2680	data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2681	data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2682	data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2683	data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2684	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2685
2686	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2687	data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2688	data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2689	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2690
2691	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2692	data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2693	data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2694	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2695
2696	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2697	data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2698
2699	/* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2700	data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2701	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2702	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 3, 0))
2703	pwr_10_0_gfxip_control_over_cgpg(adev, enable: true);
2704	}
2705	}
2706
2707	static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2708	bool enable)
2709	{
2710	uint32_t data = 0;
2711	uint32_t default_data = 0;
2712
2713	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2714	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2715	SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2716	enable ? 1 : 0);
2717	if (default_data != data)
2718	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2719	}
2720
2721	static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2722	bool enable)
2723	{
2724	uint32_t data = 0;
2725	uint32_t default_data = 0;
2726
2727	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2728	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2729	SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2730	enable ? 1 : 0);
2731	if(default_data != data)
2732	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2733	}
2734
2735	static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2736	bool enable)
2737	{
2738	uint32_t data = 0;
2739	uint32_t default_data = 0;
2740
2741	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2742	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2743	CP_PG_DISABLE,
2744	enable ? 0 : 1);
2745	if(default_data != data)
2746	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2747	}
2748
2749	static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2750	bool enable)
2751	{
2752	uint32_t data, default_data;
2753
2754	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2755	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2756	GFX_POWER_GATING_ENABLE,
2757	enable ? 1 : 0);
2758	if(default_data != data)
2759	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2760	}
2761
2762	static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2763	bool enable)
2764	{
2765	uint32_t data, default_data;
2766
2767	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2768	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2769	GFX_PIPELINE_PG_ENABLE,
2770	enable ? 1 : 0);
2771	if(default_data != data)
2772	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2773
2774	if (!enable)
2775	/* read any GFX register to wake up GFX */
2776	data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2777	}
2778
2779	static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2780	bool enable)
2781	{
2782	uint32_t data, default_data;
2783
2784	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2785	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2786	STATIC_PER_CU_PG_ENABLE,
2787	enable ? 1 : 0);
2788	if(default_data != data)
2789	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2790	}
2791
2792	static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2793	bool enable)
2794	{
2795	uint32_t data, default_data;
2796
2797	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2798	data = REG_SET_FIELD(data, RLC_PG_CNTL,
2799	DYN_PER_CU_PG_ENABLE,
2800	enable ? 1 : 0);
2801	if(default_data != data)
2802	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2803	}
2804
2805	static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2806	{
2807	gfx_v9_0_init_csb(adev);
2808
2809	/*
2810	* Rlc save restore list is workable since v2_1.
2811	* And it's needed by gfxoff feature.
2812	*/
2813	if (adev->gfx.rlc.is_rlc_v2_1) {
2814	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
2815	IP_VERSION(9, 2, 1) ||
2816	(adev->apu_flags & AMD_APU_IS_RAVEN2))
2817	gfx_v9_1_init_rlc_save_restore_list(adev);
2818	gfx_v9_0_enable_save_restore_machine(adev);
2819	}
2820
2821	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2822	AMD_PG_SUPPORT_GFX_SMG |
2823	AMD_PG_SUPPORT_GFX_DMG |
2824	AMD_PG_SUPPORT_CP |
2825	AMD_PG_SUPPORT_GDS |
2826	AMD_PG_SUPPORT_RLC_SMU_HS)) {
2827	WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
2828	adev->gfx.rlc.cp_table_gpu_addr >> 8);
2829	gfx_v9_0_init_gfx_power_gating(adev);
2830	}
2831	}
2832
2833	static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2834	{
2835	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2836	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: false);
2837	gfx_v9_0_wait_for_rlc_serdes(adev);
2838	}
2839
2840	static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2841	{
2842	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2843	udelay(50);
2844	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2845	udelay(50);
2846	}
2847
2848	static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
2849	{
2850	#ifdef AMDGPU_RLC_DEBUG_RETRY
2851	u32 rlc_ucode_ver;
2852	#endif
2853
2854	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2855	udelay(50);
2856
2857	/* carrizo do enable cp interrupt after cp inited */
2858	if (!(adev->flags & AMD_IS_APU)) {
2859	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: true);
2860	udelay(50);
2861	}
2862
2863	#ifdef AMDGPU_RLC_DEBUG_RETRY
2864	/* RLC_GPM_GENERAL_6 : RLC Ucode version */
2865	rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
2866	if(rlc_ucode_ver == 0x108) {
2867	DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
2868	rlc_ucode_ver, adev->gfx.rlc_fw_version);
2869	/* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
2870	* default is 0x9C4 to create a 100us interval */
2871	WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
2872	/* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
2873	* to disable the page fault retry interrupts, default is
2874	* 0x100 (256) */
2875	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
2876	}
2877	#endif
2878	}
2879
2880	static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
2881	{
2882	const struct rlc_firmware_header_v2_0 *hdr;
2883	const __le32 *fw_data;
2884	unsigned i, fw_size;
2885
2886	if (!adev->gfx.rlc_fw)
2887	return -EINVAL;
2888
2889	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2890	amdgpu_ucode_print_rlc_hdr(hdr: &hdr->header);
2891
2892	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2893	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2894	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2895
2896	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
2897	RLCG_UCODE_LOADING_START_ADDRESS);
2898	for (i = 0; i < fw_size; i++)
2899	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
2900	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2901
2902	return 0;
2903	}
2904
2905	static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
2906	{
2907	int r;
2908
2909	if (amdgpu_sriov_vf(adev)) {
2910	gfx_v9_0_init_csb(adev);
2911	return 0;
2912	}
2913
2914	adev->gfx.rlc.funcs->stop(adev);
2915
2916	/* disable CG */
2917	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
2918
2919	gfx_v9_0_init_pg(adev);
2920
2921	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2922	/* legacy rlc firmware loading */
2923	r = gfx_v9_0_rlc_load_microcode(adev);
2924	if (r)
2925	return r;
2926	}
2927
2928	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
2929	case IP_VERSION(9, 2, 2):
2930	case IP_VERSION(9, 1, 0):
2931	gfx_v9_0_init_lbpw(adev);
2932	if (amdgpu_lbpw == 0)
2933	gfx_v9_0_enable_lbpw(adev, enable: false);
2934	else
2935	gfx_v9_0_enable_lbpw(adev, enable: true);
2936	break;
2937	case IP_VERSION(9, 4, 0):
2938	gfx_v9_4_init_lbpw(adev);
2939	if (amdgpu_lbpw > 0)
2940	gfx_v9_0_enable_lbpw(adev, enable: true);
2941	else
2942	gfx_v9_0_enable_lbpw(adev, enable: false);
2943	break;
2944	default:
2945	break;
2946	}
2947
2948	gfx_v9_0_update_spm_vmid_internal(adev, vmid: 0xf);
2949
2950	adev->gfx.rlc.funcs->start(adev);
2951
2952	return 0;
2953	}
2954
2955	static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2956	{
2957	u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
2958
2959	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2960	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2961	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
2962	WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
2963	udelay(50);
2964	}
2965
2966	static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2967	{
2968	const struct gfx_firmware_header_v1_0 *pfp_hdr;
2969	const struct gfx_firmware_header_v1_0 *ce_hdr;
2970	const struct gfx_firmware_header_v1_0 *me_hdr;
2971	const __le32 *fw_data;
2972	unsigned i, fw_size;
2973
2974	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2975	return -EINVAL;
2976
2977	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2978	adev->gfx.pfp_fw->data;
2979	ce_hdr = (const struct gfx_firmware_header_v1_0 *)
2980	adev->gfx.ce_fw->data;
2981	me_hdr = (const struct gfx_firmware_header_v1_0 *)
2982	adev->gfx.me_fw->data;
2983
2984	amdgpu_ucode_print_gfx_hdr(hdr: &pfp_hdr->header);
2985	amdgpu_ucode_print_gfx_hdr(hdr: &ce_hdr->header);
2986	amdgpu_ucode_print_gfx_hdr(hdr: &me_hdr->header);
2987
2988	gfx_v9_0_cp_gfx_enable(adev, enable: false);
2989
2990	/* PFP */
2991	fw_data = (const __le32 *)
2992	(adev->gfx.pfp_fw->data +
2993	le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2994	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2995	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
2996	for (i = 0; i < fw_size; i++)
2997	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2998	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2999
3000	/* CE */
3001	fw_data = (const __le32 *)
3002	(adev->gfx.ce_fw->data +
3003	le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3004	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3005	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3006	for (i = 0; i < fw_size; i++)
3007	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3008	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3009
3010	/* ME */
3011	fw_data = (const __le32 *)
3012	(adev->gfx.me_fw->data +
3013	le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3014	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3015	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3016	for (i = 0; i < fw_size; i++)
3017	WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3018	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3019
3020	return 0;
3021	}
3022
3023	static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3024	{
3025	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3026	const struct cs_section_def *sect = NULL;
3027	const struct cs_extent_def *ext = NULL;
3028	int r, i, tmp;
3029
3030	/* init the CP */
3031	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3032	WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3033
3034	gfx_v9_0_cp_gfx_enable(adev, enable: true);
3035
3036	/* Now only limit the quirk on the APU gfx9 series and already
3037	* confirmed that the APU gfx10/gfx11 needn't such update.
3038	*/
3039	if (adev->flags & AMD_IS_APU &&
3040	adev->in_s3 && !adev->suspend_complete) {
3041	DRM_INFO(" Will skip the CSB packet resubmit\n");
3042	return 0;
3043	}
3044	r = amdgpu_ring_alloc(ring, ndw: gfx_v9_0_get_csb_size(adev) + 4 + 3);
3045	if (r) {
3046	DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3047	return r;
3048	}
3049
3050	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3051	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3052
3053	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3054	amdgpu_ring_write(ring, v: 0x80000000);
3055	amdgpu_ring_write(ring, v: 0x80000000);
3056
3057	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3058	for (ext = sect->section; ext->extent != NULL; ++ext) {
3059	if (sect->id == SECT_CONTEXT) {
3060	amdgpu_ring_write(ring,
3061	PACKET3(PACKET3_SET_CONTEXT_REG,
3062	ext->reg_count));
3063	amdgpu_ring_write(ring,
3064	v: ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3065	for (i = 0; i < ext->reg_count; i++)
3066	amdgpu_ring_write(ring, v: ext->extent[i]);
3067	}
3068	}
3069	}
3070
3071	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3072	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3073
3074	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3075	amdgpu_ring_write(ring, v: 0);
3076
3077	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3078	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3079	amdgpu_ring_write(ring, v: 0x8000);
3080	amdgpu_ring_write(ring, v: 0x8000);
3081
3082	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3083	tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3084	(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3085	amdgpu_ring_write(ring, v: tmp);
3086	amdgpu_ring_write(ring, v: 0);
3087
3088	amdgpu_ring_commit(ring);
3089
3090	return 0;
3091	}
3092
3093	static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3094	{
3095	struct amdgpu_ring *ring;
3096	u32 tmp;
3097	u32 rb_bufsz;
3098	u64 rb_addr, rptr_addr, wptr_gpu_addr;
3099
3100	/* Set the write pointer delay */
3101	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3102
3103	/* set the RB to use vmid 0 */
3104	WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3105
3106	/* Set ring buffer size */
3107	ring = &adev->gfx.gfx_ring[0];
3108	rb_bufsz = order_base_2(ring->ring_size / 8);
3109	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3110	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3111	#ifdef __BIG_ENDIAN
3112	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3113	#endif
3114	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3115
3116	/* Initialize the ring buffer's write pointers */
3117	ring->wptr = 0;
3118	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3119	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3120
3121	/* set the wb address wether it's enabled or not */
3122	rptr_addr = ring->rptr_gpu_addr;
3123	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3124	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3125
3126	wptr_gpu_addr = ring->wptr_gpu_addr;
3127	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3128	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3129
3130	mdelay(1);
3131	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3132
3133	rb_addr = ring->gpu_addr >> 8;
3134	WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3135	WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3136
3137	tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3138	if (ring->use_doorbell) {
3139	tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3140	DOORBELL_OFFSET, ring->doorbell_index);
3141	tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3142	DOORBELL_EN, 1);
3143	} else {
3144	tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3145	}
3146	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3147
3148	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3149	DOORBELL_RANGE_LOWER, ring->doorbell_index);
3150	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3151
3152	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3153	CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3154
3155
3156	/* start the ring */
3157	gfx_v9_0_cp_gfx_start(adev);
3158
3159	return 0;
3160	}
3161
3162	static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3163	{
3164	if (enable) {
3165	WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3166	} else {
3167	WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3168	(CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3169	adev->gfx.kiq[0].ring.sched.ready = false;
3170	}
3171	udelay(50);
3172	}
3173
3174	static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3175	{
3176	const struct gfx_firmware_header_v1_0 *mec_hdr;
3177	const __le32 *fw_data;
3178	unsigned i;
3179	u32 tmp;
3180
3181	if (!adev->gfx.mec_fw)
3182	return -EINVAL;
3183
3184	gfx_v9_0_cp_compute_enable(adev, enable: false);
3185
3186	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3187	amdgpu_ucode_print_gfx_hdr(hdr: &mec_hdr->header);
3188
3189	fw_data = (const __le32 *)
3190	(adev->gfx.mec_fw->data +
3191	le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3192	tmp = 0;
3193	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3194	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3195	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3196
3197	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3198	adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3199	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3200	upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3201
3202	/* MEC1 */
3203	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3204	mec_hdr->jt_offset);
3205	for (i = 0; i < mec_hdr->jt_size; i++)
3206	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3207	le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3208
3209	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3210	adev->gfx.mec_fw_version);
3211	/* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3212
3213	return 0;
3214	}
3215
3216	/* KIQ functions */
3217	static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3218	{
3219	uint32_t tmp;
3220	struct amdgpu_device *adev = ring->adev;
3221
3222	/* tell RLC which is KIQ queue */
3223	tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3224	tmp &= 0xffffff00;
3225	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3226	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3227	tmp |= 0x80;
3228	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3229	}
3230
3231	static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3232	{
3233	struct amdgpu_device *adev = ring->adev;
3234
3235	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3236	if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3237	mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3238	mqd->cp_hqd_queue_priority =
3239	AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3240	}
3241	}
3242	}
3243
3244	static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3245	{
3246	struct amdgpu_device *adev = ring->adev;
3247	struct v9_mqd *mqd = ring->mqd_ptr;
3248	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3249	uint32_t tmp;
3250
3251	mqd->header = 0xC0310800;
3252	mqd->compute_pipelinestat_enable = 0x00000001;
3253	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3254	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3255	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3256	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3257	mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3258	mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3259	mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3260	mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3261	mqd->compute_misc_reserved = 0x00000003;
3262
3263	mqd->dynamic_cu_mask_addr_lo =
3264	lower_32_bits(ring->mqd_gpu_addr
3265	+ offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3266	mqd->dynamic_cu_mask_addr_hi =
3267	upper_32_bits(ring->mqd_gpu_addr
3268	+ offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3269
3270	eop_base_addr = ring->eop_gpu_addr >> 8;
3271	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3272	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3273
3274	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3275	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3276	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3277	(order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3278
3279	mqd->cp_hqd_eop_control = tmp;
3280
3281	/* enable doorbell? */
3282	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3283
3284	if (ring->use_doorbell) {
3285	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3286	DOORBELL_OFFSET, ring->doorbell_index);
3287	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3288	DOORBELL_EN, 1);
3289	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3290	DOORBELL_SOURCE, 0);
3291	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3292	DOORBELL_HIT, 0);
3293	} else {
3294	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3295	DOORBELL_EN, 0);
3296	}
3297
3298	mqd->cp_hqd_pq_doorbell_control = tmp;
3299
3300	/* disable the queue if it's active */
3301	ring->wptr = 0;
3302	mqd->cp_hqd_dequeue_request = 0;
3303	mqd->cp_hqd_pq_rptr = 0;
3304	mqd->cp_hqd_pq_wptr_lo = 0;
3305	mqd->cp_hqd_pq_wptr_hi = 0;
3306
3307	/* set the pointer to the MQD */
3308	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3309	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3310
3311	/* set MQD vmid to 0 */
3312	tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3313	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3314	mqd->cp_mqd_control = tmp;
3315
3316	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3317	hqd_gpu_addr = ring->gpu_addr >> 8;
3318	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3319	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3320
3321	/* set up the HQD, this is similar to CP_RB0_CNTL */
3322	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3323	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3324	(order_base_2(ring->ring_size / 4) - 1));
3325	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3326	(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3327	#ifdef __BIG_ENDIAN
3328	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3329	#endif
3330	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3331	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3332	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3333	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3334	mqd->cp_hqd_pq_control = tmp;
3335
3336	/* set the wb address whether it's enabled or not */
3337	wb_gpu_addr = ring->rptr_gpu_addr;
3338	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3339	mqd->cp_hqd_pq_rptr_report_addr_hi =
3340	upper_32_bits(wb_gpu_addr) & 0xffff;
3341
3342	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3343	wb_gpu_addr = ring->wptr_gpu_addr;
3344	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3345	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3346
3347	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3348	ring->wptr = 0;
3349	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3350
3351	/* set the vmid for the queue */
3352	mqd->cp_hqd_vmid = 0;
3353
3354	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3355	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3356	mqd->cp_hqd_persistent_state = tmp;
3357
3358	/* set MIN_IB_AVAIL_SIZE */
3359	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3360	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3361	mqd->cp_hqd_ib_control = tmp;
3362
3363	/* set static priority for a queue/ring */
3364	gfx_v9_0_mqd_set_priority(ring, mqd);
3365	mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3366
3367	/* map_queues packet doesn't need activate the queue,
3368	* so only kiq need set this field.
3369	*/
3370	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3371	mqd->cp_hqd_active = 1;
3372
3373	return 0;
3374	}
3375
3376	static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3377	{
3378	struct amdgpu_device *adev = ring->adev;
3379	struct v9_mqd *mqd = ring->mqd_ptr;
3380	int j;
3381
3382	/* disable wptr polling */
3383	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3384
3385	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3386	mqd->cp_hqd_eop_base_addr_lo);
3387	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3388	mqd->cp_hqd_eop_base_addr_hi);
3389
3390	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3391	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3392	mqd->cp_hqd_eop_control);
3393
3394	/* enable doorbell? */
3395	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3396	mqd->cp_hqd_pq_doorbell_control);
3397
3398	/* disable the queue if it's active */
3399	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3400	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3401	for (j = 0; j < adev->usec_timeout; j++) {
3402	if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3403	break;
3404	udelay(1);
3405	}
3406	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3407	mqd->cp_hqd_dequeue_request);
3408	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3409	mqd->cp_hqd_pq_rptr);
3410	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3411	mqd->cp_hqd_pq_wptr_lo);
3412	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3413	mqd->cp_hqd_pq_wptr_hi);
3414	}
3415
3416	/* set the pointer to the MQD */
3417	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3418	mqd->cp_mqd_base_addr_lo);
3419	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3420	mqd->cp_mqd_base_addr_hi);
3421
3422	/* set MQD vmid to 0 */
3423	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3424	mqd->cp_mqd_control);
3425
3426	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3427	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3428	mqd->cp_hqd_pq_base_lo);
3429	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3430	mqd->cp_hqd_pq_base_hi);
3431
3432	/* set up the HQD, this is similar to CP_RB0_CNTL */
3433	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3434	mqd->cp_hqd_pq_control);
3435
3436	/* set the wb address whether it's enabled or not */
3437	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3438	mqd->cp_hqd_pq_rptr_report_addr_lo);
3439	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3440	mqd->cp_hqd_pq_rptr_report_addr_hi);
3441
3442	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3443	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3444	mqd->cp_hqd_pq_wptr_poll_addr_lo);
3445	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3446	mqd->cp_hqd_pq_wptr_poll_addr_hi);
3447
3448	/* enable the doorbell if requested */
3449	if (ring->use_doorbell) {
3450	WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3451	(adev->doorbell_index.kiq * 2) << 2);
3452	/* If GC has entered CGPG, ringing doorbell > first page
3453	* doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3454	* workaround this issue. And this change has to align with firmware
3455	* update.
3456	*/
3457	if (check_if_enlarge_doorbell_range(adev))
3458	WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3459	(adev->doorbell.size - 4));
3460	else
3461	WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3462	(adev->doorbell_index.userqueue_end * 2) << 2);
3463	}
3464
3465	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3466	mqd->cp_hqd_pq_doorbell_control);
3467
3468	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3469	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3470	mqd->cp_hqd_pq_wptr_lo);
3471	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3472	mqd->cp_hqd_pq_wptr_hi);
3473
3474	/* set the vmid for the queue */
3475	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3476
3477	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3478	mqd->cp_hqd_persistent_state);
3479
3480	/* activate the queue */
3481	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3482	mqd->cp_hqd_active);
3483
3484	if (ring->use_doorbell)
3485	WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3486
3487	return 0;
3488	}
3489
3490	static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3491	{
3492	struct amdgpu_device *adev = ring->adev;
3493	int j;
3494
3495	/* disable the queue if it's active */
3496	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3497
3498	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3499
3500	for (j = 0; j < adev->usec_timeout; j++) {
3501	if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3502	break;
3503	udelay(1);
3504	}
3505
3506	if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3507	DRM_DEBUG("KIQ dequeue request failed.\n");
3508
3509	/* Manual disable if dequeue request times out */
3510	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3511	}
3512
3513	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3514	0);
3515	}
3516
3517	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3518	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3519	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3520	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3521	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3522	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3523	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3524	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3525
3526	return 0;
3527	}
3528
3529	static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3530	{
3531	struct amdgpu_device *adev = ring->adev;
3532	struct v9_mqd *mqd = ring->mqd_ptr;
3533	struct v9_mqd *tmp_mqd;
3534
3535	gfx_v9_0_kiq_setting(ring);
3536
3537	/* GPU could be in bad state during probe, driver trigger the reset
3538	* after load the SMU, in this case , the mqd is not be initialized.
3539	* driver need to re-init the mqd.
3540	* check mqd->cp_hqd_pq_control since this value should not be 0
3541	*/
3542	tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3543	if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3544	/* for GPU_RESET case , reset MQD to a clean status */
3545	if (adev->gfx.kiq[0].mqd_backup)
3546	memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3547
3548	/* reset ring buffer */
3549	ring->wptr = 0;
3550	amdgpu_ring_clear_ring(ring);
3551
3552	mutex_lock(&adev->srbm_mutex);
3553	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, xcc_id: 0);
3554	gfx_v9_0_kiq_init_register(ring);
3555	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3556	mutex_unlock(lock: &adev->srbm_mutex);
3557	} else {
3558	memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3559	((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3560	((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3561	if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3562	amdgpu_ring_clear_ring(ring);
3563	mutex_lock(&adev->srbm_mutex);
3564	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, xcc_id: 0);
3565	gfx_v9_0_mqd_init(ring);
3566	gfx_v9_0_kiq_init_register(ring);
3567	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3568	mutex_unlock(lock: &adev->srbm_mutex);
3569
3570	if (adev->gfx.kiq[0].mqd_backup)
3571	memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3572	}
3573
3574	return 0;
3575	}
3576
3577	static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3578	{
3579	struct amdgpu_device *adev = ring->adev;
3580	struct v9_mqd *mqd = ring->mqd_ptr;
3581	int mqd_idx = ring - &adev->gfx.compute_ring[0];
3582	struct v9_mqd *tmp_mqd;
3583
3584	/* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3585	* is not be initialized before
3586	*/
3587	tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3588
3589	if (!tmp_mqd->cp_hqd_pq_control ||
3590	(!amdgpu_in_reset(adev) && !adev->in_suspend)) {
3591	memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3592	((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3593	((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3594	mutex_lock(&adev->srbm_mutex);
3595	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, xcc_id: 0);
3596	gfx_v9_0_mqd_init(ring);
3597	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3598	mutex_unlock(lock: &adev->srbm_mutex);
3599
3600	if (adev->gfx.mec.mqd_backup[mqd_idx])
3601	memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3602	} else {
3603	/* restore MQD to a clean status */
3604	if (adev->gfx.mec.mqd_backup[mqd_idx])
3605	memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3606	/* reset ring buffer */
3607	ring->wptr = 0;
3608	atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, i: 0);
3609	amdgpu_ring_clear_ring(ring);
3610	}
3611
3612	return 0;
3613	}
3614
3615	static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3616	{
3617	struct amdgpu_ring *ring;
3618	int r;
3619
3620	ring = &adev->gfx.kiq[0].ring;
3621
3622	r = amdgpu_bo_reserve(bo: ring->mqd_obj, no_intr: false);
3623	if (unlikely(r != 0))
3624	return r;
3625
3626	r = amdgpu_bo_kmap(bo: ring->mqd_obj, ptr: (void **)&ring->mqd_ptr);
3627	if (unlikely(r != 0)) {
3628	amdgpu_bo_unreserve(bo: ring->mqd_obj);
3629	return r;
3630	}
3631
3632	gfx_v9_0_kiq_init_queue(ring);
3633	amdgpu_bo_kunmap(bo: ring->mqd_obj);
3634	ring->mqd_ptr = NULL;
3635	amdgpu_bo_unreserve(bo: ring->mqd_obj);
3636	return 0;
3637	}
3638
3639	static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3640	{
3641	struct amdgpu_ring *ring = NULL;
3642	int r = 0, i;
3643
3644	gfx_v9_0_cp_compute_enable(adev, enable: true);
3645
3646	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3647	ring = &adev->gfx.compute_ring[i];
3648
3649	r = amdgpu_bo_reserve(bo: ring->mqd_obj, no_intr: false);
3650	if (unlikely(r != 0))
3651	goto done;
3652	r = amdgpu_bo_kmap(bo: ring->mqd_obj, ptr: (void **)&ring->mqd_ptr);
3653	if (!r) {
3654	r = gfx_v9_0_kcq_init_queue(ring);
3655	amdgpu_bo_kunmap(bo: ring->mqd_obj);
3656	ring->mqd_ptr = NULL;
3657	}
3658	amdgpu_bo_unreserve(bo: ring->mqd_obj);
3659	if (r)
3660	goto done;
3661	}
3662
3663	r = amdgpu_gfx_enable_kcq(adev, xcc_id: 0);
3664	done:
3665	return r;
3666	}
3667
3668	static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3669	{
3670	int r, i;
3671	struct amdgpu_ring *ring;
3672
3673	if (!(adev->flags & AMD_IS_APU))
3674	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: false);
3675
3676	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3677	if (adev->gfx.num_gfx_rings) {
3678	/* legacy firmware loading */
3679	r = gfx_v9_0_cp_gfx_load_microcode(adev);
3680	if (r)
3681	return r;
3682	}
3683
3684	r = gfx_v9_0_cp_compute_load_microcode(adev);
3685	if (r)
3686	return r;
3687	}
3688
3689	r = gfx_v9_0_kiq_resume(adev);
3690	if (r)
3691	return r;
3692
3693	if (adev->gfx.num_gfx_rings) {
3694	r = gfx_v9_0_cp_gfx_resume(adev);
3695	if (r)
3696	return r;
3697	}
3698
3699	r = gfx_v9_0_kcq_resume(adev);
3700	if (r)
3701	return r;
3702
3703	if (adev->gfx.num_gfx_rings) {
3704	ring = &adev->gfx.gfx_ring[0];
3705	r = amdgpu_ring_test_helper(ring);
3706	if (r)
3707	return r;
3708	}
3709
3710	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3711	ring = &adev->gfx.compute_ring[i];
3712	amdgpu_ring_test_helper(ring);
3713	}
3714
3715	gfx_v9_0_enable_gui_idle_interrupt(adev, enable: true);
3716
3717	return 0;
3718	}
3719
3720	static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3721	{
3722	u32 tmp;
3723
3724	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1) &&
3725	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 2))
3726	return;
3727
3728	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3729	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3730	adev->df.hash_status.hash_64k);
3731	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3732	adev->df.hash_status.hash_2m);
3733	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3734	adev->df.hash_status.hash_1g);
3735	WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3736	}
3737
3738	static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3739	{
3740	if (adev->gfx.num_gfx_rings)
3741	gfx_v9_0_cp_gfx_enable(adev, enable);
3742	gfx_v9_0_cp_compute_enable(adev, enable);
3743	}
3744
3745	static int gfx_v9_0_hw_init(void *handle)
3746	{
3747	int r;
3748	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3749
3750	if (!amdgpu_sriov_vf(adev))
3751	gfx_v9_0_init_golden_registers(adev);
3752
3753	gfx_v9_0_constants_init(adev);
3754
3755	gfx_v9_0_init_tcp_config(adev);
3756
3757	r = adev->gfx.rlc.funcs->resume(adev);
3758	if (r)
3759	return r;
3760
3761	r = gfx_v9_0_cp_resume(adev);
3762	if (r)
3763	return r;
3764
3765	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
3766	gfx_v9_4_2_set_power_brake_sequence(adev);
3767
3768	return r;
3769	}
3770
3771	static int gfx_v9_0_hw_fini(void *handle)
3772	{
3773	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3774
3775	if (amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
3776	amdgpu_irq_put(adev, src: &adev->gfx.cp_ecc_error_irq, type: 0);
3777	amdgpu_irq_put(adev, src: &adev->gfx.priv_reg_irq, type: 0);
3778	amdgpu_irq_put(adev, src: &adev->gfx.priv_inst_irq, type: 0);
3779
3780	/* DF freeze and kcq disable will fail */
3781	if (!amdgpu_ras_intr_triggered())
3782	/* disable KCQ to avoid CPC touch memory not valid anymore */
3783	amdgpu_gfx_disable_kcq(adev, xcc_id: 0);
3784
3785	if (amdgpu_sriov_vf(adev)) {
3786	gfx_v9_0_cp_gfx_enable(adev, enable: false);
3787	/* must disable polling for SRIOV when hw finished, otherwise
3788	* CPC engine may still keep fetching WB address which is already
3789	* invalid after sw finished and trigger DMAR reading error in
3790	* hypervisor side.
3791	*/
3792	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3793	return 0;
3794	}
3795
3796	/* Use deinitialize sequence from CAIL when unbinding device from driver,
3797	* otherwise KIQ is hanging when binding back
3798	*/
3799	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
3800	mutex_lock(&adev->srbm_mutex);
3801	soc15_grbm_select(adev, me: adev->gfx.kiq[0].ring.me,
3802	pipe: adev->gfx.kiq[0].ring.pipe,
3803	queue: adev->gfx.kiq[0].ring.queue, vmid: 0, xcc_id: 0);
3804	gfx_v9_0_kiq_fini_register(ring: &adev->gfx.kiq[0].ring);
3805	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
3806	mutex_unlock(lock: &adev->srbm_mutex);
3807	}
3808
3809	gfx_v9_0_cp_enable(adev, enable: false);
3810
3811	/* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
3812	if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
3813	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(9, 4, 2))) {
3814	dev_dbg(adev->dev, "Skipping RLC halt\n");
3815	return 0;
3816	}
3817
3818	adev->gfx.rlc.funcs->stop(adev);
3819	return 0;
3820	}
3821
3822	static int gfx_v9_0_suspend(void *handle)
3823	{
3824	return gfx_v9_0_hw_fini(handle);
3825	}
3826
3827	static int gfx_v9_0_resume(void *handle)
3828	{
3829	return gfx_v9_0_hw_init(handle);
3830	}
3831
3832	static bool gfx_v9_0_is_idle(void *handle)
3833	{
3834	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3835
3836	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3837	GRBM_STATUS, GUI_ACTIVE))
3838	return false;
3839	else
3840	return true;
3841	}
3842
3843	static int gfx_v9_0_wait_for_idle(void *handle)
3844	{
3845	unsigned i;
3846	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3847
3848	for (i = 0; i < adev->usec_timeout; i++) {
3849	if (gfx_v9_0_is_idle(handle))
3850	return 0;
3851	udelay(1);
3852	}
3853	return -ETIMEDOUT;
3854	}
3855
3856	static int gfx_v9_0_soft_reset(void *handle)
3857	{
3858	u32 grbm_soft_reset = 0;
3859	u32 tmp;
3860	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3861
3862	/* GRBM_STATUS */
3863	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3864	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3865	GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3866	GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
3867	GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
3868	GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3869	GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
3870	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3871	GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3872	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3873	GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
3874	}
3875
3876	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
3877	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3878	GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3879	}
3880
3881	/* GRBM_STATUS2 */
3882	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
3883	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
3884	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3885	GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3886
3887
3888	if (grbm_soft_reset) {
3889	/* stop the rlc */
3890	adev->gfx.rlc.funcs->stop(adev);
3891
3892	if (adev->gfx.num_gfx_rings)
3893	/* Disable GFX parsing/prefetching */
3894	gfx_v9_0_cp_gfx_enable(adev, enable: false);
3895
3896	/* Disable MEC parsing/prefetching */
3897	gfx_v9_0_cp_compute_enable(adev, enable: false);
3898
3899	if (grbm_soft_reset) {
3900	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3901	tmp |= grbm_soft_reset;
3902	dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3903	WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3904	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3905
3906	udelay(50);
3907
3908	tmp &= ~grbm_soft_reset;
3909	WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3910	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3911	}
3912
3913	/* Wait a little for things to settle down */
3914	udelay(50);
3915	}
3916	return 0;
3917	}
3918
3919	static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
3920	{
3921	signed long r, cnt = 0;
3922	unsigned long flags;
3923	uint32_t seq, reg_val_offs = 0;
3924	uint64_t value = 0;
3925	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
3926	struct amdgpu_ring *ring = &kiq->ring;
3927
3928	BUG_ON(!ring->funcs->emit_rreg);
3929
3930	spin_lock_irqsave(&kiq->ring_lock, flags);
3931	if (amdgpu_device_wb_get(adev, wb: &reg_val_offs)) {
3932	pr_err("critical bug! too many kiq readers\n");
3933	goto failed_unlock;
3934	}
3935	amdgpu_ring_alloc(ring, ndw: 32);
3936	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3937	amdgpu_ring_write(ring, v: 9 | /* src: register*/
3938	(5 << 8) | /* dst: memory */
3939	(1 << 16) | /* count sel */
3940	(1 << 20)); /* write confirm */
3941	amdgpu_ring_write(ring, v: 0);
3942	amdgpu_ring_write(ring, v: 0);
3943	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3944	reg_val_offs * 4));
3945	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3946	reg_val_offs * 4));
3947	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
3948	if (r)
3949	goto failed_undo;
3950
3951	amdgpu_ring_commit(ring);
3952	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3953
3954	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
3955
3956	/* don't wait anymore for gpu reset case because this way may
3957	* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
3958	* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
3959	* never return if we keep waiting in virt_kiq_rreg, which cause
3960	* gpu_recover() hang there.
3961	*
3962	* also don't wait anymore for IRQ context
3963	* */
3964	if (r < 1 && (amdgpu_in_reset(adev)))
3965	goto failed_kiq_read;
3966
3967	might_sleep();
3968	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
3969	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
3970	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
3971	}
3972
3973	if (cnt > MAX_KIQ_REG_TRY)
3974	goto failed_kiq_read;
3975
3976	mb();
3977	value = (uint64_t)adev->wb.wb[reg_val_offs] |
3978	(uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
3979	amdgpu_device_wb_free(adev, wb: reg_val_offs);
3980	return value;
3981
3982	failed_undo:
3983	amdgpu_ring_undo(ring);
3984	failed_unlock:
3985	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3986	failed_kiq_read:
3987	if (reg_val_offs)
3988	amdgpu_device_wb_free(adev, wb: reg_val_offs);
3989	pr_err("failed to read gpu clock\n");
3990	return ~0;
3991	}
3992
3993	static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3994	{
3995	uint64_t clock, clock_lo, clock_hi, hi_check;
3996
3997	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
3998	case IP_VERSION(9, 3, 0):
3999	preempt_disable();
4000	clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4001	clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4002	hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4003	/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4004	* roughly every 42 seconds.
4005	*/
4006	if (hi_check != clock_hi) {
4007	clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4008	clock_hi = hi_check;
4009	}
4010	preempt_enable();
4011	clock = clock_lo | (clock_hi << 32ULL);
4012	break;
4013	default:
4014	amdgpu_gfx_off_ctrl(adev, enable: false);
4015	mutex_lock(&adev->gfx.gpu_clock_mutex);
4016	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
4017	IP_VERSION(9, 0, 1) &&
4018	amdgpu_sriov_runtime(adev)) {
4019	clock = gfx_v9_0_kiq_read_clock(adev);
4020	} else {
4021	WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4022	clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4023	((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4024	}
4025	mutex_unlock(lock: &adev->gfx.gpu_clock_mutex);
4026	amdgpu_gfx_off_ctrl(adev, enable: true);
4027	break;
4028	}
4029	return clock;
4030	}
4031
4032	static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4033	uint32_t vmid,
4034	uint32_t gds_base, uint32_t gds_size,
4035	uint32_t gws_base, uint32_t gws_size,
4036	uint32_t oa_base, uint32_t oa_size)
4037	{
4038	struct amdgpu_device *adev = ring->adev;
4039
4040	/* GDS Base */
4041	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4042	SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4043	val: gds_base);
4044
4045	/* GDS Size */
4046	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4047	SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4048	val: gds_size);
4049
4050	/* GWS */
4051	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4052	SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4053	val: gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4054
4055	/* OA */
4056	gfx_v9_0_write_data_to_reg(ring, eng_sel: 0, wc: false,
4057	SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4058	val: (1 << (oa_size + oa_base)) - (1 << oa_base));
4059	}
4060
4061	static const u32 vgpr_init_compute_shader[] =
4062	{
4063	0xb07c0000, 0xbe8000ff,
4064	0x000000f8, 0xbf110800,
4065	0x7e000280, 0x7e020280,
4066	0x7e040280, 0x7e060280,
4067	0x7e080280, 0x7e0a0280,
4068	0x7e0c0280, 0x7e0e0280,
4069	0x80808800, 0xbe803200,
4070	0xbf84fff5, 0xbf9c0000,
4071	0xd28c0001, 0x0001007f,
4072	0xd28d0001, 0x0002027e,
4073	0x10020288, 0xb8810904,
4074	0xb7814000, 0xd1196a01,
4075	0x00000301, 0xbe800087,
4076	0xbefc00c1, 0xd89c4000,
4077	0x00020201, 0xd89cc080,
4078	0x00040401, 0x320202ff,
4079	0x00000800, 0x80808100,
4080	0xbf84fff8, 0x7e020280,
4081	0xbf810000, 0x00000000,
4082	};
4083
4084	static const u32 sgpr_init_compute_shader[] =
4085	{
4086	0xb07c0000, 0xbe8000ff,
4087	0x0000005f, 0xbee50080,
4088	0xbe812c65, 0xbe822c65,
4089	0xbe832c65, 0xbe842c65,
4090	0xbe852c65, 0xb77c0005,
4091	0x80808500, 0xbf84fff8,
4092	0xbe800080, 0xbf810000,
4093	};
4094
4095	static const u32 vgpr_init_compute_shader_arcturus[] = {
4096	0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4097	0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4098	0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4099	0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4100	0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4101	0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4102	0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4103	0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4104	0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4105	0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4106	0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4107	0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4108	0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4109	0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4110	0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4111	0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4112	0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4113	0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4114	0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4115	0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4116	0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4117	0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4118	0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4119	0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4120	0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4121	0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4122	0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4123	0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4124	0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4125	0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4126	0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4127	0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4128	0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4129	0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4130	0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4131	0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4132	0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4133	0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4134	0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4135	0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4136	0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4137	0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4138	0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4139	0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4140	0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4141	0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4142	0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4143	0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4144	0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4145	0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4146	0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4147	0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4148	0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4149	0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4150	0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4151	0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4152	0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4153	0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4154	0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4155	0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4156	0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4157	0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4158	0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4159	0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4160	0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4161	0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4162	0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4163	0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4164	0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4165	0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4166	0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4167	0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4168	0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4169	0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4170	0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4171	0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4172	0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4173	0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4174	0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4175	0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4176	0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4177	0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4178	0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4179	0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4180	0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4181	0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4182	0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4183	0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4184	0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4185	0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4186	0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4187	0xbf84fff8, 0xbf810000,
4188	};
4189
4190	/* When below register arrays changed, please update gpr_reg_size,
4191	and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4192	to cover all gfx9 ASICs */
4193	static const struct soc15_reg_entry vgpr_init_regs[] = {
4194	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4195	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4196	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4197	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4198	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4199	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4200	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4201	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4202	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4203	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4204	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4205	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4206	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4207	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4208	};
4209
4210	static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4211	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4212	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4213	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4214	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4215	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4216	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4217	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4218	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4219	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4220	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4221	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4222	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4223	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4224	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4225	};
4226
4227	static const struct soc15_reg_entry sgpr1_init_regs[] = {
4228	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4229	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4230	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4231	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4232	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4233	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4234	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4235	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4236	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4237	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4238	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4239	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4240	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4241	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4242	};
4243
4244	static const struct soc15_reg_entry sgpr2_init_regs[] = {
4245	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4246	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4247	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4248	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4249	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4250	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4251	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4252	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4253	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4254	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4255	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4256	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4257	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4258	{ SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4259	};
4260
4261	static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4262	{ SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4263	{ SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4264	{ SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4265	{ SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4266	{ SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4267	{ SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4268	{ SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4269	{ SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4270	{ SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4271	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4272	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4273	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4274	{ SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4275	{ SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4276	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4277	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4278	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4279	{ SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4280	{ SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4281	{ SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4282	{ SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4283	{ SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4284	{ SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4285	{ SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4286	{ SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4287	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4288	{ SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4289	{ SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4290	{ SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4291	{ SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4292	{ SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4293	{ SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4294	{ SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4295	};
4296
4297	static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4298	{
4299	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4300	int i, r;
4301
4302	/* only support when RAS is enabled */
4303	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
4304	return 0;
4305
4306	r = amdgpu_ring_alloc(ring, ndw: 7);
4307	if (r) {
4308	DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4309	ring->name, r);
4310	return r;
4311	}
4312
4313	WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4314	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4315
4316	amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4317	amdgpu_ring_write(ring, v: (PACKET3_DMA_DATA_CP_SYNC |
4318	PACKET3_DMA_DATA_DST_SEL(1) |
4319	PACKET3_DMA_DATA_SRC_SEL(2) |
4320	PACKET3_DMA_DATA_ENGINE(0)));
4321	amdgpu_ring_write(ring, v: 0);
4322	amdgpu_ring_write(ring, v: 0);
4323	amdgpu_ring_write(ring, v: 0);
4324	amdgpu_ring_write(ring, v: 0);
4325	amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4326	adev->gds.gds_size);
4327
4328	amdgpu_ring_commit(ring);
4329
4330	for (i = 0; i < adev->usec_timeout; i++) {
4331	if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4332	break;
4333	udelay(1);
4334	}
4335
4336	if (i >= adev->usec_timeout)
4337	r = -ETIMEDOUT;
4338
4339	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4340
4341	return r;
4342	}
4343
4344	static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4345	{
4346	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4347	struct amdgpu_ib ib;
4348	struct dma_fence *f = NULL;
4349	int r, i;
4350	unsigned total_size, vgpr_offset, sgpr_offset;
4351	u64 gpu_addr;
4352
4353	int compute_dim_x = adev->gfx.config.max_shader_engines *
4354	adev->gfx.config.max_cu_per_sh *
4355	adev->gfx.config.max_sh_per_se;
4356	int sgpr_work_group_size = 5;
4357	int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4358	int vgpr_init_shader_size;
4359	const u32 *vgpr_init_shader_ptr;
4360	const struct soc15_reg_entry *vgpr_init_regs_ptr;
4361
4362	/* only support when RAS is enabled */
4363	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
4364	return 0;
4365
4366	/* bail if the compute ring is not ready */
4367	if (!ring->sched.ready)
4368	return 0;
4369
4370	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1)) {
4371	vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4372	vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4373	vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4374	} else {
4375	vgpr_init_shader_ptr = vgpr_init_compute_shader;
4376	vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4377	vgpr_init_regs_ptr = vgpr_init_regs;
4378	}
4379
4380	total_size =
4381	(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4382	total_size +=
4383	(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4384	total_size +=
4385	(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4386	total_size = ALIGN(total_size, 256);
4387	vgpr_offset = total_size;
4388	total_size += ALIGN(vgpr_init_shader_size, 256);
4389	sgpr_offset = total_size;
4390	total_size += sizeof(sgpr_init_compute_shader);
4391
4392	/* allocate an indirect buffer to put the commands in */
4393	memset(&ib, 0, sizeof(ib));
4394	r = amdgpu_ib_get(adev, NULL, size: total_size,
4395	pool: AMDGPU_IB_POOL_DIRECT, ib: &ib);
4396	if (r) {
4397	DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4398	return r;
4399	}
4400
4401	/* load the compute shaders */
4402	for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4403	ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4404
4405	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4406	ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4407
4408	/* init the ib length to 0 */
4409	ib.length_dw = 0;
4410
4411	/* VGPR */
4412	/* write the register state for the compute dispatch */
4413	for (i = 0; i < gpr_reg_size; i++) {
4414	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4415	ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4416	- PACKET3_SET_SH_REG_START;
4417	ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4418	}
4419	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4420	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4421	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4422	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4423	- PACKET3_SET_SH_REG_START;
4424	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4425	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4426
4427	/* write dispatch packet */
4428	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4429	ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4430	ib.ptr[ib.length_dw++] = 1; /* y */
4431	ib.ptr[ib.length_dw++] = 1; /* z */
4432	ib.ptr[ib.length_dw++] =
4433	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4434
4435	/* write CS partial flush packet */
4436	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4437	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4438
4439	/* SGPR1 */
4440	/* write the register state for the compute dispatch */
4441	for (i = 0; i < gpr_reg_size; i++) {
4442	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4443	ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4444	- PACKET3_SET_SH_REG_START;
4445	ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4446	}
4447	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4448	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4449	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4450	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4451	- PACKET3_SET_SH_REG_START;
4452	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4453	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4454
4455	/* write dispatch packet */
4456	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4457	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4458	ib.ptr[ib.length_dw++] = 1; /* y */
4459	ib.ptr[ib.length_dw++] = 1; /* z */
4460	ib.ptr[ib.length_dw++] =
4461	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4462
4463	/* write CS partial flush packet */
4464	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4465	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4466
4467	/* SGPR2 */
4468	/* write the register state for the compute dispatch */
4469	for (i = 0; i < gpr_reg_size; i++) {
4470	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4471	ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4472	- PACKET3_SET_SH_REG_START;
4473	ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4474	}
4475	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4476	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4477	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4478	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4479	- PACKET3_SET_SH_REG_START;
4480	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4481	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4482
4483	/* write dispatch packet */
4484	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4485	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4486	ib.ptr[ib.length_dw++] = 1; /* y */
4487	ib.ptr[ib.length_dw++] = 1; /* z */
4488	ib.ptr[ib.length_dw++] =
4489	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4490
4491	/* write CS partial flush packet */
4492	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4493	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4494
4495	/* shedule the ib on the ring */
4496	r = amdgpu_ib_schedule(ring, num_ibs: 1, ibs: &ib, NULL, f: &f);
4497	if (r) {
4498	DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4499	goto fail;
4500	}
4501
4502	/* wait for the GPU to finish processing the IB */
4503	r = dma_fence_wait(fence: f, intr: false);
4504	if (r) {
4505	DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4506	goto fail;
4507	}
4508
4509	fail:
4510	amdgpu_ib_free(adev, ib: &ib, NULL);
4511	dma_fence_put(fence: f);
4512
4513	return r;
4514	}
4515
4516	static int gfx_v9_0_early_init(void *handle)
4517	{
4518	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4519
4520	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4521
4522	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
4523	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
4524	adev->gfx.num_gfx_rings = 0;
4525	else
4526	adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4527	adev->gfx.xcc_mask = 1;
4528	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4529	AMDGPU_MAX_COMPUTE_RINGS);
4530	gfx_v9_0_set_kiq_pm4_funcs(adev);
4531	gfx_v9_0_set_ring_funcs(adev);
4532	gfx_v9_0_set_irq_funcs(adev);
4533	gfx_v9_0_set_gds_init(adev);
4534	gfx_v9_0_set_rlc_funcs(adev);
4535
4536	/* init rlcg reg access ctrl */
4537	gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4538
4539	return gfx_v9_0_init_microcode(adev);
4540	}
4541
4542	static int gfx_v9_0_ecc_late_init(void *handle)
4543	{
4544	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4545	int r;
4546
4547	/*
4548	* Temp workaround to fix the issue that CP firmware fails to
4549	* update read pointer when CPDMA is writing clearing operation
4550	* to GDS in suspend/resume sequence on several cards. So just
4551	* limit this operation in cold boot sequence.
4552	*/
4553	if ((!adev->in_suspend) &&
4554	(adev->gds.gds_size)) {
4555	r = gfx_v9_0_do_edc_gds_workarounds(adev);
4556	if (r)
4557	return r;
4558	}
4559
4560	/* requires IBs so do in late init after IB pool is initialized */
4561	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
4562	r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4563	else
4564	r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4565
4566	if (r)
4567	return r;
4568
4569	if (adev->gfx.ras &&
4570	adev->gfx.ras->enable_watchdog_timer)
4571	adev->gfx.ras->enable_watchdog_timer(adev);
4572
4573	return 0;
4574	}
4575
4576	static int gfx_v9_0_late_init(void *handle)
4577	{
4578	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4579	int r;
4580
4581	r = amdgpu_irq_get(adev, src: &adev->gfx.priv_reg_irq, type: 0);
4582	if (r)
4583	return r;
4584
4585	r = amdgpu_irq_get(adev, src: &adev->gfx.priv_inst_irq, type: 0);
4586	if (r)
4587	return r;
4588
4589	r = gfx_v9_0_ecc_late_init(handle);
4590	if (r)
4591	return r;
4592
4593	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
4594	gfx_v9_4_2_debug_trap_config_init(adev,
4595	first_vmid: adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4596	else
4597	gfx_v9_0_debug_trap_config_init(adev,
4598	first_vmid: adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4599
4600	return 0;
4601	}
4602
4603	static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4604	{
4605	uint32_t rlc_setting;
4606
4607	/* if RLC is not enabled, do nothing */
4608	rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4609	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4610	return false;
4611
4612	return true;
4613	}
4614
4615	static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4616	{
4617	uint32_t data;
4618	unsigned i;
4619
4620	data = RLC_SAFE_MODE__CMD_MASK;
4621	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4622	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4623
4624	/* wait for RLC_SAFE_MODE */
4625	for (i = 0; i < adev->usec_timeout; i++) {
4626	if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4627	break;
4628	udelay(1);
4629	}
4630	}
4631
4632	static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4633	{
4634	uint32_t data;
4635
4636	data = RLC_SAFE_MODE__CMD_MASK;
4637	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4638	}
4639
4640	static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4641	bool enable)
4642	{
4643	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4644
4645	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4646	gfx_v9_0_enable_gfx_cg_power_gating(adev, enable: true);
4647	if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4648	gfx_v9_0_enable_gfx_pipeline_powergating(adev, enable: true);
4649	} else {
4650	gfx_v9_0_enable_gfx_cg_power_gating(adev, enable: false);
4651	if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4652	gfx_v9_0_enable_gfx_pipeline_powergating(adev, enable: false);
4653	}
4654
4655	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4656	}
4657
4658	static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4659	bool enable)
4660	{
4661	/* TODO: double check if we need to perform under safe mode */
4662	/* gfx_v9_0_enter_rlc_safe_mode(adev); */
4663
4664	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4665	gfx_v9_0_enable_gfx_static_mg_power_gating(adev, enable: true);
4666	else
4667	gfx_v9_0_enable_gfx_static_mg_power_gating(adev, enable: false);
4668
4669	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4670	gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, enable: true);
4671	else
4672	gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, enable: false);
4673
4674	/* gfx_v9_0_exit_rlc_safe_mode(adev); */
4675	}
4676
4677	static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4678	bool enable)
4679	{
4680	uint32_t data, def;
4681
4682	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4683
4684	/* It is disabled by HW by default */
4685	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4686	/* 1 - RLC_CGTT_MGCG_OVERRIDE */
4687	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4688
4689	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 2, 1))
4690	data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4691
4692	data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4693	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4694	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4695
4696	/* only for Vega10 & Raven1 */
4697	data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4698
4699	if (def != data)
4700	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4701
4702	/* MGLS is a global flag to control all MGLS in GFX */
4703	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4704	/* 2 - RLC memory Light sleep */
4705	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4706	def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4707	data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4708	if (def != data)
4709	WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4710	}
4711	/* 3 - CP memory Light sleep */
4712	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4713	def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4714	data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4715	if (def != data)
4716	WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4717	}
4718	}
4719	} else {
4720	/* 1 - MGCG_OVERRIDE */
4721	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4722
4723	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 2, 1))
4724	data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4725
4726	data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4727	RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4728	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4729	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4730
4731	if (def != data)
4732	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4733
4734	/* 2 - disable MGLS in RLC */
4735	data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4736	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4737	data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4738	WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4739	}
4740
4741	/* 3 - disable MGLS in CP */
4742	data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4743	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4744	data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4745	WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4746	}
4747	}
4748
4749	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4750	}
4751
4752	static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4753	bool enable)
4754	{
4755	uint32_t data, def;
4756
4757	if (!adev->gfx.num_gfx_rings)
4758	return;
4759
4760	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4761
4762	/* Enable 3D CGCG/CGLS */
4763	if (enable) {
4764	/* write cmd to clear cgcg/cgls ov */
4765	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4766	/* unset CGCG override */
4767	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4768	/* update CGCG and CGLS override bits */
4769	if (def != data)
4770	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4771
4772	/* enable 3Dcgcg FSM(0x0000363f) */
4773	def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4774
4775	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
4776	data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4777	RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4778	else
4779	data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
4780
4781	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4782	data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4783	RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4784	if (def != data)
4785	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4786
4787	/* set IDLE_POLL_COUNT(0x00900100) */
4788	def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4789	data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4790	(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4791	if (def != data)
4792	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4793	} else {
4794	/* Disable CGCG/CGLS */
4795	def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4796	/* disable cgcg, cgls should be disabled */
4797	data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4798	RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4799	/* disable cgcg and cgls in FSM */
4800	if (def != data)
4801	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4802	}
4803
4804	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4805	}
4806
4807	static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4808	bool enable)
4809	{
4810	uint32_t def, data;
4811
4812	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: 0);
4813
4814	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4815	def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4816	/* unset CGCG override */
4817	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4818	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4819	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4820	else
4821	data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4822	/* update CGCG and CGLS override bits */
4823	if (def != data)
4824	WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4825
4826	/* enable cgcg FSM(0x0000363F) */
4827	def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4828
4829	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1))
4830	data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4831	RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4832	else
4833	data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4834	RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4835	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4836	data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4837	RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4838	if (def != data)
4839	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4840
4841	/* set IDLE_POLL_COUNT(0x00900100) */
4842	def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4843	data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4844	(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4845	if (def != data)
4846	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4847	} else {
4848	def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4849	/* reset CGCG/CGLS bits */
4850	data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4851	/* disable cgcg and cgls in FSM */
4852	if (def != data)
4853	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4854	}
4855
4856	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: 0);
4857	}
4858
4859	static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4860	bool enable)
4861	{
4862	if (enable) {
4863	/* CGCG/CGLS should be enabled after MGCG/MGLS
4864	* === MGCG + MGLS ===
4865	*/
4866	gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4867	/* === CGCG /CGLS for GFX 3D Only === */
4868	gfx_v9_0_update_3d_clock_gating(adev, enable);
4869	/* === CGCG + CGLS === */
4870	gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4871	} else {
4872	/* CGCG/CGLS should be disabled before MGCG/MGLS
4873	* === CGCG + CGLS ===
4874	*/
4875	gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4876	/* === CGCG /CGLS for GFX 3D Only === */
4877	gfx_v9_0_update_3d_clock_gating(adev, enable);
4878	/* === MGCG + MGLS === */
4879	gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4880	}
4881	return 0;
4882	}
4883
4884	static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
4885	unsigned int vmid)
4886	{
4887	u32 reg, data;
4888
4889	reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
4890	if (amdgpu_sriov_is_pp_one_vf(adev))
4891	data = RREG32_NO_KIQ(reg);
4892	else
4893	data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
4894
4895	data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4896	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4897
4898	if (amdgpu_sriov_is_pp_one_vf(adev))
4899	WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
4900	else
4901	WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
4902	}
4903
4904	static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
4905	{
4906	amdgpu_gfx_off_ctrl(adev, enable: false);
4907
4908	gfx_v9_0_update_spm_vmid_internal(adev, vmid);
4909
4910	amdgpu_gfx_off_ctrl(adev, enable: true);
4911	}
4912
4913	static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
4914	uint32_t offset,
4915	struct soc15_reg_rlcg *entries, int arr_size)
4916	{
4917	int i;
4918	uint32_t reg;
4919
4920	if (!entries)
4921	return false;
4922
4923	for (i = 0; i < arr_size; i++) {
4924	const struct soc15_reg_rlcg *entry;
4925
4926	entry = &entries[i];
4927	reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
4928	if (offset == reg)
4929	return true;
4930	}
4931
4932	return false;
4933	}
4934
4935	static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
4936	{
4937	return gfx_v9_0_check_rlcg_range(adev, offset,
4938	entries: (void *)rlcg_access_gc_9_0,
4939	ARRAY_SIZE(rlcg_access_gc_9_0));
4940	}
4941
4942	static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
4943	.is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
4944	.set_safe_mode = gfx_v9_0_set_safe_mode,
4945	.unset_safe_mode = gfx_v9_0_unset_safe_mode,
4946	.init = gfx_v9_0_rlc_init,
4947	.get_csb_size = gfx_v9_0_get_csb_size,
4948	.get_csb_buffer = gfx_v9_0_get_csb_buffer,
4949	.get_cp_table_num = gfx_v9_0_cp_jump_table_num,
4950	.resume = gfx_v9_0_rlc_resume,
4951	.stop = gfx_v9_0_rlc_stop,
4952	.reset = gfx_v9_0_rlc_reset,
4953	.start = gfx_v9_0_rlc_start,
4954	.update_spm_vmid = gfx_v9_0_update_spm_vmid,
4955	.is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
4956	};
4957
4958	static int gfx_v9_0_set_powergating_state(void *handle,
4959	enum amd_powergating_state state)
4960	{
4961	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4962	bool enable = (state == AMD_PG_STATE_GATE);
4963
4964	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
4965	case IP_VERSION(9, 2, 2):
4966	case IP_VERSION(9, 1, 0):
4967	case IP_VERSION(9, 3, 0):
4968	if (!enable)
4969	amdgpu_gfx_off_ctrl(adev, enable: false);
4970
4971	if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
4972	gfx_v9_0_enable_sck_slow_down_on_power_up(adev, enable: true);
4973	gfx_v9_0_enable_sck_slow_down_on_power_down(adev, enable: true);
4974	} else {
4975	gfx_v9_0_enable_sck_slow_down_on_power_up(adev, enable: false);
4976	gfx_v9_0_enable_sck_slow_down_on_power_down(adev, enable: false);
4977	}
4978
4979	if (adev->pg_flags & AMD_PG_SUPPORT_CP)
4980	gfx_v9_0_enable_cp_power_gating(adev, enable: true);
4981	else
4982	gfx_v9_0_enable_cp_power_gating(adev, enable: false);
4983
4984	/* update gfx cgpg state */
4985	gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
4986
4987	/* update mgcg state */
4988	gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
4989
4990	if (enable)
4991	amdgpu_gfx_off_ctrl(adev, enable: true);
4992	break;
4993	case IP_VERSION(9, 2, 1):
4994	amdgpu_gfx_off_ctrl(adev, enable);
4995	break;
4996	default:
4997	break;
4998	}
4999
5000	return 0;
5001	}
5002
5003	static int gfx_v9_0_set_clockgating_state(void *handle,
5004	enum amd_clockgating_state state)
5005	{
5006	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5007
5008	if (amdgpu_sriov_vf(adev))
5009	return 0;
5010
5011	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
5012	case IP_VERSION(9, 0, 1):
5013	case IP_VERSION(9, 2, 1):
5014	case IP_VERSION(9, 4, 0):
5015	case IP_VERSION(9, 2, 2):
5016	case IP_VERSION(9, 1, 0):
5017	case IP_VERSION(9, 4, 1):
5018	case IP_VERSION(9, 3, 0):
5019	case IP_VERSION(9, 4, 2):
5020	gfx_v9_0_update_gfx_clock_gating(adev,
5021	enable: state == AMD_CG_STATE_GATE);
5022	break;
5023	default:
5024	break;
5025	}
5026	return 0;
5027	}
5028
5029	static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)
5030	{
5031	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5032	int data;
5033
5034	if (amdgpu_sriov_vf(adev))
5035	*flags = 0;
5036
5037	/* AMD_CG_SUPPORT_GFX_MGCG */
5038	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5039	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5040	*flags |= AMD_CG_SUPPORT_GFX_MGCG;
5041
5042	/* AMD_CG_SUPPORT_GFX_CGCG */
5043	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5044	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5045	*flags |= AMD_CG_SUPPORT_GFX_CGCG;
5046
5047	/* AMD_CG_SUPPORT_GFX_CGLS */
5048	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5049	*flags |= AMD_CG_SUPPORT_GFX_CGLS;
5050
5051	/* AMD_CG_SUPPORT_GFX_RLC_LS */
5052	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5053	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5054	*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5055
5056	/* AMD_CG_SUPPORT_GFX_CP_LS */
5057	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5058	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5059	*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5060
5061	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 1)) {
5062	/* AMD_CG_SUPPORT_GFX_3D_CGCG */
5063	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5064	if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5065	*flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5066
5067	/* AMD_CG_SUPPORT_GFX_3D_CGLS */
5068	if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5069	*flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5070	}
5071	}
5072
5073	static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5074	{
5075	return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5076	}
5077
5078	static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5079	{
5080	struct amdgpu_device *adev = ring->adev;
5081	u64 wptr;
5082
5083	/* XXX check if swapping is necessary on BE */
5084	if (ring->use_doorbell) {
5085	wptr = atomic64_read(v: (atomic64_t *)ring->wptr_cpu_addr);
5086	} else {
5087	wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5088	wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5089	}
5090
5091	return wptr;
5092	}
5093
5094	static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5095	{
5096	struct amdgpu_device *adev = ring->adev;
5097
5098	if (ring->use_doorbell) {
5099	/* XXX check if swapping is necessary on BE */
5100	atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, i: ring->wptr);
5101	WDOORBELL64(ring->doorbell_index, ring->wptr);
5102	} else {
5103	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5104	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5105	}
5106	}
5107
5108	static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5109	{
5110	struct amdgpu_device *adev = ring->adev;
5111	u32 ref_and_mask, reg_mem_engine;
5112	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5113
5114	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5115	switch (ring->me) {
5116	case 1:
5117	ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5118	break;
5119	case 2:
5120	ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5121	break;
5122	default:
5123	return;
5124	}
5125	reg_mem_engine = 0;
5126	} else {
5127	ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5128	reg_mem_engine = 1; /* pfp */
5129	}
5130
5131	gfx_v9_0_wait_reg_mem(ring, eng_sel: reg_mem_engine, mem_space: 0, opt: 1,
5132	addr0: adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5133	addr1: adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5134	ref: ref_and_mask, mask: ref_and_mask, inv: 0x20);
5135	}
5136
5137	static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5138	struct amdgpu_job *job,
5139	struct amdgpu_ib *ib,
5140	uint32_t flags)
5141	{
5142	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5143	u32 header, control = 0;
5144
5145	if (ib->flags & AMDGPU_IB_FLAG_CE)
5146	header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5147	else
5148	header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5149
5150	control |= ib->length_dw | (vmid << 24);
5151
5152	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5153	control |= INDIRECT_BUFFER_PRE_ENB(1);
5154
5155	if (flags & AMDGPU_IB_PREEMPTED)
5156	control |= INDIRECT_BUFFER_PRE_RESUME(1);
5157
5158	if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5159	gfx_v9_0_ring_emit_de_meta(ring,
5160	resume: (!amdgpu_sriov_vf(ring->adev) &&
5161	flags & AMDGPU_IB_PREEMPTED) ?
5162	true : false,
5163	usegds: job->gds_size > 0 && job->gds_base != 0);
5164	}
5165
5166	amdgpu_ring_write(ring, v: header);
5167	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5168	amdgpu_ring_write(ring,
5169	#ifdef __BIG_ENDIAN
5170	(2 << 0) |
5171	#endif
5172	lower_32_bits(ib->gpu_addr));
5173	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5174	amdgpu_ring_ib_on_emit_cntl(ring);
5175	amdgpu_ring_write(ring, v: control);
5176	}
5177
5178	static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5179	unsigned offset)
5180	{
5181	u32 control = ring->ring[offset];
5182
5183	control |= INDIRECT_BUFFER_PRE_RESUME(1);
5184	ring->ring[offset] = control;
5185	}
5186
5187	static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5188	unsigned offset)
5189	{
5190	struct amdgpu_device *adev = ring->adev;
5191	void *ce_payload_cpu_addr;
5192	uint64_t payload_offset, payload_size;
5193
5194	payload_size = sizeof(struct v9_ce_ib_state);
5195
5196	if (ring->is_mes_queue) {
5197	payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5198	gfx[0].gfx_meta_data) +
5199	offsetof(struct v9_gfx_meta_data, ce_payload);
5200	ce_payload_cpu_addr =
5201	amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5202	} else {
5203	payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5204	ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5205	}
5206
5207	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5208	memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5209	} else {
5210	memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5211	(ring->buf_mask + 1 - offset) << 2);
5212	payload_size -= (ring->buf_mask + 1 - offset) << 2;
5213	memcpy((void *)&ring->ring[0],
5214	ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5215	payload_size);
5216	}
5217	}
5218
5219	static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5220	unsigned offset)
5221	{
5222	struct amdgpu_device *adev = ring->adev;
5223	void *de_payload_cpu_addr;
5224	uint64_t payload_offset, payload_size;
5225
5226	payload_size = sizeof(struct v9_de_ib_state);
5227
5228	if (ring->is_mes_queue) {
5229	payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5230	gfx[0].gfx_meta_data) +
5231	offsetof(struct v9_gfx_meta_data, de_payload);
5232	de_payload_cpu_addr =
5233	amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5234	} else {
5235	payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5236	de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5237	}
5238
5239	((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5240	IB_COMPLETION_STATUS_PREEMPTED;
5241
5242	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5243	memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5244	} else {
5245	memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5246	(ring->buf_mask + 1 - offset) << 2);
5247	payload_size -= (ring->buf_mask + 1 - offset) << 2;
5248	memcpy((void *)&ring->ring[0],
5249	de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5250	payload_size);
5251	}
5252	}
5253
5254	static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5255	struct amdgpu_job *job,
5256	struct amdgpu_ib *ib,
5257	uint32_t flags)
5258	{
5259	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5260	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5261
5262	/* Currently, there is a high possibility to get wave ID mismatch
5263	* between ME and GDS, leading to a hw deadlock, because ME generates
5264	* different wave IDs than the GDS expects. This situation happens
5265	* randomly when at least 5 compute pipes use GDS ordered append.
5266	* The wave IDs generated by ME are also wrong after suspend/resume.
5267	* Those are probably bugs somewhere else in the kernel driver.
5268	*
5269	* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5270	* GDS to 0 for this ring (me/pipe).
5271	*/
5272	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5273	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5274	amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5275	amdgpu_ring_write(ring, v: ring->adev->gds.gds_compute_max_wave_id);
5276	}
5277
5278	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5279	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5280	amdgpu_ring_write(ring,
5281	#ifdef __BIG_ENDIAN
5282	(2 << 0) |
5283	#endif
5284	lower_32_bits(ib->gpu_addr));
5285	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5286	amdgpu_ring_write(ring, v: control);
5287	}
5288
5289	static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5290	u64 seq, unsigned flags)
5291	{
5292	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5293	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5294	bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5295	bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5296	uint32_t dw2 = 0;
5297
5298	/* RELEASE_MEM - flush caches, send int */
5299	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5300
5301	if (writeback) {
5302	dw2 = EOP_TC_NC_ACTION_EN;
5303	} else {
5304	dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5305	EOP_TC_MD_ACTION_EN;
5306	}
5307	dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5308	EVENT_INDEX(5);
5309	if (exec)
5310	dw2 |= EOP_EXEC;
5311
5312	amdgpu_ring_write(ring, v: dw2);
5313	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5314
5315	/*
5316	* the address should be Qword aligned if 64bit write, Dword
5317	* aligned if only send 32bit data low (discard data high)
5318	*/
5319	if (write64bit)
5320	BUG_ON(addr & 0x7);
5321	else
5322	BUG_ON(addr & 0x3);
5323	amdgpu_ring_write(ring, lower_32_bits(addr));
5324	amdgpu_ring_write(ring, upper_32_bits(addr));
5325	amdgpu_ring_write(ring, lower_32_bits(seq));
5326	amdgpu_ring_write(ring, upper_32_bits(seq));
5327	amdgpu_ring_write(ring, v: 0);
5328	}
5329
5330	static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5331	{
5332	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5333	uint32_t seq = ring->fence_drv.sync_seq;
5334	uint64_t addr = ring->fence_drv.gpu_addr;
5335
5336	gfx_v9_0_wait_reg_mem(ring, eng_sel: usepfp, mem_space: 1, opt: 0,
5337	lower_32_bits(addr), upper_32_bits(addr),
5338	ref: seq, mask: 0xffffffff, inv: 4);
5339	}
5340
5341	static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5342	unsigned vmid, uint64_t pd_addr)
5343	{
5344	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5345
5346	/* compute doesn't have PFP */
5347	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5348	/* sync PFP to ME, otherwise we might get invalid PFP reads */
5349	amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5350	amdgpu_ring_write(ring, v: 0x0);
5351	}
5352	}
5353
5354	static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5355	{
5356	return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5357	}
5358
5359	static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5360	{
5361	u64 wptr;
5362
5363	/* XXX check if swapping is necessary on BE */
5364	if (ring->use_doorbell)
5365	wptr = atomic64_read(v: (atomic64_t *)ring->wptr_cpu_addr);
5366	else
5367	BUG();
5368	return wptr;
5369	}
5370
5371	static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5372	{
5373	struct amdgpu_device *adev = ring->adev;
5374
5375	/* XXX check if swapping is necessary on BE */
5376	if (ring->use_doorbell) {
5377	atomic64_set(v: (atomic64_t *)ring->wptr_cpu_addr, i: ring->wptr);
5378	WDOORBELL64(ring->doorbell_index, ring->wptr);
5379	} else{
5380	BUG(); /* only DOORBELL method supported on gfx9 now */
5381	}
5382	}
5383
5384	static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5385	u64 seq, unsigned int flags)
5386	{
5387	struct amdgpu_device *adev = ring->adev;
5388
5389	/* we only allocate 32bit for each seq wb address */
5390	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5391
5392	/* write fence seq to the "addr" */
5393	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5394	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(0) |
5395	WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5396	amdgpu_ring_write(ring, lower_32_bits(addr));
5397	amdgpu_ring_write(ring, upper_32_bits(addr));
5398	amdgpu_ring_write(ring, lower_32_bits(seq));
5399
5400	if (flags & AMDGPU_FENCE_FLAG_INT) {
5401	/* set register to trigger INT */
5402	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5403	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(0) |
5404	WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5405	amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5406	amdgpu_ring_write(ring, v: 0);
5407	amdgpu_ring_write(ring, v: 0x20000000); /* src_id is 178 */
5408	}
5409	}
5410
5411	static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5412	{
5413	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5414	amdgpu_ring_write(ring, v: 0);
5415	}
5416
5417	static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5418	{
5419	struct amdgpu_device *adev = ring->adev;
5420	struct v9_ce_ib_state ce_payload = {0};
5421	uint64_t offset, ce_payload_gpu_addr;
5422	void *ce_payload_cpu_addr;
5423	int cnt;
5424
5425	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5426
5427	if (ring->is_mes_queue) {
5428	offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5429	gfx[0].gfx_meta_data) +
5430	offsetof(struct v9_gfx_meta_data, ce_payload);
5431	ce_payload_gpu_addr =
5432	amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5433	ce_payload_cpu_addr =
5434	amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5435	} else {
5436	offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5437	ce_payload_gpu_addr = amdgpu_csa_vaddr(adev: ring->adev) + offset;
5438	ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5439	}
5440
5441	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5442	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(2) |
5443	WRITE_DATA_DST_SEL(8) |
5444	WR_CONFIRM) |
5445	WRITE_DATA_CACHE_POLICY(0));
5446	amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5447	amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5448
5449	amdgpu_ring_ib_on_emit_ce(ring);
5450
5451	if (resume)
5452	amdgpu_ring_write_multiple(ring, src: ce_payload_cpu_addr,
5453	count_dw: sizeof(ce_payload) >> 2);
5454	else
5455	amdgpu_ring_write_multiple(ring, src: (void *)&ce_payload,
5456	count_dw: sizeof(ce_payload) >> 2);
5457	}
5458
5459	static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5460	{
5461	int i, r = 0;
5462	struct amdgpu_device *adev = ring->adev;
5463	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5464	struct amdgpu_ring *kiq_ring = &kiq->ring;
5465	unsigned long flags;
5466
5467	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5468	return -EINVAL;
5469
5470	spin_lock_irqsave(&kiq->ring_lock, flags);
5471
5472	if (amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->unmap_queues_size)) {
5473	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
5474	return -ENOMEM;
5475	}
5476
5477	/* assert preemption condition */
5478	amdgpu_ring_set_preempt_cond_exec(ring, cond_exec: false);
5479
5480	ring->trail_seq += 1;
5481	amdgpu_ring_alloc(ring, ndw: 13);
5482	gfx_v9_0_ring_emit_fence(ring, addr: ring->trail_fence_gpu_addr,
5483	seq: ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5484
5485	/* assert IB preemption, emit the trailing fence */
5486	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5487	ring->trail_fence_gpu_addr,
5488	ring->trail_seq);
5489
5490	amdgpu_ring_commit(ring: kiq_ring);
5491	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
5492
5493	/* poll the trailing fence */
5494	for (i = 0; i < adev->usec_timeout; i++) {
5495	if (ring->trail_seq ==
5496	le32_to_cpu(*ring->trail_fence_cpu_addr))
5497	break;
5498	udelay(1);
5499	}
5500
5501	if (i >= adev->usec_timeout) {
5502	r = -EINVAL;
5503	DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5504	}
5505
5506	/*reset the CP_VMID_PREEMPT after trailing fence*/
5507	amdgpu_ring_emit_wreg(ring,
5508	SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5509	0x0);
5510	amdgpu_ring_commit(ring);
5511
5512	/* deassert preemption condition */
5513	amdgpu_ring_set_preempt_cond_exec(ring, cond_exec: true);
5514	return r;
5515	}
5516
5517	static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5518	{
5519	struct amdgpu_device *adev = ring->adev;
5520	struct v9_de_ib_state de_payload = {0};
5521	uint64_t offset, gds_addr, de_payload_gpu_addr;
5522	void *de_payload_cpu_addr;
5523	int cnt;
5524
5525	if (ring->is_mes_queue) {
5526	offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5527	gfx[0].gfx_meta_data) +
5528	offsetof(struct v9_gfx_meta_data, de_payload);
5529	de_payload_gpu_addr =
5530	amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5531	de_payload_cpu_addr =
5532	amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5533
5534	offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5535	gfx[0].gds_backup) +
5536	offsetof(struct v9_gfx_meta_data, de_payload);
5537	gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5538	} else {
5539	offset = offsetof(struct v9_gfx_meta_data, de_payload);
5540	de_payload_gpu_addr = amdgpu_csa_vaddr(adev: ring->adev) + offset;
5541	de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5542
5543	gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5544	AMDGPU_CSA_SIZE - adev->gds.gds_size,
5545	PAGE_SIZE);
5546	}
5547
5548	if (usegds) {
5549	de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5550	de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5551	}
5552
5553	cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5554	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5555	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(1) |
5556	WRITE_DATA_DST_SEL(8) |
5557	WR_CONFIRM) |
5558	WRITE_DATA_CACHE_POLICY(0));
5559	amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5560	amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5561
5562	amdgpu_ring_ib_on_emit_de(ring);
5563	if (resume)
5564	amdgpu_ring_write_multiple(ring, src: de_payload_cpu_addr,
5565	count_dw: sizeof(de_payload) >> 2);
5566	else
5567	amdgpu_ring_write_multiple(ring, src: (void *)&de_payload,
5568	count_dw: sizeof(de_payload) >> 2);
5569	}
5570
5571	static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5572	bool secure)
5573	{
5574	uint32_t v = secure ? FRAME_TMZ : 0;
5575
5576	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5577	amdgpu_ring_write(ring, v: v | FRAME_CMD(start ? 0 : 1));
5578	}
5579
5580	static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5581	{
5582	uint32_t dw2 = 0;
5583
5584	gfx_v9_0_ring_emit_ce_meta(ring,
5585	resume: (!amdgpu_sriov_vf(ring->adev) &&
5586	flags & AMDGPU_IB_PREEMPTED) ? true : false);
5587
5588	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5589	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5590	/* set load_global_config & load_global_uconfig */
5591	dw2 |= 0x8001;
5592	/* set load_cs_sh_regs */
5593	dw2 |= 0x01000000;
5594	/* set load_per_context_state & load_gfx_sh_regs for GFX */
5595	dw2 |= 0x10002;
5596
5597	/* set load_ce_ram if preamble presented */
5598	if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5599	dw2 |= 0x10000000;
5600	} else {
5601	/* still load_ce_ram if this is the first time preamble presented
5602	* although there is no context switch happens.
5603	*/
5604	if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5605	dw2 |= 0x10000000;
5606	}
5607
5608	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5609	amdgpu_ring_write(ring, v: dw2);
5610	amdgpu_ring_write(ring, v: 0);
5611	}
5612
5613	static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5614	uint64_t addr)
5615	{
5616	unsigned ret;
5617	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5618	amdgpu_ring_write(ring, lower_32_bits(addr));
5619	amdgpu_ring_write(ring, upper_32_bits(addr));
5620	/* discard following DWs if *cond_exec_gpu_addr==0 */
5621	amdgpu_ring_write(ring, v: 0);
5622	ret = ring->wptr & ring->buf_mask;
5623	/* patch dummy value later */
5624	amdgpu_ring_write(ring, v: 0);
5625	return ret;
5626	}
5627
5628	static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5629	uint32_t reg_val_offs)
5630	{
5631	struct amdgpu_device *adev = ring->adev;
5632
5633	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5634	amdgpu_ring_write(ring, v: 0 | /* src: register*/
5635	(5 << 8) | /* dst: memory */
5636	(1 << 20)); /* write confirm */
5637	amdgpu_ring_write(ring, v: reg);
5638	amdgpu_ring_write(ring, v: 0);
5639	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5640	reg_val_offs * 4));
5641	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5642	reg_val_offs * 4));
5643	}
5644
5645	static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5646	uint32_t val)
5647	{
5648	uint32_t cmd = 0;
5649
5650	switch (ring->funcs->type) {
5651	case AMDGPU_RING_TYPE_GFX:
5652	cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5653	break;
5654	case AMDGPU_RING_TYPE_KIQ:
5655	cmd = (1 << 16); /* no inc addr */
5656	break;
5657	default:
5658	cmd = WR_CONFIRM;
5659	break;
5660	}
5661	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5662	amdgpu_ring_write(ring, v: cmd);
5663	amdgpu_ring_write(ring, v: reg);
5664	amdgpu_ring_write(ring, v: 0);
5665	amdgpu_ring_write(ring, v: val);
5666	}
5667
5668	static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5669	uint32_t val, uint32_t mask)
5670	{
5671	gfx_v9_0_wait_reg_mem(ring, eng_sel: 0, mem_space: 0, opt: 0, addr0: reg, addr1: 0, ref: val, mask, inv: 0x20);
5672	}
5673
5674	static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5675	uint32_t reg0, uint32_t reg1,
5676	uint32_t ref, uint32_t mask)
5677	{
5678	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5679	struct amdgpu_device *adev = ring->adev;
5680	bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5681	adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5682
5683	if (fw_version_ok)
5684	gfx_v9_0_wait_reg_mem(ring, eng_sel: usepfp, mem_space: 0, opt: 1, addr0: reg0, addr1: reg1,
5685	ref, mask, inv: 0x20);
5686	else
5687	amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, val0: reg1,
5688	reg1: ref, val1: mask);
5689	}
5690
5691	static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5692	{
5693	struct amdgpu_device *adev = ring->adev;
5694	uint32_t value = 0;
5695
5696	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5697	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5698	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5699	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5700	WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5701	}
5702
5703	static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5704	enum amdgpu_interrupt_state state)
5705	{
5706	switch (state) {
5707	case AMDGPU_IRQ_STATE_DISABLE:
5708	case AMDGPU_IRQ_STATE_ENABLE:
5709	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5710	TIME_STAMP_INT_ENABLE,
5711	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5712	break;
5713	default:
5714	break;
5715	}
5716	}
5717
5718	static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5719	int me, int pipe,
5720	enum amdgpu_interrupt_state state)
5721	{
5722	u32 mec_int_cntl, mec_int_cntl_reg;
5723
5724	/*
5725	* amdgpu controls only the first MEC. That's why this function only
5726	* handles the setting of interrupts for this specific MEC. All other
5727	* pipes' interrupts are set by amdkfd.
5728	*/
5729
5730	if (me == 1) {
5731	switch (pipe) {
5732	case 0:
5733	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5734	break;
5735	case 1:
5736	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5737	break;
5738	case 2:
5739	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5740	break;
5741	case 3:
5742	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5743	break;
5744	default:
5745	DRM_DEBUG("invalid pipe %d\n", pipe);
5746	return;
5747	}
5748	} else {
5749	DRM_DEBUG("invalid me %d\n", me);
5750	return;
5751	}
5752
5753	switch (state) {
5754	case AMDGPU_IRQ_STATE_DISABLE:
5755	mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
5756	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5757	TIME_STAMP_INT_ENABLE, 0);
5758	WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5759	break;
5760	case AMDGPU_IRQ_STATE_ENABLE:
5761	mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5762	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5763	TIME_STAMP_INT_ENABLE, 1);
5764	WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5765	break;
5766	default:
5767	break;
5768	}
5769	}
5770
5771	static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5772	struct amdgpu_irq_src *source,
5773	unsigned type,
5774	enum amdgpu_interrupt_state state)
5775	{
5776	switch (state) {
5777	case AMDGPU_IRQ_STATE_DISABLE:
5778	case AMDGPU_IRQ_STATE_ENABLE:
5779	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5780	PRIV_REG_INT_ENABLE,
5781	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5782	break;
5783	default:
5784	break;
5785	}
5786
5787	return 0;
5788	}
5789
5790	static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5791	struct amdgpu_irq_src *source,
5792	unsigned type,
5793	enum amdgpu_interrupt_state state)
5794	{
5795	switch (state) {
5796	case AMDGPU_IRQ_STATE_DISABLE:
5797	case AMDGPU_IRQ_STATE_ENABLE:
5798	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5799	PRIV_INSTR_INT_ENABLE,
5800	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5801	break;
5802	default:
5803	break;
5804	}
5805
5806	return 0;
5807	}
5808
5809	#define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
5810	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5811	CP_ECC_ERROR_INT_ENABLE, 1)
5812
5813	#define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
5814	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5815	CP_ECC_ERROR_INT_ENABLE, 0)
5816
5817	static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5818	struct amdgpu_irq_src *source,
5819	unsigned type,
5820	enum amdgpu_interrupt_state state)
5821	{
5822	switch (state) {
5823	case AMDGPU_IRQ_STATE_DISABLE:
5824	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5825	CP_ECC_ERROR_INT_ENABLE, 0);
5826	DISABLE_ECC_ON_ME_PIPE(1, 0);
5827	DISABLE_ECC_ON_ME_PIPE(1, 1);
5828	DISABLE_ECC_ON_ME_PIPE(1, 2);
5829	DISABLE_ECC_ON_ME_PIPE(1, 3);
5830	break;
5831
5832	case AMDGPU_IRQ_STATE_ENABLE:
5833	WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5834	CP_ECC_ERROR_INT_ENABLE, 1);
5835	ENABLE_ECC_ON_ME_PIPE(1, 0);
5836	ENABLE_ECC_ON_ME_PIPE(1, 1);
5837	ENABLE_ECC_ON_ME_PIPE(1, 2);
5838	ENABLE_ECC_ON_ME_PIPE(1, 3);
5839	break;
5840	default:
5841	break;
5842	}
5843
5844	return 0;
5845	}
5846
5847
5848	static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5849	struct amdgpu_irq_src *src,
5850	unsigned type,
5851	enum amdgpu_interrupt_state state)
5852	{
5853	switch (type) {
5854	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5855	gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5856	break;
5857	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5858	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 0, state);
5859	break;
5860	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5861	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 1, state);
5862	break;
5863	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5864	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 2, state);
5865	break;
5866	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5867	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 1, pipe: 3, state);
5868	break;
5869	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5870	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 0, state);
5871	break;
5872	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5873	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 1, state);
5874	break;
5875	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5876	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 2, state);
5877	break;
5878	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5879	gfx_v9_0_set_compute_eop_interrupt_state(adev, me: 2, pipe: 3, state);
5880	break;
5881	default:
5882	break;
5883	}
5884	return 0;
5885	}
5886
5887	static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5888	struct amdgpu_irq_src *source,
5889	struct amdgpu_iv_entry *entry)
5890	{
5891	int i;
5892	u8 me_id, pipe_id, queue_id;
5893	struct amdgpu_ring *ring;
5894
5895	DRM_DEBUG("IH: CP EOP\n");
5896	me_id = (entry->ring_id & 0x0c) >> 2;
5897	pipe_id = (entry->ring_id & 0x03) >> 0;
5898	queue_id = (entry->ring_id & 0x70) >> 4;
5899
5900	switch (me_id) {
5901	case 0:
5902	if (adev->gfx.num_gfx_rings) {
5903	if (!adev->gfx.mcbp) {
5904	amdgpu_fence_process(ring: &adev->gfx.gfx_ring[0]);
5905	} else if (!amdgpu_mcbp_handle_trailing_fence_irq(mux: &adev->gfx.muxer)) {
5906	/* Fence signals are handled on the software rings*/
5907	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
5908	amdgpu_fence_process(ring: &adev->gfx.sw_gfx_ring[i]);
5909	}
5910	}
5911	break;
5912	case 1:
5913	case 2:
5914	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5915	ring = &adev->gfx.compute_ring[i];
5916	/* Per-queue interrupt is supported for MEC starting from VI.
5917	* The interrupt can only be enabled/disabled per pipe instead of per queue.
5918	*/
5919	if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5920	amdgpu_fence_process(ring);
5921	}
5922	break;
5923	}
5924	return 0;
5925	}
5926
5927	static void gfx_v9_0_fault(struct amdgpu_device *adev,
5928	struct amdgpu_iv_entry *entry)
5929	{
5930	u8 me_id, pipe_id, queue_id;
5931	struct amdgpu_ring *ring;
5932	int i;
5933
5934	me_id = (entry->ring_id & 0x0c) >> 2;
5935	pipe_id = (entry->ring_id & 0x03) >> 0;
5936	queue_id = (entry->ring_id & 0x70) >> 4;
5937
5938	switch (me_id) {
5939	case 0:
5940	drm_sched_fault(sched: &adev->gfx.gfx_ring[0].sched);
5941	break;
5942	case 1:
5943	case 2:
5944	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5945	ring = &adev->gfx.compute_ring[i];
5946	if (ring->me == me_id && ring->pipe == pipe_id &&
5947	ring->queue == queue_id)
5948	drm_sched_fault(sched: &ring->sched);
5949	}
5950	break;
5951	}
5952	}
5953
5954	static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5955	struct amdgpu_irq_src *source,
5956	struct amdgpu_iv_entry *entry)
5957	{
5958	DRM_ERROR("Illegal register access in command stream\n");
5959	gfx_v9_0_fault(adev, entry);
5960	return 0;
5961	}
5962
5963	static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5964	struct amdgpu_irq_src *source,
5965	struct amdgpu_iv_entry *entry)
5966	{
5967	DRM_ERROR("Illegal instruction in command stream\n");
5968	gfx_v9_0_fault(adev, entry);
5969	return 0;
5970	}
5971
5972
5973	static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5974	{ "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5975	SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5976	SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5977	},
5978	{ "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5979	SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5980	SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5981	},
5982	{ "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5983	SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5984	0, 0
5985	},
5986	{ "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5987	SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5988	0, 0
5989	},
5990	{ "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5991	SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5992	SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5993	},
5994	{ "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5995	SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5996	0, 0
5997	},
5998	{ "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5999	SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6000	SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6001	},
6002	{ "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6003	SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6004	SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6005	},
6006	{ "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6007	SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6008	0, 0
6009	},
6010	{ "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6011	SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6012	0, 0
6013	},
6014	{ "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6015	SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6016	0, 0
6017	},
6018	{ "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6019	SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6020	SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6021	},
6022	{ "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6023	SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6024	0, 0
6025	},
6026	{ "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6027	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6028	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6029	},
6030	{ "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6031	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6032	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6033	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6034	},
6035	{ "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6036	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6037	SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6038	0, 0
6039	},
6040	{ "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6041	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6042	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6043	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6044	},
6045	{ "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6046	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6047	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6048	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6049	},
6050	{ "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6051	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6052	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6053	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6054	},
6055	{ "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6056	SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6057	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6058	SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6059	},
6060	{ "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6061	SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6062	0, 0
6063	},
6064	{ "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6065	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6066	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6067	},
6068	{ "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6069	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6070	0, 0
6071	},
6072	{ "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6073	SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6074	0, 0
6075	},
6076	{ "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6077	SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6078	0, 0
6079	},
6080	{ "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6081	SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6082	0, 0
6083	},
6084	{ "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6085	SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6086	0, 0
6087	},
6088	{ "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6089	SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6090	0, 0
6091	},
6092	{ "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6093	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6094	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6095	},
6096	{ "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6097	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6098	SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6099	},
6100	{ "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6101	SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6102	SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6103	},
6104	{ "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6105	SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6106	SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6107	},
6108	{ "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6109	SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6110	SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6111	},
6112	{ "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6113	SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6114	0, 0
6115	},
6116	{ "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6117	SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6118	0, 0
6119	},
6120	{ "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6121	SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6122	0, 0
6123	},
6124	{ "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6125	SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6126	0, 0
6127	},
6128	{ "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6129	SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6130	0, 0
6131	},
6132	{ "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6133	SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6134	0, 0
6135	},
6136	{ "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6137	SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6138	0, 0
6139	},
6140	{ "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6141	SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6142	0, 0
6143	},
6144	{ "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6145	SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6146	0, 0
6147	},
6148	{ "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6149	SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6150	0, 0
6151	},
6152	{ "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6153	SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6154	0, 0
6155	},
6156	{ "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6157	SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6158	0, 0
6159	},
6160	{ "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6161	SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6162	0, 0
6163	},
6164	{ "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6165	SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6166	0, 0
6167	},
6168	{ "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6169	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6170	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6171	},
6172	{ "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6173	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6174	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6175	},
6176	{ "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6177	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6178	0, 0
6179	},
6180	{ "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6181	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6182	0, 0
6183	},
6184	{ "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6185	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6186	0, 0
6187	},
6188	{ "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6189	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6190	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6191	},
6192	{ "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6193	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6194	SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6195	},
6196	{ "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6197	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6198	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6199	},
6200	{ "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6201	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6202	SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6203	},
6204	{ "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6205	SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6206	0, 0
6207	},
6208	{ "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6209	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6210	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6211	},
6212	{ "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6213	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6214	SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6215	},
6216	{ "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6217	SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6218	SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6219	},
6220	{ "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6221	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6222	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6223	},
6224	{ "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6225	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6226	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6227	},
6228	{ "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6229	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6230	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6231	},
6232	{ "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6233	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6234	SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6235	},
6236	{ "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6237	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6238	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6239	},
6240	{ "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6241	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6242	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6243	},
6244	{ "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6245	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6246	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6247	},
6248	{ "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6249	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6250	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6251	},
6252	{ "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6253	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6254	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6255	},
6256	{ "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6257	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6258	SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6259	},
6260	{ "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6261	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6262	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6263	},
6264	{ "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6265	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6266	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6267	},
6268	{ "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6269	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6270	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6271	},
6272	{ "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6273	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6274	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6275	},
6276	{ "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6277	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6278	0, 0
6279	},
6280	{ "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6281	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6282	0, 0
6283	},
6284	{ "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6285	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6286	0, 0
6287	},
6288	{ "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6289	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6290	0, 0
6291	},
6292	{ "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6293	SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6294	0, 0
6295	},
6296	{ "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6297	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6298	SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6299	},
6300	{ "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6301	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6302	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6303	},
6304	{ "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6305	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6306	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6307	},
6308	{ "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6309	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6310	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6311	},
6312	{ "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6313	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6314	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6315	},
6316	{ "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6317	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6318	0, 0
6319	},
6320	{ "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6321	SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6322	0, 0
6323	},
6324	{ "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6325	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6326	0, 0
6327	},
6328	{ "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6329	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6330	0, 0
6331	},
6332	{ "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6333	SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6334	0, 0
6335	},
6336	{ "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6337	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6338	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6339	},
6340	{ "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6341	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6342	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6343	},
6344	{ "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6345	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6346	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6347	},
6348	{ "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6349	SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6350	SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6351	},
6352	{ "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6353	SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6354	SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6355	},
6356	{ "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6357	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6358	0, 0
6359	},
6360	{ "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6361	SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6362	0, 0
6363	},
6364	{ "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6365	SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6366	0, 0
6367	},
6368	{ "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6369	SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6370	0, 0
6371	},
6372	{ "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6373	SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6374	0, 0
6375	},
6376	{ "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6377	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6378	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6379	},
6380	{ "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6381	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6382	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6383	},
6384	{ "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6385	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6386	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6387	},
6388	{ "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6389	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6390	0, 0
6391	},
6392	{ "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6393	SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6394	0, 0
6395	},
6396	{ "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6397	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6398	0, 0
6399	},
6400	{ "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6401	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6402	0, 0
6403	},
6404	{ "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6405	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6406	0, 0
6407	},
6408	{ "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6409	SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6410	0, 0
6411	}
6412	};
6413
6414	static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6415	void *inject_if, uint32_t instance_mask)
6416	{
6417	struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6418	int ret;
6419	struct ta_ras_trigger_error_input block_info = { 0 };
6420
6421	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
6422	return -EINVAL;
6423
6424	if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6425	return -EINVAL;
6426
6427	if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6428	return -EPERM;
6429
6430	if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6431	info->head.type)) {
6432	DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6433	ras_gfx_subblocks[info->head.sub_block_index].name,
6434	info->head.type);
6435	return -EPERM;
6436	}
6437
6438	if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6439	info->head.type)) {
6440	DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6441	ras_gfx_subblocks[info->head.sub_block_index].name,
6442	info->head.type);
6443	return -EPERM;
6444	}
6445
6446	block_info.block_id = amdgpu_ras_block_to_ta(block: info->head.block);
6447	block_info.sub_block_index =
6448	ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6449	block_info.inject_error_type = amdgpu_ras_error_to_ta(error: info->head.type);
6450	block_info.address = info->address;
6451	block_info.value = info->value;
6452
6453	mutex_lock(&adev->grbm_idx_mutex);
6454	ret = psp_ras_trigger_error(psp: &adev->psp, info: &block_info, instance_mask);
6455	mutex_unlock(lock: &adev->grbm_idx_mutex);
6456
6457	return ret;
6458	}
6459
6460	static const char * const vml2_mems[] = {
6461	"UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6462	"UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6463	"UTC_VML2_BANK_CACHE_0_4K_MEM0",
6464	"UTC_VML2_BANK_CACHE_0_4K_MEM1",
6465	"UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6466	"UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6467	"UTC_VML2_BANK_CACHE_1_4K_MEM0",
6468	"UTC_VML2_BANK_CACHE_1_4K_MEM1",
6469	"UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6470	"UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6471	"UTC_VML2_BANK_CACHE_2_4K_MEM0",
6472	"UTC_VML2_BANK_CACHE_2_4K_MEM1",
6473	"UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6474	"UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6475	"UTC_VML2_BANK_CACHE_3_4K_MEM0",
6476	"UTC_VML2_BANK_CACHE_3_4K_MEM1",
6477	};
6478
6479	static const char * const vml2_walker_mems[] = {
6480	"UTC_VML2_CACHE_PDE0_MEM0",
6481	"UTC_VML2_CACHE_PDE0_MEM1",
6482	"UTC_VML2_CACHE_PDE1_MEM0",
6483	"UTC_VML2_CACHE_PDE1_MEM1",
6484	"UTC_VML2_CACHE_PDE2_MEM0",
6485	"UTC_VML2_CACHE_PDE2_MEM1",
6486	"UTC_VML2_RDIF_LOG_FIFO",
6487	};
6488
6489	static const char * const atc_l2_cache_2m_mems[] = {
6490	"UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6491	"UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6492	"UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6493	"UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6494	};
6495
6496	static const char *atc_l2_cache_4k_mems[] = {
6497	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6498	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6499	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6500	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6501	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6502	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6503	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6504	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6505	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6506	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6507	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6508	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6509	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6510	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6511	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6512	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6513	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6514	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6515	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6516	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6517	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6518	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6519	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6520	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6521	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6522	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6523	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6524	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6525	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6526	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6527	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6528	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6529	};
6530
6531	static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6532	struct ras_err_data *err_data)
6533	{
6534	uint32_t i, data;
6535	uint32_t sec_count, ded_count;
6536
6537	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6538	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6539	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6540	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6541	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6542	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6543	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6544	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6545
6546	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6547	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6548	data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6549
6550	sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6551	if (sec_count) {
6552	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6553	"SEC %d\n", i, vml2_mems[i], sec_count);
6554	err_data->ce_count += sec_count;
6555	}
6556
6557	ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6558	if (ded_count) {
6559	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6560	"DED %d\n", i, vml2_mems[i], ded_count);
6561	err_data->ue_count += ded_count;
6562	}
6563	}
6564
6565	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6566	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6567	data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6568
6569	sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6570	SEC_COUNT);
6571	if (sec_count) {
6572	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6573	"SEC %d\n", i, vml2_walker_mems[i], sec_count);
6574	err_data->ce_count += sec_count;
6575	}
6576
6577	ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6578	DED_COUNT);
6579	if (ded_count) {
6580	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6581	"DED %d\n", i, vml2_walker_mems[i], ded_count);
6582	err_data->ue_count += ded_count;
6583	}
6584	}
6585
6586	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6587	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6588	data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6589
6590	sec_count = (data & 0x00006000L) >> 0xd;
6591	if (sec_count) {
6592	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6593	"SEC %d\n", i, atc_l2_cache_2m_mems[i],
6594	sec_count);
6595	err_data->ce_count += sec_count;
6596	}
6597	}
6598
6599	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6600	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6601	data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6602
6603	sec_count = (data & 0x00006000L) >> 0xd;
6604	if (sec_count) {
6605	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6606	"SEC %d\n", i, atc_l2_cache_4k_mems[i],
6607	sec_count);
6608	err_data->ce_count += sec_count;
6609	}
6610
6611	ded_count = (data & 0x00018000L) >> 0xf;
6612	if (ded_count) {
6613	dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6614	"DED %d\n", i, atc_l2_cache_4k_mems[i],
6615	ded_count);
6616	err_data->ue_count += ded_count;
6617	}
6618	}
6619
6620	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6621	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6622	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6623	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6624
6625	return 0;
6626	}
6627
6628	static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6629	const struct soc15_reg_entry *reg,
6630	uint32_t se_id, uint32_t inst_id, uint32_t value,
6631	uint32_t *sec_count, uint32_t *ded_count)
6632	{
6633	uint32_t i;
6634	uint32_t sec_cnt, ded_cnt;
6635
6636	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6637	if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6638	gfx_v9_0_ras_fields[i].seg != reg->seg ||
6639	gfx_v9_0_ras_fields[i].inst != reg->inst)
6640	continue;
6641
6642	sec_cnt = (value &
6643	gfx_v9_0_ras_fields[i].sec_count_mask) >>
6644	gfx_v9_0_ras_fields[i].sec_count_shift;
6645	if (sec_cnt) {
6646	dev_info(adev->dev, "GFX SubBlock %s, "
6647	"Instance[%d][%d], SEC %d\n",
6648	gfx_v9_0_ras_fields[i].name,
6649	se_id, inst_id,
6650	sec_cnt);
6651	*sec_count += sec_cnt;
6652	}
6653
6654	ded_cnt = (value &
6655	gfx_v9_0_ras_fields[i].ded_count_mask) >>
6656	gfx_v9_0_ras_fields[i].ded_count_shift;
6657	if (ded_cnt) {
6658	dev_info(adev->dev, "GFX SubBlock %s, "
6659	"Instance[%d][%d], DED %d\n",
6660	gfx_v9_0_ras_fields[i].name,
6661	se_id, inst_id,
6662	ded_cnt);
6663	*ded_count += ded_cnt;
6664	}
6665	}
6666
6667	return 0;
6668	}
6669
6670	static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6671	{
6672	int i, j, k;
6673
6674	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
6675	return;
6676
6677	/* read back registers to clear the counters */
6678	mutex_lock(&adev->grbm_idx_mutex);
6679	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6680	for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6681	for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6682	amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
6683	RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6684	}
6685	}
6686	}
6687	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6688	mutex_unlock(lock: &adev->grbm_idx_mutex);
6689
6690	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6691	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6692	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6693	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6694	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6695	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6696	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6697	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6698
6699	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6700	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6701	RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6702	}
6703
6704	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6705	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6706	RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6707	}
6708
6709	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6710	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6711	RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6712	}
6713
6714	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6715	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6716	RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6717	}
6718
6719	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6720	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6721	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6722	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6723	}
6724
6725	static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6726	void *ras_error_status)
6727	{
6728	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6729	uint32_t sec_count = 0, ded_count = 0;
6730	uint32_t i, j, k;
6731	uint32_t reg_value;
6732
6733	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
6734	return;
6735
6736	err_data->ue_count = 0;
6737	err_data->ce_count = 0;
6738
6739	mutex_lock(&adev->grbm_idx_mutex);
6740
6741	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6742	for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6743	for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6744	amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
6745	reg_value =
6746	RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6747	if (reg_value)
6748	gfx_v9_0_ras_error_count(adev,
6749	reg: &gfx_v9_0_edc_counter_regs[i],
6750	se_id: j, inst_id: k, value: reg_value,
6751	sec_count: &sec_count, ded_count: &ded_count);
6752	}
6753	}
6754	}
6755
6756	err_data->ce_count += sec_count;
6757	err_data->ue_count += ded_count;
6758
6759	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
6760	mutex_unlock(lock: &adev->grbm_idx_mutex);
6761
6762	gfx_v9_0_query_utc_edc_status(adev, err_data);
6763	}
6764
6765	static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
6766	{
6767	const unsigned int cp_coher_cntl =
6768	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
6769	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
6770	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
6771	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
6772	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
6773
6774	/* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
6775	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6776	amdgpu_ring_write(ring, v: cp_coher_cntl); /* CP_COHER_CNTL */
6777	amdgpu_ring_write(ring, v: 0xffffffff); /* CP_COHER_SIZE */
6778	amdgpu_ring_write(ring, v: 0xffffff); /* CP_COHER_SIZE_HI */
6779	amdgpu_ring_write(ring, v: 0); /* CP_COHER_BASE */
6780	amdgpu_ring_write(ring, v: 0); /* CP_COHER_BASE_HI */
6781	amdgpu_ring_write(ring, v: 0x0000000A); /* POLL_INTERVAL */
6782	}
6783
6784	static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6785	uint32_t pipe, bool enable)
6786	{
6787	struct amdgpu_device *adev = ring->adev;
6788	uint32_t val;
6789	uint32_t wcl_cs_reg;
6790
6791	/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6792	val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
6793
6794	switch (pipe) {
6795	case 0:
6796	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
6797	break;
6798	case 1:
6799	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
6800	break;
6801	case 2:
6802	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
6803	break;
6804	case 3:
6805	wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
6806	break;
6807	default:
6808	DRM_DEBUG("invalid pipe %d\n", pipe);
6809	return;
6810	}
6811
6812	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6813
6814	}
6815	static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6816	{
6817	struct amdgpu_device *adev = ring->adev;
6818	uint32_t val;
6819	int i;
6820
6821
6822	/* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6823	* number of gfx waves. Setting 5 bit will make sure gfx only gets
6824	* around 25% of gpu resources.
6825	*/
6826	val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6827	amdgpu_ring_emit_wreg(ring,
6828	SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
6829	val);
6830
6831	/* Restrict waves for normal/low priority compute queues as well
6832	* to get best QoS for high priority compute jobs.
6833	*
6834	* amdgpu controls only 1st ME(0-3 CS pipes).
6835	*/
6836	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6837	if (i != ring->pipe)
6838	gfx_v9_0_emit_wave_limit_cs(ring, pipe: i, enable);
6839
6840	}
6841	}
6842
6843	static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6844	.name = "gfx_v9_0",
6845	.early_init = gfx_v9_0_early_init,
6846	.late_init = gfx_v9_0_late_init,
6847	.sw_init = gfx_v9_0_sw_init,
6848	.sw_fini = gfx_v9_0_sw_fini,
6849	.hw_init = gfx_v9_0_hw_init,
6850	.hw_fini = gfx_v9_0_hw_fini,
6851	.suspend = gfx_v9_0_suspend,
6852	.resume = gfx_v9_0_resume,
6853	.is_idle = gfx_v9_0_is_idle,
6854	.wait_for_idle = gfx_v9_0_wait_for_idle,
6855	.soft_reset = gfx_v9_0_soft_reset,
6856	.set_clockgating_state = gfx_v9_0_set_clockgating_state,
6857	.set_powergating_state = gfx_v9_0_set_powergating_state,
6858	.get_clockgating_state = gfx_v9_0_get_clockgating_state,
6859	};
6860
6861	static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6862	.type = AMDGPU_RING_TYPE_GFX,
6863	.align_mask = 0xff,
6864	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6865	.support_64bit_ptrs = true,
6866	.secure_submission_supported = true,
6867	.get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6868	.get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6869	.set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6870	.emit_frame_size = /* totally 242 maximum if 16 IBs */
6871	5 + /* COND_EXEC */
6872	7 + /* PIPELINE_SYNC */
6873	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6874	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6875	2 + /* VM_FLUSH */
6876	8 + /* FENCE for VM_FLUSH */
6877	20 + /* GDS switch */
6878	4 + /* double SWITCH_BUFFER,
6879	the first COND_EXEC jump to the place just
6880	prior to this double SWITCH_BUFFER */
6881	5 + /* COND_EXEC */
6882	7 + /* HDP_flush */
6883	4 + /* VGT_flush */
6884	14 + /* CE_META */
6885	31 + /* DE_META */
6886	3 + /* CNTX_CTRL */
6887	5 + /* HDP_INVL */
6888	8 + 8 + /* FENCE x2 */
6889	2 + /* SWITCH_BUFFER */
6890	7, /* gfx_v9_0_emit_mem_sync */
6891	.emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6892	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6893	.emit_fence = gfx_v9_0_ring_emit_fence,
6894	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6895	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6896	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6897	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6898	.test_ring = gfx_v9_0_ring_test_ring,
6899	.insert_nop = amdgpu_ring_insert_nop,
6900	.pad_ib = amdgpu_ring_generic_pad_ib,
6901	.emit_switch_buffer = gfx_v9_ring_emit_sb,
6902	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6903	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6904	.preempt_ib = gfx_v9_0_ring_preempt_ib,
6905	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6906	.emit_wreg = gfx_v9_0_ring_emit_wreg,
6907	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6908	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6909	.soft_recovery = gfx_v9_0_ring_soft_recovery,
6910	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
6911	};
6912
6913	static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
6914	.type = AMDGPU_RING_TYPE_GFX,
6915	.align_mask = 0xff,
6916	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6917	.support_64bit_ptrs = true,
6918	.secure_submission_supported = true,
6919	.get_rptr = amdgpu_sw_ring_get_rptr_gfx,
6920	.get_wptr = amdgpu_sw_ring_get_wptr_gfx,
6921	.set_wptr = amdgpu_sw_ring_set_wptr_gfx,
6922	.emit_frame_size = /* totally 242 maximum if 16 IBs */
6923	5 + /* COND_EXEC */
6924	7 + /* PIPELINE_SYNC */
6925	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6926	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6927	2 + /* VM_FLUSH */
6928	8 + /* FENCE for VM_FLUSH */
6929	20 + /* GDS switch */
6930	4 + /* double SWITCH_BUFFER,
6931	* the first COND_EXEC jump to the place just
6932	* prior to this double SWITCH_BUFFER
6933	*/
6934	5 + /* COND_EXEC */
6935	7 + /* HDP_flush */
6936	4 + /* VGT_flush */
6937	14 + /* CE_META */
6938	31 + /* DE_META */
6939	3 + /* CNTX_CTRL */
6940	5 + /* HDP_INVL */
6941	8 + 8 + /* FENCE x2 */
6942	2 + /* SWITCH_BUFFER */
6943	7, /* gfx_v9_0_emit_mem_sync */
6944	.emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6945	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6946	.emit_fence = gfx_v9_0_ring_emit_fence,
6947	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6948	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6949	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6950	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6951	.test_ring = gfx_v9_0_ring_test_ring,
6952	.test_ib = gfx_v9_0_ring_test_ib,
6953	.insert_nop = amdgpu_sw_ring_insert_nop,
6954	.pad_ib = amdgpu_ring_generic_pad_ib,
6955	.emit_switch_buffer = gfx_v9_ring_emit_sb,
6956	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6957	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6958	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
6959	.emit_wreg = gfx_v9_0_ring_emit_wreg,
6960	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6961	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6962	.soft_recovery = gfx_v9_0_ring_soft_recovery,
6963	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
6964	.patch_cntl = gfx_v9_0_ring_patch_cntl,
6965	.patch_de = gfx_v9_0_ring_patch_de_meta,
6966	.patch_ce = gfx_v9_0_ring_patch_ce_meta,
6967	};
6968
6969	static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6970	.type = AMDGPU_RING_TYPE_COMPUTE,
6971	.align_mask = 0xff,
6972	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6973	.support_64bit_ptrs = true,
6974	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
6975	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
6976	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
6977	.emit_frame_size =
6978	20 + /* gfx_v9_0_ring_emit_gds_switch */
6979	7 + /* gfx_v9_0_ring_emit_hdp_flush */
6980	5 + /* hdp invalidate */
6981	7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6982	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6983	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6984	2 + /* gfx_v9_0_ring_emit_vm_flush */
6985	8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6986	7 + /* gfx_v9_0_emit_mem_sync */
6987	5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6988	15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6989	.emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6990	.emit_ib = gfx_v9_0_ring_emit_ib_compute,
6991	.emit_fence = gfx_v9_0_ring_emit_fence,
6992	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6993	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6994	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6995	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6996	.test_ring = gfx_v9_0_ring_test_ring,
6997	.test_ib = gfx_v9_0_ring_test_ib,
6998	.insert_nop = amdgpu_ring_insert_nop,
6999	.pad_ib = amdgpu_ring_generic_pad_ib,
7000	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7001	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7002	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7003	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
7004	.emit_wave_limit = gfx_v9_0_emit_wave_limit,
7005	};
7006
7007	static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7008	.type = AMDGPU_RING_TYPE_KIQ,
7009	.align_mask = 0xff,
7010	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
7011	.support_64bit_ptrs = true,
7012	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
7013	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
7014	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
7015	.emit_frame_size =
7016	20 + /* gfx_v9_0_ring_emit_gds_switch */
7017	7 + /* gfx_v9_0_ring_emit_hdp_flush */
7018	5 + /* hdp invalidate */
7019	7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7020	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7021	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7022	2 + /* gfx_v9_0_ring_emit_vm_flush */
7023	8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7024	.emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7025	.emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7026	.test_ring = gfx_v9_0_ring_test_ring,
7027	.insert_nop = amdgpu_ring_insert_nop,
7028	.pad_ib = amdgpu_ring_generic_pad_ib,
7029	.emit_rreg = gfx_v9_0_ring_emit_rreg,
7030	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7031	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7032	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7033	};
7034
7035	static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7036	{
7037	int i;
7038
7039	adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7040
7041	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7042	adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7043
7044	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7045	for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7046	adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7047	}
7048
7049	for (i = 0; i < adev->gfx.num_compute_rings; i++)
7050	adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7051	}
7052
7053	static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7054	.set = gfx_v9_0_set_eop_interrupt_state,
7055	.process = gfx_v9_0_eop_irq,
7056	};
7057
7058	static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7059	.set = gfx_v9_0_set_priv_reg_fault_state,
7060	.process = gfx_v9_0_priv_reg_irq,
7061	};
7062
7063	static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7064	.set = gfx_v9_0_set_priv_inst_fault_state,
7065	.process = gfx_v9_0_priv_inst_irq,
7066	};
7067
7068	static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7069	.set = gfx_v9_0_set_cp_ecc_error_state,
7070	.process = amdgpu_gfx_cp_ecc_error_irq,
7071	};
7072
7073
7074	static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7075	{
7076	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7077	adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7078
7079	adev->gfx.priv_reg_irq.num_types = 1;
7080	adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7081
7082	adev->gfx.priv_inst_irq.num_types = 1;
7083	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7084
7085	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7086	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7087	}
7088
7089	static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7090	{
7091	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
7092	case IP_VERSION(9, 0, 1):
7093	case IP_VERSION(9, 2, 1):
7094	case IP_VERSION(9, 4, 0):
7095	case IP_VERSION(9, 2, 2):
7096	case IP_VERSION(9, 1, 0):
7097	case IP_VERSION(9, 4, 1):
7098	case IP_VERSION(9, 3, 0):
7099	case IP_VERSION(9, 4, 2):
7100	adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7101	break;
7102	default:
7103	break;
7104	}
7105	}
7106
7107	static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7108	{
7109	/* init asci gds info */
7110	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
7111	case IP_VERSION(9, 0, 1):
7112	case IP_VERSION(9, 2, 1):
7113	case IP_VERSION(9, 4, 0):
7114	adev->gds.gds_size = 0x10000;
7115	break;
7116	case IP_VERSION(9, 2, 2):
7117	case IP_VERSION(9, 1, 0):
7118	case IP_VERSION(9, 4, 1):
7119	adev->gds.gds_size = 0x1000;
7120	break;
7121	case IP_VERSION(9, 4, 2):
7122	/* aldebaran removed all the GDS internal memory,
7123	* only support GWS opcode in kernel, like barrier
7124	* semaphore.etc */
7125	adev->gds.gds_size = 0;
7126	break;
7127	default:
7128	adev->gds.gds_size = 0x10000;
7129	break;
7130	}
7131
7132	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
7133	case IP_VERSION(9, 0, 1):
7134	case IP_VERSION(9, 4, 0):
7135	adev->gds.gds_compute_max_wave_id = 0x7ff;
7136	break;
7137	case IP_VERSION(9, 2, 1):
7138	adev->gds.gds_compute_max_wave_id = 0x27f;
7139	break;
7140	case IP_VERSION(9, 2, 2):
7141	case IP_VERSION(9, 1, 0):
7142	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7143	adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7144	else
7145	adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7146	break;
7147	case IP_VERSION(9, 4, 1):
7148	adev->gds.gds_compute_max_wave_id = 0xfff;
7149	break;
7150	case IP_VERSION(9, 4, 2):
7151	/* deprecated for Aldebaran, no usage at all */
7152	adev->gds.gds_compute_max_wave_id = 0;
7153	break;
7154	default:
7155	/* this really depends on the chip */
7156	adev->gds.gds_compute_max_wave_id = 0x7ff;
7157	break;
7158	}
7159
7160	adev->gds.gws_size = 64;
7161	adev->gds.oa_size = 16;
7162	}
7163
7164	static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7165	u32 bitmap)
7166	{
7167	u32 data;
7168
7169	if (!bitmap)
7170	return;
7171
7172	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7173	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7174
7175	WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7176	}
7177
7178	static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7179	{
7180	u32 data, mask;
7181
7182	data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7183	data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7184
7185	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7186	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7187
7188	mask = amdgpu_gfx_create_bitmask(bit_width: adev->gfx.config.max_cu_per_sh);
7189
7190	return (~data) & mask;
7191	}
7192
7193	static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7194	struct amdgpu_cu_info *cu_info)
7195	{
7196	int i, j, k, counter, active_cu_number = 0;
7197	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7198	unsigned disable_masks[4 * 4];
7199
7200	if (!adev || !cu_info)
7201	return -EINVAL;
7202
7203	/*
7204	* 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7205	*/
7206	if (adev->gfx.config.max_shader_engines *
7207	adev->gfx.config.max_sh_per_se > 16)
7208	return -EINVAL;
7209
7210	amdgpu_gfx_parse_disable_cu(mask: disable_masks,
7211	max_se: adev->gfx.config.max_shader_engines,
7212	max_sh: adev->gfx.config.max_sh_per_se);
7213
7214	mutex_lock(&adev->grbm_idx_mutex);
7215	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7216	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7217	mask = 1;
7218	ao_bitmap = 0;
7219	counter = 0;
7220	amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7221	gfx_v9_0_set_user_cu_inactive_bitmap(
7222	adev, bitmap: disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7223	bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7224
7225	/*
7226	* The bitmap(and ao_cu_bitmap) in cu_info structure is
7227	* 4x4 size array, and it's usually suitable for Vega
7228	* ASICs which has 4*2 SE/SH layout.
7229	* But for Arcturus, SE/SH layout is changed to 8*1.
7230	* To mostly reduce the impact, we make it compatible
7231	* with current bitmap array as below:
7232	* SE4,SH0 --> bitmap[0][1]
7233	* SE5,SH0 --> bitmap[1][1]
7234	* SE6,SH0 --> bitmap[2][1]
7235	* SE7,SH0 --> bitmap[3][1]
7236	*/
7237	cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7238
7239	for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7240	if (bitmap & mask) {
7241	if (counter < adev->gfx.config.max_cu_per_sh)
7242	ao_bitmap |= mask;
7243	counter ++;
7244	}
7245	mask <<= 1;
7246	}
7247	active_cu_number += counter;
7248	if (i < 2 && j < 2)
7249	ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7250	cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7251	}
7252	}
7253	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7254	mutex_unlock(lock: &adev->grbm_idx_mutex);
7255
7256	cu_info->number = active_cu_number;
7257	cu_info->ao_cu_mask = ao_cu_mask;
7258	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7259
7260	return 0;
7261	}
7262
7263	const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7264	{
7265	.type = AMD_IP_BLOCK_TYPE_GFX,
7266	.major = 9,
7267	.minor = 0,
7268	.rev = 0,
7269	.funcs = &gfx_v9_0_ip_funcs,
7270	};
7271