1	/*
2	* Copyright 2010 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	* Authors: Alex Deucher
23	*/
24
25	#include <linux/firmware.h>
26	#include <linux/pci.h>
27	#include <linux/slab.h>
28
29	#include <drm/drm_edid.h>
30	#include <drm/drm_vblank.h>
31	#include <drm/radeon_drm.h>
32	#include <drm/drm_fourcc.h>
33	#include <drm/drm_framebuffer.h>
34
35	#include "atom.h"
36	#include "avivod.h"
37	#include "cik.h"
38	#include "ni.h"
39	#include "rv770.h"
40	#include "evergreen.h"
41	#include "evergreen_blit_shaders.h"
42	#include "evergreen_reg.h"
43	#include "evergreend.h"
44	#include "radeon.h"
45	#include "radeon_asic.h"
46	#include "radeon_audio.h"
47	#include "radeon_ucode.h"
48	#include "si.h"
49
50	#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
51	#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
52	#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
53
54	/*
55	* Indirect registers accessor
56	*/
57	u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
58	{
59	unsigned long flags;
60	u32 r;
61
62	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
63	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
64	r = RREG32(EVERGREEN_CG_IND_DATA);
65	spin_unlock_irqrestore(lock: &rdev->cg_idx_lock, flags);
66	return r;
67	}
68
69	void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
70	{
71	unsigned long flags;
72
73	spin_lock_irqsave(&rdev->cg_idx_lock, flags);
74	WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
75	WREG32(EVERGREEN_CG_IND_DATA, (v));
76	spin_unlock_irqrestore(lock: &rdev->cg_idx_lock, flags);
77	}
78
79	u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
80	{
81	unsigned long flags;
82	u32 r;
83
84	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
85	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
86	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
87	spin_unlock_irqrestore(lock: &rdev->pif_idx_lock, flags);
88	return r;
89	}
90
91	void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
92	{
93	unsigned long flags;
94
95	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
96	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
97	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
98	spin_unlock_irqrestore(lock: &rdev->pif_idx_lock, flags);
99	}
100
101	u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
102	{
103	unsigned long flags;
104	u32 r;
105
106	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
107	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
108	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
109	spin_unlock_irqrestore(lock: &rdev->pif_idx_lock, flags);
110	return r;
111	}
112
113	void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
114	{
115	unsigned long flags;
116
117	spin_lock_irqsave(&rdev->pif_idx_lock, flags);
118	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
119	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
120	spin_unlock_irqrestore(lock: &rdev->pif_idx_lock, flags);
121	}
122
123	static const u32 crtc_offsets[6] =
124	{
125	EVERGREEN_CRTC0_REGISTER_OFFSET,
126	EVERGREEN_CRTC1_REGISTER_OFFSET,
127	EVERGREEN_CRTC2_REGISTER_OFFSET,
128	EVERGREEN_CRTC3_REGISTER_OFFSET,
129	EVERGREEN_CRTC4_REGISTER_OFFSET,
130	EVERGREEN_CRTC5_REGISTER_OFFSET
131	};
132
133	#include "clearstate_evergreen.h"
134
135	static const u32 sumo_rlc_save_restore_register_list[] =
136	{
137	0x98fc,
138	0x9830,
139	0x9834,
140	0x9838,
141	0x9870,
142	0x9874,
143	0x8a14,
144	0x8b24,
145	0x8bcc,
146	0x8b10,
147	0x8d00,
148	0x8d04,
149	0x8c00,
150	0x8c04,
151	0x8c08,
152	0x8c0c,
153	0x8d8c,
154	0x8c20,
155	0x8c24,
156	0x8c28,
157	0x8c18,
158	0x8c1c,
159	0x8cf0,
160	0x8e2c,
161	0x8e38,
162	0x8c30,
163	0x9508,
164	0x9688,
165	0x9608,
166	0x960c,
167	0x9610,
168	0x9614,
169	0x88c4,
170	0x88d4,
171	0xa008,
172	0x900c,
173	0x9100,
174	0x913c,
175	0x98f8,
176	0x98f4,
177	0x9b7c,
178	0x3f8c,
179	0x8950,
180	0x8954,
181	0x8a18,
182	0x8b28,
183	0x9144,
184	0x9148,
185	0x914c,
186	0x3f90,
187	0x3f94,
188	0x915c,
189	0x9160,
190	0x9178,
191	0x917c,
192	0x9180,
193	0x918c,
194	0x9190,
195	0x9194,
196	0x9198,
197	0x919c,
198	0x91a8,
199	0x91ac,
200	0x91b0,
201	0x91b4,
202	0x91b8,
203	0x91c4,
204	0x91c8,
205	0x91cc,
206	0x91d0,
207	0x91d4,
208	0x91e0,
209	0x91e4,
210	0x91ec,
211	0x91f0,
212	0x91f4,
213	0x9200,
214	0x9204,
215	0x929c,
216	0x9150,
217	0x802c,
218	};
219
220	static void evergreen_gpu_init(struct radeon_device *rdev);
221	void evergreen_fini(struct radeon_device *rdev);
222	void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
223	void evergreen_program_aspm(struct radeon_device *rdev);
224
225	static const u32 evergreen_golden_registers[] =
226	{
227	0x3f90, 0xffff0000, 0xff000000,
228	0x9148, 0xffff0000, 0xff000000,
229	0x3f94, 0xffff0000, 0xff000000,
230	0x914c, 0xffff0000, 0xff000000,
231	0x9b7c, 0xffffffff, 0x00000000,
232	0x8a14, 0xffffffff, 0x00000007,
233	0x8b10, 0xffffffff, 0x00000000,
234	0x960c, 0xffffffff, 0x54763210,
235	0x88c4, 0xffffffff, 0x000000c2,
236	0x88d4, 0xffffffff, 0x00000010,
237	0x8974, 0xffffffff, 0x00000000,
238	0xc78, 0x00000080, 0x00000080,
239	0x5eb4, 0xffffffff, 0x00000002,
240	0x5e78, 0xffffffff, 0x001000f0,
241	0x6104, 0x01000300, 0x00000000,
242	0x5bc0, 0x00300000, 0x00000000,
243	0x7030, 0xffffffff, 0x00000011,
244	0x7c30, 0xffffffff, 0x00000011,
245	0x10830, 0xffffffff, 0x00000011,
246	0x11430, 0xffffffff, 0x00000011,
247	0x12030, 0xffffffff, 0x00000011,
248	0x12c30, 0xffffffff, 0x00000011,
249	0xd02c, 0xffffffff, 0x08421000,
250	0x240c, 0xffffffff, 0x00000380,
251	0x8b24, 0xffffffff, 0x00ff0fff,
252	0x28a4c, 0x06000000, 0x06000000,
253	0x10c, 0x00000001, 0x00000001,
254	0x8d00, 0xffffffff, 0x100e4848,
255	0x8d04, 0xffffffff, 0x00164745,
256	0x8c00, 0xffffffff, 0xe4000003,
257	0x8c04, 0xffffffff, 0x40600060,
258	0x8c08, 0xffffffff, 0x001c001c,
259	0x8cf0, 0xffffffff, 0x08e00620,
260	0x8c20, 0xffffffff, 0x00800080,
261	0x8c24, 0xffffffff, 0x00800080,
262	0x8c18, 0xffffffff, 0x20202078,
263	0x8c1c, 0xffffffff, 0x00001010,
264	0x28350, 0xffffffff, 0x00000000,
265	0xa008, 0xffffffff, 0x00010000,
266	0x5c4, 0xffffffff, 0x00000001,
267	0x9508, 0xffffffff, 0x00000002,
268	0x913c, 0x0000000f, 0x0000000a
269	};
270
271	static const u32 evergreen_golden_registers2[] =
272	{
273	0x2f4c, 0xffffffff, 0x00000000,
274	0x54f4, 0xffffffff, 0x00000000,
275	0x54f0, 0xffffffff, 0x00000000,
276	0x5498, 0xffffffff, 0x00000000,
277	0x549c, 0xffffffff, 0x00000000,
278	0x5494, 0xffffffff, 0x00000000,
279	0x53cc, 0xffffffff, 0x00000000,
280	0x53c8, 0xffffffff, 0x00000000,
281	0x53c4, 0xffffffff, 0x00000000,
282	0x53c0, 0xffffffff, 0x00000000,
283	0x53bc, 0xffffffff, 0x00000000,
284	0x53b8, 0xffffffff, 0x00000000,
285	0x53b4, 0xffffffff, 0x00000000,
286	0x53b0, 0xffffffff, 0x00000000
287	};
288
289	static const u32 cypress_mgcg_init[] =
290	{
291	0x802c, 0xffffffff, 0xc0000000,
292	0x5448, 0xffffffff, 0x00000100,
293	0x55e4, 0xffffffff, 0x00000100,
294	0x160c, 0xffffffff, 0x00000100,
295	0x5644, 0xffffffff, 0x00000100,
296	0xc164, 0xffffffff, 0x00000100,
297	0x8a18, 0xffffffff, 0x00000100,
298	0x897c, 0xffffffff, 0x06000100,
299	0x8b28, 0xffffffff, 0x00000100,
300	0x9144, 0xffffffff, 0x00000100,
301	0x9a60, 0xffffffff, 0x00000100,
302	0x9868, 0xffffffff, 0x00000100,
303	0x8d58, 0xffffffff, 0x00000100,
304	0x9510, 0xffffffff, 0x00000100,
305	0x949c, 0xffffffff, 0x00000100,
306	0x9654, 0xffffffff, 0x00000100,
307	0x9030, 0xffffffff, 0x00000100,
308	0x9034, 0xffffffff, 0x00000100,
309	0x9038, 0xffffffff, 0x00000100,
310	0x903c, 0xffffffff, 0x00000100,
311	0x9040, 0xffffffff, 0x00000100,
312	0xa200, 0xffffffff, 0x00000100,
313	0xa204, 0xffffffff, 0x00000100,
314	0xa208, 0xffffffff, 0x00000100,
315	0xa20c, 0xffffffff, 0x00000100,
316	0x971c, 0xffffffff, 0x00000100,
317	0x977c, 0xffffffff, 0x00000100,
318	0x3f80, 0xffffffff, 0x00000100,
319	0xa210, 0xffffffff, 0x00000100,
320	0xa214, 0xffffffff, 0x00000100,
321	0x4d8, 0xffffffff, 0x00000100,
322	0x9784, 0xffffffff, 0x00000100,
323	0x9698, 0xffffffff, 0x00000100,
324	0x4d4, 0xffffffff, 0x00000200,
325	0x30cc, 0xffffffff, 0x00000100,
326	0xd0c0, 0xffffffff, 0xff000100,
327	0x802c, 0xffffffff, 0x40000000,
328	0x915c, 0xffffffff, 0x00010000,
329	0x9160, 0xffffffff, 0x00030002,
330	0x9178, 0xffffffff, 0x00070000,
331	0x917c, 0xffffffff, 0x00030002,
332	0x9180, 0xffffffff, 0x00050004,
333	0x918c, 0xffffffff, 0x00010006,
334	0x9190, 0xffffffff, 0x00090008,
335	0x9194, 0xffffffff, 0x00070000,
336	0x9198, 0xffffffff, 0x00030002,
337	0x919c, 0xffffffff, 0x00050004,
338	0x91a8, 0xffffffff, 0x00010006,
339	0x91ac, 0xffffffff, 0x00090008,
340	0x91b0, 0xffffffff, 0x00070000,
341	0x91b4, 0xffffffff, 0x00030002,
342	0x91b8, 0xffffffff, 0x00050004,
343	0x91c4, 0xffffffff, 0x00010006,
344	0x91c8, 0xffffffff, 0x00090008,
345	0x91cc, 0xffffffff, 0x00070000,
346	0x91d0, 0xffffffff, 0x00030002,
347	0x91d4, 0xffffffff, 0x00050004,
348	0x91e0, 0xffffffff, 0x00010006,
349	0x91e4, 0xffffffff, 0x00090008,
350	0x91e8, 0xffffffff, 0x00000000,
351	0x91ec, 0xffffffff, 0x00070000,
352	0x91f0, 0xffffffff, 0x00030002,
353	0x91f4, 0xffffffff, 0x00050004,
354	0x9200, 0xffffffff, 0x00010006,
355	0x9204, 0xffffffff, 0x00090008,
356	0x9208, 0xffffffff, 0x00070000,
357	0x920c, 0xffffffff, 0x00030002,
358	0x9210, 0xffffffff, 0x00050004,
359	0x921c, 0xffffffff, 0x00010006,
360	0x9220, 0xffffffff, 0x00090008,
361	0x9224, 0xffffffff, 0x00070000,
362	0x9228, 0xffffffff, 0x00030002,
363	0x922c, 0xffffffff, 0x00050004,
364	0x9238, 0xffffffff, 0x00010006,
365	0x923c, 0xffffffff, 0x00090008,
366	0x9240, 0xffffffff, 0x00070000,
367	0x9244, 0xffffffff, 0x00030002,
368	0x9248, 0xffffffff, 0x00050004,
369	0x9254, 0xffffffff, 0x00010006,
370	0x9258, 0xffffffff, 0x00090008,
371	0x925c, 0xffffffff, 0x00070000,
372	0x9260, 0xffffffff, 0x00030002,
373	0x9264, 0xffffffff, 0x00050004,
374	0x9270, 0xffffffff, 0x00010006,
375	0x9274, 0xffffffff, 0x00090008,
376	0x9278, 0xffffffff, 0x00070000,
377	0x927c, 0xffffffff, 0x00030002,
378	0x9280, 0xffffffff, 0x00050004,
379	0x928c, 0xffffffff, 0x00010006,
380	0x9290, 0xffffffff, 0x00090008,
381	0x9294, 0xffffffff, 0x00000000,
382	0x929c, 0xffffffff, 0x00000001,
383	0x802c, 0xffffffff, 0x40010000,
384	0x915c, 0xffffffff, 0x00010000,
385	0x9160, 0xffffffff, 0x00030002,
386	0x9178, 0xffffffff, 0x00070000,
387	0x917c, 0xffffffff, 0x00030002,
388	0x9180, 0xffffffff, 0x00050004,
389	0x918c, 0xffffffff, 0x00010006,
390	0x9190, 0xffffffff, 0x00090008,
391	0x9194, 0xffffffff, 0x00070000,
392	0x9198, 0xffffffff, 0x00030002,
393	0x919c, 0xffffffff, 0x00050004,
394	0x91a8, 0xffffffff, 0x00010006,
395	0x91ac, 0xffffffff, 0x00090008,
396	0x91b0, 0xffffffff, 0x00070000,
397	0x91b4, 0xffffffff, 0x00030002,
398	0x91b8, 0xffffffff, 0x00050004,
399	0x91c4, 0xffffffff, 0x00010006,
400	0x91c8, 0xffffffff, 0x00090008,
401	0x91cc, 0xffffffff, 0x00070000,
402	0x91d0, 0xffffffff, 0x00030002,
403	0x91d4, 0xffffffff, 0x00050004,
404	0x91e0, 0xffffffff, 0x00010006,
405	0x91e4, 0xffffffff, 0x00090008,
406	0x91e8, 0xffffffff, 0x00000000,
407	0x91ec, 0xffffffff, 0x00070000,
408	0x91f0, 0xffffffff, 0x00030002,
409	0x91f4, 0xffffffff, 0x00050004,
410	0x9200, 0xffffffff, 0x00010006,
411	0x9204, 0xffffffff, 0x00090008,
412	0x9208, 0xffffffff, 0x00070000,
413	0x920c, 0xffffffff, 0x00030002,
414	0x9210, 0xffffffff, 0x00050004,
415	0x921c, 0xffffffff, 0x00010006,
416	0x9220, 0xffffffff, 0x00090008,
417	0x9224, 0xffffffff, 0x00070000,
418	0x9228, 0xffffffff, 0x00030002,
419	0x922c, 0xffffffff, 0x00050004,
420	0x9238, 0xffffffff, 0x00010006,
421	0x923c, 0xffffffff, 0x00090008,
422	0x9240, 0xffffffff, 0x00070000,
423	0x9244, 0xffffffff, 0x00030002,
424	0x9248, 0xffffffff, 0x00050004,
425	0x9254, 0xffffffff, 0x00010006,
426	0x9258, 0xffffffff, 0x00090008,
427	0x925c, 0xffffffff, 0x00070000,
428	0x9260, 0xffffffff, 0x00030002,
429	0x9264, 0xffffffff, 0x00050004,
430	0x9270, 0xffffffff, 0x00010006,
431	0x9274, 0xffffffff, 0x00090008,
432	0x9278, 0xffffffff, 0x00070000,
433	0x927c, 0xffffffff, 0x00030002,
434	0x9280, 0xffffffff, 0x00050004,
435	0x928c, 0xffffffff, 0x00010006,
436	0x9290, 0xffffffff, 0x00090008,
437	0x9294, 0xffffffff, 0x00000000,
438	0x929c, 0xffffffff, 0x00000001,
439	0x802c, 0xffffffff, 0xc0000000
440	};
441
442	static const u32 redwood_mgcg_init[] =
443	{
444	0x802c, 0xffffffff, 0xc0000000,
445	0x5448, 0xffffffff, 0x00000100,
446	0x55e4, 0xffffffff, 0x00000100,
447	0x160c, 0xffffffff, 0x00000100,
448	0x5644, 0xffffffff, 0x00000100,
449	0xc164, 0xffffffff, 0x00000100,
450	0x8a18, 0xffffffff, 0x00000100,
451	0x897c, 0xffffffff, 0x06000100,
452	0x8b28, 0xffffffff, 0x00000100,
453	0x9144, 0xffffffff, 0x00000100,
454	0x9a60, 0xffffffff, 0x00000100,
455	0x9868, 0xffffffff, 0x00000100,
456	0x8d58, 0xffffffff, 0x00000100,
457	0x9510, 0xffffffff, 0x00000100,
458	0x949c, 0xffffffff, 0x00000100,
459	0x9654, 0xffffffff, 0x00000100,
460	0x9030, 0xffffffff, 0x00000100,
461	0x9034, 0xffffffff, 0x00000100,
462	0x9038, 0xffffffff, 0x00000100,
463	0x903c, 0xffffffff, 0x00000100,
464	0x9040, 0xffffffff, 0x00000100,
465	0xa200, 0xffffffff, 0x00000100,
466	0xa204, 0xffffffff, 0x00000100,
467	0xa208, 0xffffffff, 0x00000100,
468	0xa20c, 0xffffffff, 0x00000100,
469	0x971c, 0xffffffff, 0x00000100,
470	0x977c, 0xffffffff, 0x00000100,
471	0x3f80, 0xffffffff, 0x00000100,
472	0xa210, 0xffffffff, 0x00000100,
473	0xa214, 0xffffffff, 0x00000100,
474	0x4d8, 0xffffffff, 0x00000100,
475	0x9784, 0xffffffff, 0x00000100,
476	0x9698, 0xffffffff, 0x00000100,
477	0x4d4, 0xffffffff, 0x00000200,
478	0x30cc, 0xffffffff, 0x00000100,
479	0xd0c0, 0xffffffff, 0xff000100,
480	0x802c, 0xffffffff, 0x40000000,
481	0x915c, 0xffffffff, 0x00010000,
482	0x9160, 0xffffffff, 0x00030002,
483	0x9178, 0xffffffff, 0x00070000,
484	0x917c, 0xffffffff, 0x00030002,
485	0x9180, 0xffffffff, 0x00050004,
486	0x918c, 0xffffffff, 0x00010006,
487	0x9190, 0xffffffff, 0x00090008,
488	0x9194, 0xffffffff, 0x00070000,
489	0x9198, 0xffffffff, 0x00030002,
490	0x919c, 0xffffffff, 0x00050004,
491	0x91a8, 0xffffffff, 0x00010006,
492	0x91ac, 0xffffffff, 0x00090008,
493	0x91b0, 0xffffffff, 0x00070000,
494	0x91b4, 0xffffffff, 0x00030002,
495	0x91b8, 0xffffffff, 0x00050004,
496	0x91c4, 0xffffffff, 0x00010006,
497	0x91c8, 0xffffffff, 0x00090008,
498	0x91cc, 0xffffffff, 0x00070000,
499	0x91d0, 0xffffffff, 0x00030002,
500	0x91d4, 0xffffffff, 0x00050004,
501	0x91e0, 0xffffffff, 0x00010006,
502	0x91e4, 0xffffffff, 0x00090008,
503	0x91e8, 0xffffffff, 0x00000000,
504	0x91ec, 0xffffffff, 0x00070000,
505	0x91f0, 0xffffffff, 0x00030002,
506	0x91f4, 0xffffffff, 0x00050004,
507	0x9200, 0xffffffff, 0x00010006,
508	0x9204, 0xffffffff, 0x00090008,
509	0x9294, 0xffffffff, 0x00000000,
510	0x929c, 0xffffffff, 0x00000001,
511	0x802c, 0xffffffff, 0xc0000000
512	};
513
514	static const u32 cedar_golden_registers[] =
515	{
516	0x3f90, 0xffff0000, 0xff000000,
517	0x9148, 0xffff0000, 0xff000000,
518	0x3f94, 0xffff0000, 0xff000000,
519	0x914c, 0xffff0000, 0xff000000,
520	0x9b7c, 0xffffffff, 0x00000000,
521	0x8a14, 0xffffffff, 0x00000007,
522	0x8b10, 0xffffffff, 0x00000000,
523	0x960c, 0xffffffff, 0x54763210,
524	0x88c4, 0xffffffff, 0x000000c2,
525	0x88d4, 0xffffffff, 0x00000000,
526	0x8974, 0xffffffff, 0x00000000,
527	0xc78, 0x00000080, 0x00000080,
528	0x5eb4, 0xffffffff, 0x00000002,
529	0x5e78, 0xffffffff, 0x001000f0,
530	0x6104, 0x01000300, 0x00000000,
531	0x5bc0, 0x00300000, 0x00000000,
532	0x7030, 0xffffffff, 0x00000011,
533	0x7c30, 0xffffffff, 0x00000011,
534	0x10830, 0xffffffff, 0x00000011,
535	0x11430, 0xffffffff, 0x00000011,
536	0xd02c, 0xffffffff, 0x08421000,
537	0x240c, 0xffffffff, 0x00000380,
538	0x8b24, 0xffffffff, 0x00ff0fff,
539	0x28a4c, 0x06000000, 0x06000000,
540	0x10c, 0x00000001, 0x00000001,
541	0x8d00, 0xffffffff, 0x100e4848,
542	0x8d04, 0xffffffff, 0x00164745,
543	0x8c00, 0xffffffff, 0xe4000003,
544	0x8c04, 0xffffffff, 0x40600060,
545	0x8c08, 0xffffffff, 0x001c001c,
546	0x8cf0, 0xffffffff, 0x08e00410,
547	0x8c20, 0xffffffff, 0x00800080,
548	0x8c24, 0xffffffff, 0x00800080,
549	0x8c18, 0xffffffff, 0x20202078,
550	0x8c1c, 0xffffffff, 0x00001010,
551	0x28350, 0xffffffff, 0x00000000,
552	0xa008, 0xffffffff, 0x00010000,
553	0x5c4, 0xffffffff, 0x00000001,
554	0x9508, 0xffffffff, 0x00000002
555	};
556
557	static const u32 cedar_mgcg_init[] =
558	{
559	0x802c, 0xffffffff, 0xc0000000,
560	0x5448, 0xffffffff, 0x00000100,
561	0x55e4, 0xffffffff, 0x00000100,
562	0x160c, 0xffffffff, 0x00000100,
563	0x5644, 0xffffffff, 0x00000100,
564	0xc164, 0xffffffff, 0x00000100,
565	0x8a18, 0xffffffff, 0x00000100,
566	0x897c, 0xffffffff, 0x06000100,
567	0x8b28, 0xffffffff, 0x00000100,
568	0x9144, 0xffffffff, 0x00000100,
569	0x9a60, 0xffffffff, 0x00000100,
570	0x9868, 0xffffffff, 0x00000100,
571	0x8d58, 0xffffffff, 0x00000100,
572	0x9510, 0xffffffff, 0x00000100,
573	0x949c, 0xffffffff, 0x00000100,
574	0x9654, 0xffffffff, 0x00000100,
575	0x9030, 0xffffffff, 0x00000100,
576	0x9034, 0xffffffff, 0x00000100,
577	0x9038, 0xffffffff, 0x00000100,
578	0x903c, 0xffffffff, 0x00000100,
579	0x9040, 0xffffffff, 0x00000100,
580	0xa200, 0xffffffff, 0x00000100,
581	0xa204, 0xffffffff, 0x00000100,
582	0xa208, 0xffffffff, 0x00000100,
583	0xa20c, 0xffffffff, 0x00000100,
584	0x971c, 0xffffffff, 0x00000100,
585	0x977c, 0xffffffff, 0x00000100,
586	0x3f80, 0xffffffff, 0x00000100,
587	0xa210, 0xffffffff, 0x00000100,
588	0xa214, 0xffffffff, 0x00000100,
589	0x4d8, 0xffffffff, 0x00000100,
590	0x9784, 0xffffffff, 0x00000100,
591	0x9698, 0xffffffff, 0x00000100,
592	0x4d4, 0xffffffff, 0x00000200,
593	0x30cc, 0xffffffff, 0x00000100,
594	0xd0c0, 0xffffffff, 0xff000100,
595	0x802c, 0xffffffff, 0x40000000,
596	0x915c, 0xffffffff, 0x00010000,
597	0x9178, 0xffffffff, 0x00050000,
598	0x917c, 0xffffffff, 0x00030002,
599	0x918c, 0xffffffff, 0x00010004,
600	0x9190, 0xffffffff, 0x00070006,
601	0x9194, 0xffffffff, 0x00050000,
602	0x9198, 0xffffffff, 0x00030002,
603	0x91a8, 0xffffffff, 0x00010004,
604	0x91ac, 0xffffffff, 0x00070006,
605	0x91e8, 0xffffffff, 0x00000000,
606	0x9294, 0xffffffff, 0x00000000,
607	0x929c, 0xffffffff, 0x00000001,
608	0x802c, 0xffffffff, 0xc0000000
609	};
610
611	static const u32 juniper_mgcg_init[] =
612	{
613	0x802c, 0xffffffff, 0xc0000000,
614	0x5448, 0xffffffff, 0x00000100,
615	0x55e4, 0xffffffff, 0x00000100,
616	0x160c, 0xffffffff, 0x00000100,
617	0x5644, 0xffffffff, 0x00000100,
618	0xc164, 0xffffffff, 0x00000100,
619	0x8a18, 0xffffffff, 0x00000100,
620	0x897c, 0xffffffff, 0x06000100,
621	0x8b28, 0xffffffff, 0x00000100,
622	0x9144, 0xffffffff, 0x00000100,
623	0x9a60, 0xffffffff, 0x00000100,
624	0x9868, 0xffffffff, 0x00000100,
625	0x8d58, 0xffffffff, 0x00000100,
626	0x9510, 0xffffffff, 0x00000100,
627	0x949c, 0xffffffff, 0x00000100,
628	0x9654, 0xffffffff, 0x00000100,
629	0x9030, 0xffffffff, 0x00000100,
630	0x9034, 0xffffffff, 0x00000100,
631	0x9038, 0xffffffff, 0x00000100,
632	0x903c, 0xffffffff, 0x00000100,
633	0x9040, 0xffffffff, 0x00000100,
634	0xa200, 0xffffffff, 0x00000100,
635	0xa204, 0xffffffff, 0x00000100,
636	0xa208, 0xffffffff, 0x00000100,
637	0xa20c, 0xffffffff, 0x00000100,
638	0x971c, 0xffffffff, 0x00000100,
639	0xd0c0, 0xffffffff, 0xff000100,
640	0x802c, 0xffffffff, 0x40000000,
641	0x915c, 0xffffffff, 0x00010000,
642	0x9160, 0xffffffff, 0x00030002,
643	0x9178, 0xffffffff, 0x00070000,
644	0x917c, 0xffffffff, 0x00030002,
645	0x9180, 0xffffffff, 0x00050004,
646	0x918c, 0xffffffff, 0x00010006,
647	0x9190, 0xffffffff, 0x00090008,
648	0x9194, 0xffffffff, 0x00070000,
649	0x9198, 0xffffffff, 0x00030002,
650	0x919c, 0xffffffff, 0x00050004,
651	0x91a8, 0xffffffff, 0x00010006,
652	0x91ac, 0xffffffff, 0x00090008,
653	0x91b0, 0xffffffff, 0x00070000,
654	0x91b4, 0xffffffff, 0x00030002,
655	0x91b8, 0xffffffff, 0x00050004,
656	0x91c4, 0xffffffff, 0x00010006,
657	0x91c8, 0xffffffff, 0x00090008,
658	0x91cc, 0xffffffff, 0x00070000,
659	0x91d0, 0xffffffff, 0x00030002,
660	0x91d4, 0xffffffff, 0x00050004,
661	0x91e0, 0xffffffff, 0x00010006,
662	0x91e4, 0xffffffff, 0x00090008,
663	0x91e8, 0xffffffff, 0x00000000,
664	0x91ec, 0xffffffff, 0x00070000,
665	0x91f0, 0xffffffff, 0x00030002,
666	0x91f4, 0xffffffff, 0x00050004,
667	0x9200, 0xffffffff, 0x00010006,
668	0x9204, 0xffffffff, 0x00090008,
669	0x9208, 0xffffffff, 0x00070000,
670	0x920c, 0xffffffff, 0x00030002,
671	0x9210, 0xffffffff, 0x00050004,
672	0x921c, 0xffffffff, 0x00010006,
673	0x9220, 0xffffffff, 0x00090008,
674	0x9224, 0xffffffff, 0x00070000,
675	0x9228, 0xffffffff, 0x00030002,
676	0x922c, 0xffffffff, 0x00050004,
677	0x9238, 0xffffffff, 0x00010006,
678	0x923c, 0xffffffff, 0x00090008,
679	0x9240, 0xffffffff, 0x00070000,
680	0x9244, 0xffffffff, 0x00030002,
681	0x9248, 0xffffffff, 0x00050004,
682	0x9254, 0xffffffff, 0x00010006,
683	0x9258, 0xffffffff, 0x00090008,
684	0x925c, 0xffffffff, 0x00070000,
685	0x9260, 0xffffffff, 0x00030002,
686	0x9264, 0xffffffff, 0x00050004,
687	0x9270, 0xffffffff, 0x00010006,
688	0x9274, 0xffffffff, 0x00090008,
689	0x9278, 0xffffffff, 0x00070000,
690	0x927c, 0xffffffff, 0x00030002,
691	0x9280, 0xffffffff, 0x00050004,
692	0x928c, 0xffffffff, 0x00010006,
693	0x9290, 0xffffffff, 0x00090008,
694	0x9294, 0xffffffff, 0x00000000,
695	0x929c, 0xffffffff, 0x00000001,
696	0x802c, 0xffffffff, 0xc0000000,
697	0x977c, 0xffffffff, 0x00000100,
698	0x3f80, 0xffffffff, 0x00000100,
699	0xa210, 0xffffffff, 0x00000100,
700	0xa214, 0xffffffff, 0x00000100,
701	0x4d8, 0xffffffff, 0x00000100,
702	0x9784, 0xffffffff, 0x00000100,
703	0x9698, 0xffffffff, 0x00000100,
704	0x4d4, 0xffffffff, 0x00000200,
705	0x30cc, 0xffffffff, 0x00000100,
706	0x802c, 0xffffffff, 0xc0000000
707	};
708
709	static const u32 supersumo_golden_registers[] =
710	{
711	0x5eb4, 0xffffffff, 0x00000002,
712	0x5c4, 0xffffffff, 0x00000001,
713	0x7030, 0xffffffff, 0x00000011,
714	0x7c30, 0xffffffff, 0x00000011,
715	0x6104, 0x01000300, 0x00000000,
716	0x5bc0, 0x00300000, 0x00000000,
717	0x8c04, 0xffffffff, 0x40600060,
718	0x8c08, 0xffffffff, 0x001c001c,
719	0x8c20, 0xffffffff, 0x00800080,
720	0x8c24, 0xffffffff, 0x00800080,
721	0x8c18, 0xffffffff, 0x20202078,
722	0x8c1c, 0xffffffff, 0x00001010,
723	0x918c, 0xffffffff, 0x00010006,
724	0x91a8, 0xffffffff, 0x00010006,
725	0x91c4, 0xffffffff, 0x00010006,
726	0x91e0, 0xffffffff, 0x00010006,
727	0x9200, 0xffffffff, 0x00010006,
728	0x9150, 0xffffffff, 0x6e944040,
729	0x917c, 0xffffffff, 0x00030002,
730	0x9180, 0xffffffff, 0x00050004,
731	0x9198, 0xffffffff, 0x00030002,
732	0x919c, 0xffffffff, 0x00050004,
733	0x91b4, 0xffffffff, 0x00030002,
734	0x91b8, 0xffffffff, 0x00050004,
735	0x91d0, 0xffffffff, 0x00030002,
736	0x91d4, 0xffffffff, 0x00050004,
737	0x91f0, 0xffffffff, 0x00030002,
738	0x91f4, 0xffffffff, 0x00050004,
739	0x915c, 0xffffffff, 0x00010000,
740	0x9160, 0xffffffff, 0x00030002,
741	0x3f90, 0xffff0000, 0xff000000,
742	0x9178, 0xffffffff, 0x00070000,
743	0x9194, 0xffffffff, 0x00070000,
744	0x91b0, 0xffffffff, 0x00070000,
745	0x91cc, 0xffffffff, 0x00070000,
746	0x91ec, 0xffffffff, 0x00070000,
747	0x9148, 0xffff0000, 0xff000000,
748	0x9190, 0xffffffff, 0x00090008,
749	0x91ac, 0xffffffff, 0x00090008,
750	0x91c8, 0xffffffff, 0x00090008,
751	0x91e4, 0xffffffff, 0x00090008,
752	0x9204, 0xffffffff, 0x00090008,
753	0x3f94, 0xffff0000, 0xff000000,
754	0x914c, 0xffff0000, 0xff000000,
755	0x929c, 0xffffffff, 0x00000001,
756	0x8a18, 0xffffffff, 0x00000100,
757	0x8b28, 0xffffffff, 0x00000100,
758	0x9144, 0xffffffff, 0x00000100,
759	0x5644, 0xffffffff, 0x00000100,
760	0x9b7c, 0xffffffff, 0x00000000,
761	0x8030, 0xffffffff, 0x0000100a,
762	0x8a14, 0xffffffff, 0x00000007,
763	0x8b24, 0xffffffff, 0x00ff0fff,
764	0x8b10, 0xffffffff, 0x00000000,
765	0x28a4c, 0x06000000, 0x06000000,
766	0x4d8, 0xffffffff, 0x00000100,
767	0x913c, 0xffff000f, 0x0100000a,
768	0x960c, 0xffffffff, 0x54763210,
769	0x88c4, 0xffffffff, 0x000000c2,
770	0x88d4, 0xffffffff, 0x00000010,
771	0x8974, 0xffffffff, 0x00000000,
772	0xc78, 0x00000080, 0x00000080,
773	0x5e78, 0xffffffff, 0x001000f0,
774	0xd02c, 0xffffffff, 0x08421000,
775	0xa008, 0xffffffff, 0x00010000,
776	0x8d00, 0xffffffff, 0x100e4848,
777	0x8d04, 0xffffffff, 0x00164745,
778	0x8c00, 0xffffffff, 0xe4000003,
779	0x8cf0, 0x1fffffff, 0x08e00620,
780	0x28350, 0xffffffff, 0x00000000,
781	0x9508, 0xffffffff, 0x00000002
782	};
783
784	static const u32 sumo_golden_registers[] =
785	{
786	0x900c, 0x00ffffff, 0x0017071f,
787	0x8c18, 0xffffffff, 0x10101060,
788	0x8c1c, 0xffffffff, 0x00001010,
789	0x8c30, 0x0000000f, 0x00000005,
790	0x9688, 0x0000000f, 0x00000007
791	};
792
793	static const u32 wrestler_golden_registers[] =
794	{
795	0x5eb4, 0xffffffff, 0x00000002,
796	0x5c4, 0xffffffff, 0x00000001,
797	0x7030, 0xffffffff, 0x00000011,
798	0x7c30, 0xffffffff, 0x00000011,
799	0x6104, 0x01000300, 0x00000000,
800	0x5bc0, 0x00300000, 0x00000000,
801	0x918c, 0xffffffff, 0x00010006,
802	0x91a8, 0xffffffff, 0x00010006,
803	0x9150, 0xffffffff, 0x6e944040,
804	0x917c, 0xffffffff, 0x00030002,
805	0x9198, 0xffffffff, 0x00030002,
806	0x915c, 0xffffffff, 0x00010000,
807	0x3f90, 0xffff0000, 0xff000000,
808	0x9178, 0xffffffff, 0x00070000,
809	0x9194, 0xffffffff, 0x00070000,
810	0x9148, 0xffff0000, 0xff000000,
811	0x9190, 0xffffffff, 0x00090008,
812	0x91ac, 0xffffffff, 0x00090008,
813	0x3f94, 0xffff0000, 0xff000000,
814	0x914c, 0xffff0000, 0xff000000,
815	0x929c, 0xffffffff, 0x00000001,
816	0x8a18, 0xffffffff, 0x00000100,
817	0x8b28, 0xffffffff, 0x00000100,
818	0x9144, 0xffffffff, 0x00000100,
819	0x9b7c, 0xffffffff, 0x00000000,
820	0x8030, 0xffffffff, 0x0000100a,
821	0x8a14, 0xffffffff, 0x00000001,
822	0x8b24, 0xffffffff, 0x00ff0fff,
823	0x8b10, 0xffffffff, 0x00000000,
824	0x28a4c, 0x06000000, 0x06000000,
825	0x4d8, 0xffffffff, 0x00000100,
826	0x913c, 0xffff000f, 0x0100000a,
827	0x960c, 0xffffffff, 0x54763210,
828	0x88c4, 0xffffffff, 0x000000c2,
829	0x88d4, 0xffffffff, 0x00000010,
830	0x8974, 0xffffffff, 0x00000000,
831	0xc78, 0x00000080, 0x00000080,
832	0x5e78, 0xffffffff, 0x001000f0,
833	0xd02c, 0xffffffff, 0x08421000,
834	0xa008, 0xffffffff, 0x00010000,
835	0x8d00, 0xffffffff, 0x100e4848,
836	0x8d04, 0xffffffff, 0x00164745,
837	0x8c00, 0xffffffff, 0xe4000003,
838	0x8cf0, 0x1fffffff, 0x08e00410,
839	0x28350, 0xffffffff, 0x00000000,
840	0x9508, 0xffffffff, 0x00000002,
841	0x900c, 0xffffffff, 0x0017071f,
842	0x8c18, 0xffffffff, 0x10101060,
843	0x8c1c, 0xffffffff, 0x00001010
844	};
845
846	static const u32 barts_golden_registers[] =
847	{
848	0x5eb4, 0xffffffff, 0x00000002,
849	0x5e78, 0x8f311ff1, 0x001000f0,
850	0x3f90, 0xffff0000, 0xff000000,
851	0x9148, 0xffff0000, 0xff000000,
852	0x3f94, 0xffff0000, 0xff000000,
853	0x914c, 0xffff0000, 0xff000000,
854	0xc78, 0x00000080, 0x00000080,
855	0xbd4, 0x70073777, 0x00010001,
856	0xd02c, 0xbfffff1f, 0x08421000,
857	0xd0b8, 0x03773777, 0x02011003,
858	0x5bc0, 0x00200000, 0x50100000,
859	0x98f8, 0x33773777, 0x02011003,
860	0x98fc, 0xffffffff, 0x76543210,
861	0x7030, 0x31000311, 0x00000011,
862	0x2f48, 0x00000007, 0x02011003,
863	0x6b28, 0x00000010, 0x00000012,
864	0x7728, 0x00000010, 0x00000012,
865	0x10328, 0x00000010, 0x00000012,
866	0x10f28, 0x00000010, 0x00000012,
867	0x11b28, 0x00000010, 0x00000012,
868	0x12728, 0x00000010, 0x00000012,
869	0x240c, 0x000007ff, 0x00000380,
870	0x8a14, 0xf000001f, 0x00000007,
871	0x8b24, 0x3fff3fff, 0x00ff0fff,
872	0x8b10, 0x0000ff0f, 0x00000000,
873	0x28a4c, 0x07ffffff, 0x06000000,
874	0x10c, 0x00000001, 0x00010003,
875	0xa02c, 0xffffffff, 0x0000009b,
876	0x913c, 0x0000000f, 0x0100000a,
877	0x8d00, 0xffff7f7f, 0x100e4848,
878	0x8d04, 0x00ffffff, 0x00164745,
879	0x8c00, 0xfffc0003, 0xe4000003,
880	0x8c04, 0xf8ff00ff, 0x40600060,
881	0x8c08, 0x00ff00ff, 0x001c001c,
882	0x8cf0, 0x1fff1fff, 0x08e00620,
883	0x8c20, 0x0fff0fff, 0x00800080,
884	0x8c24, 0x0fff0fff, 0x00800080,
885	0x8c18, 0xffffffff, 0x20202078,
886	0x8c1c, 0x0000ffff, 0x00001010,
887	0x28350, 0x00000f01, 0x00000000,
888	0x9508, 0x3700001f, 0x00000002,
889	0x960c, 0xffffffff, 0x54763210,
890	0x88c4, 0x001f3ae3, 0x000000c2,
891	0x88d4, 0x0000001f, 0x00000010,
892	0x8974, 0xffffffff, 0x00000000
893	};
894
895	static const u32 turks_golden_registers[] =
896	{
897	0x5eb4, 0xffffffff, 0x00000002,
898	0x5e78, 0x8f311ff1, 0x001000f0,
899	0x8c8, 0x00003000, 0x00001070,
900	0x8cc, 0x000fffff, 0x00040035,
901	0x3f90, 0xffff0000, 0xfff00000,
902	0x9148, 0xffff0000, 0xfff00000,
903	0x3f94, 0xffff0000, 0xfff00000,
904	0x914c, 0xffff0000, 0xfff00000,
905	0xc78, 0x00000080, 0x00000080,
906	0xbd4, 0x00073007, 0x00010002,
907	0xd02c, 0xbfffff1f, 0x08421000,
908	0xd0b8, 0x03773777, 0x02010002,
909	0x5bc0, 0x00200000, 0x50100000,
910	0x98f8, 0x33773777, 0x00010002,
911	0x98fc, 0xffffffff, 0x33221100,
912	0x7030, 0x31000311, 0x00000011,
913	0x2f48, 0x33773777, 0x00010002,
914	0x6b28, 0x00000010, 0x00000012,
915	0x7728, 0x00000010, 0x00000012,
916	0x10328, 0x00000010, 0x00000012,
917	0x10f28, 0x00000010, 0x00000012,
918	0x11b28, 0x00000010, 0x00000012,
919	0x12728, 0x00000010, 0x00000012,
920	0x240c, 0x000007ff, 0x00000380,
921	0x8a14, 0xf000001f, 0x00000007,
922	0x8b24, 0x3fff3fff, 0x00ff0fff,
923	0x8b10, 0x0000ff0f, 0x00000000,
924	0x28a4c, 0x07ffffff, 0x06000000,
925	0x10c, 0x00000001, 0x00010003,
926	0xa02c, 0xffffffff, 0x0000009b,
927	0x913c, 0x0000000f, 0x0100000a,
928	0x8d00, 0xffff7f7f, 0x100e4848,
929	0x8d04, 0x00ffffff, 0x00164745,
930	0x8c00, 0xfffc0003, 0xe4000003,
931	0x8c04, 0xf8ff00ff, 0x40600060,
932	0x8c08, 0x00ff00ff, 0x001c001c,
933	0x8cf0, 0x1fff1fff, 0x08e00410,
934	0x8c20, 0x0fff0fff, 0x00800080,
935	0x8c24, 0x0fff0fff, 0x00800080,
936	0x8c18, 0xffffffff, 0x20202078,
937	0x8c1c, 0x0000ffff, 0x00001010,
938	0x28350, 0x00000f01, 0x00000000,
939	0x9508, 0x3700001f, 0x00000002,
940	0x960c, 0xffffffff, 0x54763210,
941	0x88c4, 0x001f3ae3, 0x000000c2,
942	0x88d4, 0x0000001f, 0x00000010,
943	0x8974, 0xffffffff, 0x00000000
944	};
945
946	static const u32 caicos_golden_registers[] =
947	{
948	0x5eb4, 0xffffffff, 0x00000002,
949	0x5e78, 0x8f311ff1, 0x001000f0,
950	0x8c8, 0x00003420, 0x00001450,
951	0x8cc, 0x000fffff, 0x00040035,
952	0x3f90, 0xffff0000, 0xfffc0000,
953	0x9148, 0xffff0000, 0xfffc0000,
954	0x3f94, 0xffff0000, 0xfffc0000,
955	0x914c, 0xffff0000, 0xfffc0000,
956	0xc78, 0x00000080, 0x00000080,
957	0xbd4, 0x00073007, 0x00010001,
958	0xd02c, 0xbfffff1f, 0x08421000,
959	0xd0b8, 0x03773777, 0x02010001,
960	0x5bc0, 0x00200000, 0x50100000,
961	0x98f8, 0x33773777, 0x02010001,
962	0x98fc, 0xffffffff, 0x33221100,
963	0x7030, 0x31000311, 0x00000011,
964	0x2f48, 0x33773777, 0x02010001,
965	0x6b28, 0x00000010, 0x00000012,
966	0x7728, 0x00000010, 0x00000012,
967	0x10328, 0x00000010, 0x00000012,
968	0x10f28, 0x00000010, 0x00000012,
969	0x11b28, 0x00000010, 0x00000012,
970	0x12728, 0x00000010, 0x00000012,
971	0x240c, 0x000007ff, 0x00000380,
972	0x8a14, 0xf000001f, 0x00000001,
973	0x8b24, 0x3fff3fff, 0x00ff0fff,
974	0x8b10, 0x0000ff0f, 0x00000000,
975	0x28a4c, 0x07ffffff, 0x06000000,
976	0x10c, 0x00000001, 0x00010003,
977	0xa02c, 0xffffffff, 0x0000009b,
978	0x913c, 0x0000000f, 0x0100000a,
979	0x8d00, 0xffff7f7f, 0x100e4848,
980	0x8d04, 0x00ffffff, 0x00164745,
981	0x8c00, 0xfffc0003, 0xe4000003,
982	0x8c04, 0xf8ff00ff, 0x40600060,
983	0x8c08, 0x00ff00ff, 0x001c001c,
984	0x8cf0, 0x1fff1fff, 0x08e00410,
985	0x8c20, 0x0fff0fff, 0x00800080,
986	0x8c24, 0x0fff0fff, 0x00800080,
987	0x8c18, 0xffffffff, 0x20202078,
988	0x8c1c, 0x0000ffff, 0x00001010,
989	0x28350, 0x00000f01, 0x00000000,
990	0x9508, 0x3700001f, 0x00000002,
991	0x960c, 0xffffffff, 0x54763210,
992	0x88c4, 0x001f3ae3, 0x000000c2,
993	0x88d4, 0x0000001f, 0x00000010,
994	0x8974, 0xffffffff, 0x00000000
995	};
996
997	static void evergreen_init_golden_registers(struct radeon_device *rdev)
998	{
999	switch (rdev->family) {
1000	case CHIP_CYPRESS:
1001	case CHIP_HEMLOCK:
1002	radeon_program_register_sequence(rdev,
1003	registers: evergreen_golden_registers,
1004	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005	radeon_program_register_sequence(rdev,
1006	registers: evergreen_golden_registers2,
1007	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008	radeon_program_register_sequence(rdev,
1009	registers: cypress_mgcg_init,
1010	array_size: (const u32)ARRAY_SIZE(cypress_mgcg_init));
1011	break;
1012	case CHIP_JUNIPER:
1013	radeon_program_register_sequence(rdev,
1014	registers: evergreen_golden_registers,
1015	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016	radeon_program_register_sequence(rdev,
1017	registers: evergreen_golden_registers2,
1018	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019	radeon_program_register_sequence(rdev,
1020	registers: juniper_mgcg_init,
1021	array_size: (const u32)ARRAY_SIZE(juniper_mgcg_init));
1022	break;
1023	case CHIP_REDWOOD:
1024	radeon_program_register_sequence(rdev,
1025	registers: evergreen_golden_registers,
1026	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027	radeon_program_register_sequence(rdev,
1028	registers: evergreen_golden_registers2,
1029	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030	radeon_program_register_sequence(rdev,
1031	registers: redwood_mgcg_init,
1032	array_size: (const u32)ARRAY_SIZE(redwood_mgcg_init));
1033	break;
1034	case CHIP_CEDAR:
1035	radeon_program_register_sequence(rdev,
1036	registers: cedar_golden_registers,
1037	array_size: (const u32)ARRAY_SIZE(cedar_golden_registers));
1038	radeon_program_register_sequence(rdev,
1039	registers: evergreen_golden_registers2,
1040	array_size: (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041	radeon_program_register_sequence(rdev,
1042	registers: cedar_mgcg_init,
1043	array_size: (const u32)ARRAY_SIZE(cedar_mgcg_init));
1044	break;
1045	case CHIP_PALM:
1046	radeon_program_register_sequence(rdev,
1047	registers: wrestler_golden_registers,
1048	array_size: (const u32)ARRAY_SIZE(wrestler_golden_registers));
1049	break;
1050	case CHIP_SUMO:
1051	radeon_program_register_sequence(rdev,
1052	registers: supersumo_golden_registers,
1053	array_size: (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054	break;
1055	case CHIP_SUMO2:
1056	radeon_program_register_sequence(rdev,
1057	registers: supersumo_golden_registers,
1058	array_size: (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059	radeon_program_register_sequence(rdev,
1060	registers: sumo_golden_registers,
1061	array_size: (const u32)ARRAY_SIZE(sumo_golden_registers));
1062	break;
1063	case CHIP_BARTS:
1064	radeon_program_register_sequence(rdev,
1065	registers: barts_golden_registers,
1066	array_size: (const u32)ARRAY_SIZE(barts_golden_registers));
1067	break;
1068	case CHIP_TURKS:
1069	radeon_program_register_sequence(rdev,
1070	registers: turks_golden_registers,
1071	array_size: (const u32)ARRAY_SIZE(turks_golden_registers));
1072	break;
1073	case CHIP_CAICOS:
1074	radeon_program_register_sequence(rdev,
1075	registers: caicos_golden_registers,
1076	array_size: (const u32)ARRAY_SIZE(caicos_golden_registers));
1077	break;
1078	default:
1079	break;
1080	}
1081	}
1082
1083	/**
1084	* evergreen_get_allowed_info_register - fetch the register for the info ioctl
1085	*
1086	* @rdev: radeon_device pointer
1087	* @reg: register offset in bytes
1088	* @val: register value
1089	*
1090	* Returns 0 for success or -EINVAL for an invalid register
1091	*
1092	*/
1093	int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1094	u32 reg, u32 *val)
1095	{
1096	switch (reg) {
1097	case GRBM_STATUS:
1098	case GRBM_STATUS_SE0:
1099	case GRBM_STATUS_SE1:
1100	case SRBM_STATUS:
1101	case SRBM_STATUS2:
1102	case DMA_STATUS_REG:
1103	case UVD_STATUS:
1104	*val = RREG32(reg);
1105	return 0;
1106	default:
1107	return -EINVAL;
1108	}
1109	}
1110
1111	void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112	unsigned *bankh, unsigned *mtaspect,
1113	unsigned *tile_split)
1114	{
1115	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1119	switch (*bankw) {
1120	default:
1121	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1125	}
1126	switch (*bankh) {
1127	default:
1128	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1132	}
1133	switch (*mtaspect) {
1134	default:
1135	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1139	}
1140	}
1141
1142	static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143	u32 cntl_reg, u32 status_reg)
1144	{
1145	int r, i;
1146	struct atom_clock_dividers dividers;
1147
1148	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149	clock, strobe_mode: false, dividers: &dividers);
1150	if (r)
1151	return r;
1152
1153	WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1154
1155	for (i = 0; i < 100; i++) {
1156	if (RREG32(status_reg) & DCLK_STATUS)
1157	break;
1158	mdelay(10);
1159	}
1160	if (i == 100)
1161	return -ETIMEDOUT;
1162
1163	return 0;
1164	}
1165
1166	int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1167	{
1168	int r = 0;
1169	u32 cg_scratch = RREG32(CG_SCRATCH1);
1170
1171	r = sumo_set_uvd_clock(rdev, clock: vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1172	if (r)
1173	goto done;
1174	cg_scratch &= 0xffff0000;
1175	cg_scratch |= vclk / 100; /* Mhz */
1176
1177	r = sumo_set_uvd_clock(rdev, clock: dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1178	if (r)
1179	goto done;
1180	cg_scratch &= 0x0000ffff;
1181	cg_scratch |= (dclk / 100) << 16; /* Mhz */
1182
1183	done:
1184	WREG32(CG_SCRATCH1, cg_scratch);
1185
1186	return r;
1187	}
1188
1189	int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1190	{
1191	/* start off with something large */
1192	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1193	int r;
1194
1195	/* bypass vclk and dclk with bclk */
1196	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197	VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198	~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1199
1200	/* put PLL in bypass mode */
1201	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1202
1203	if (!vclk || !dclk) {
1204	/* keep the Bypass mode, put PLL to sleep */
1205	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1206	return 0;
1207	}
1208
1209	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, vco_min: 125000, vco_max: 250000,
1210	fb_factor: 16384, fb_mask: 0x03FFFFFF, pd_min: 0, pd_max: 128, pd_even: 5,
1211	optimal_fb_div: &fb_div, optimal_vclk_div: &vclk_div, optimal_dclk_div: &dclk_div);
1212	if (r)
1213	return r;
1214
1215	/* set VCO_MODE to 1 */
1216	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1217
1218	/* toggle UPLL_SLEEP to 1 then back to 0 */
1219	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1221
1222	/* deassert UPLL_RESET */
1223	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1224
1225	mdelay(1);
1226
1227	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1228	if (r)
1229	return r;
1230
1231	/* assert UPLL_RESET again */
1232	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1233
1234	/* disable spread spectrum. */
1235	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1236
1237	/* set feedback divider */
1238	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1239
1240	/* set ref divider to 0 */
1241	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1242
1243	if (fb_div < 307200)
1244	WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1245	else
1246	WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1247
1248	/* set PDIV_A and PDIV_B */
1249	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250	UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251	~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1252
1253	/* give the PLL some time to settle */
1254	mdelay(15);
1255
1256	/* deassert PLL_RESET */
1257	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1258
1259	mdelay(15);
1260
1261	/* switch from bypass mode to normal mode */
1262	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1263
1264	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1265	if (r)
1266	return r;
1267
1268	/* switch VCLK and DCLK selection */
1269	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270	VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271	~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1272
1273	mdelay(100);
1274
1275	return 0;
1276	}
1277
1278	void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1279	{
1280	int readrq;
1281	u16 v;
1282
1283	readrq = pcie_get_readrq(dev: rdev->pdev);
1284	v = ffs(readrq) - 8;
1285	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286	* to avoid hangs or perfomance issues
1287	*/
1288	if ((v == 0) || (v == 6) || (v == 7))
1289	pcie_set_readrq(dev: rdev->pdev, rq: 512);
1290	}
1291
1292	void dce4_program_fmt(struct drm_encoder *encoder)
1293	{
1294	struct drm_device *dev = encoder->dev;
1295	struct radeon_device *rdev = dev->dev_private;
1296	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1299	int bpc = 0;
1300	u32 tmp = 0;
1301	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1302
1303	if (connector) {
1304	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305	bpc = radeon_get_monitor_bpc(connector);
1306	dither = radeon_connector->dither;
1307	}
1308
1309	/* LVDS/eDP FMT is set up by atom */
1310	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1311	return;
1312
1313	/* not needed for analog */
1314	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315	(radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1316	return;
1317
1318	if (bpc == 0)
1319	return;
1320
1321	switch (bpc) {
1322	case 6:
1323	if (dither == RADEON_FMT_DITHER_ENABLE)
1324	/* XXX sort out optimal dither settings */
1325	tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326	FMT_SPATIAL_DITHER_EN);
1327	else
1328	tmp |= FMT_TRUNCATE_EN;
1329	break;
1330	case 8:
1331	if (dither == RADEON_FMT_DITHER_ENABLE)
1332	/* XXX sort out optimal dither settings */
1333	tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334	FMT_RGB_RANDOM_ENABLE |
1335	FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1336	else
1337	tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1338	break;
1339	case 10:
1340	default:
1341	/* not needed */
1342	break;
1343	}
1344
1345	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1346	}
1347
1348	static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1349	{
1350	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1351	return true;
1352	else
1353	return false;
1354	}
1355
1356	static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1357	{
1358	u32 pos1, pos2;
1359
1360	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1362
1363	if (pos1 != pos2)
1364	return true;
1365	else
1366	return false;
1367	}
1368
1369	/**
1370	* dce4_wait_for_vblank - vblank wait asic callback.
1371	*
1372	* @rdev: radeon_device pointer
1373	* @crtc: crtc to wait for vblank on
1374	*
1375	* Wait for vblank on the requested crtc (evergreen+).
1376	*/
1377	void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1378	{
1379	unsigned i = 0;
1380
1381	if (crtc >= rdev->num_crtc)
1382	return;
1383
1384	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1385	return;
1386
1387	/* depending on when we hit vblank, we may be close to active; if so,
1388	* wait for another frame.
1389	*/
1390	while (dce4_is_in_vblank(rdev, crtc)) {
1391	if (i++ % 100 == 0) {
1392	if (!dce4_is_counter_moving(rdev, crtc))
1393	break;
1394	}
1395	}
1396
1397	while (!dce4_is_in_vblank(rdev, crtc)) {
1398	if (i++ % 100 == 0) {
1399	if (!dce4_is_counter_moving(rdev, crtc))
1400	break;
1401	}
1402	}
1403	}
1404
1405	/**
1406	* evergreen_page_flip - pageflip callback.
1407	*
1408	* @rdev: radeon_device pointer
1409	* @crtc_id: crtc to cleanup pageflip on
1410	* @crtc_base: new address of the crtc (GPU MC address)
1411	* @async: asynchronous flip
1412	*
1413	* Triggers the actual pageflip by updating the primary
1414	* surface base address (evergreen+).
1415	*/
1416	void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1417	bool async)
1418	{
1419	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420	struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1421
1422	/* flip at hsync for async, default is vsync */
1423	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424	async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1425	/* update pitch */
1426	WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427	fb->pitches[0] / fb->format->cpp[0]);
1428	/* update the scanout addresses */
1429	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430	upper_32_bits(crtc_base));
1431	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432	(u32)crtc_base);
1433	/* post the write */
1434	RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1435	}
1436
1437	/**
1438	* evergreen_page_flip_pending - check if page flip is still pending
1439	*
1440	* @rdev: radeon_device pointer
1441	* @crtc_id: crtc to check
1442	*
1443	* Returns the current update pending status.
1444	*/
1445	bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1446	{
1447	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1448
1449	/* Return current update_pending status: */
1450	return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451	EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1452	}
1453
1454	/* get temperature in millidegrees */
1455	int evergreen_get_temp(struct radeon_device *rdev)
1456	{
1457	u32 temp, toffset;
1458	int actual_temp = 0;
1459
1460	if (rdev->family == CHIP_JUNIPER) {
1461	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1462	TOFFSET_SHIFT;
1463	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1464	TS0_ADC_DOUT_SHIFT;
1465
1466	if (toffset & 0x100)
1467	actual_temp = temp / 2 - (0x200 - toffset);
1468	else
1469	actual_temp = temp / 2 + toffset;
1470
1471	actual_temp = actual_temp * 1000;
1472
1473	} else {
1474	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1475	ASIC_T_SHIFT;
1476
1477	if (temp & 0x400)
1478	actual_temp = -256;
1479	else if (temp & 0x200)
1480	actual_temp = 255;
1481	else if (temp & 0x100) {
1482	actual_temp = temp & 0x1ff;
1483	actual_temp |= ~0x1ff;
1484	} else
1485	actual_temp = temp & 0xff;
1486
1487	actual_temp = (actual_temp * 1000) / 2;
1488	}
1489
1490	return actual_temp;
1491	}
1492
1493	int sumo_get_temp(struct radeon_device *rdev)
1494	{
1495	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496	int actual_temp = temp - 49;
1497
1498	return actual_temp * 1000;
1499	}
1500
1501	/**
1502	* sumo_pm_init_profile - Initialize power profiles callback.
1503	*
1504	* @rdev: radeon_device pointer
1505	*
1506	* Initialize the power states used in profile mode
1507	* (sumo, trinity, SI).
1508	* Used for profile mode only.
1509	*/
1510	void sumo_pm_init_profile(struct radeon_device *rdev)
1511	{
1512	int idx;
1513
1514	/* default */
1515	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1519
1520	/* low,mid sh/mh */
1521	if (rdev->flags & RADEON_IS_MOBILITY)
1522	idx = radeon_pm_get_type_index(rdev, ps_type: POWER_STATE_TYPE_BATTERY, instance: 0);
1523	else
1524	idx = radeon_pm_get_type_index(rdev, ps_type: POWER_STATE_TYPE_PERFORMANCE, instance: 0);
1525
1526	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1530
1531	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535
1536	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1540
1541	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1545
1546	/* high sh/mh */
1547	idx = radeon_pm_get_type_index(rdev, ps_type: POWER_STATE_TYPE_PERFORMANCE, instance: 0);
1548	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552	rdev->pm.power_state[idx].num_clock_modes - 1;
1553
1554	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558	rdev->pm.power_state[idx].num_clock_modes - 1;
1559	}
1560
1561	/**
1562	* btc_pm_init_profile - Initialize power profiles callback.
1563	*
1564	* @rdev: radeon_device pointer
1565	*
1566	* Initialize the power states used in profile mode
1567	* (BTC, cayman).
1568	* Used for profile mode only.
1569	*/
1570	void btc_pm_init_profile(struct radeon_device *rdev)
1571	{
1572	int idx;
1573
1574	/* default */
1575	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579	/* starting with BTC, there is one state that is used for both
1580	* MH and SH. Difference is that we always use the high clock index for
1581	* mclk.
1582	*/
1583	if (rdev->flags & RADEON_IS_MOBILITY)
1584	idx = radeon_pm_get_type_index(rdev, ps_type: POWER_STATE_TYPE_BATTERY, instance: 0);
1585	else
1586	idx = radeon_pm_get_type_index(rdev, ps_type: POWER_STATE_TYPE_PERFORMANCE, instance: 0);
1587	/* low sh */
1588	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1592	/* mid sh */
1593	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1597	/* high sh */
1598	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1602	/* low mh */
1603	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1607	/* mid mh */
1608	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1612	/* high mh */
1613	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1617	}
1618
1619	/**
1620	* evergreen_pm_misc - set additional pm hw parameters callback.
1621	*
1622	* @rdev: radeon_device pointer
1623	*
1624	* Set non-clock parameters associated with a power state
1625	* (voltage, etc.) (evergreen+).
1626	*/
1627	void evergreen_pm_misc(struct radeon_device *rdev)
1628	{
1629	int req_ps_idx = rdev->pm.requested_power_state_index;
1630	int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1633
1634	if (voltage->type == VOLTAGE_SW) {
1635	/* 0xff0x are flags rather then an actual voltage */
1636	if ((voltage->voltage & 0xff00) == 0xff00)
1637	return;
1638	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1639	radeon_atom_set_voltage(rdev, voltage_level: voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1640	rdev->pm.current_vddc = voltage->voltage;
1641	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1642	}
1643
1644	/* starting with BTC, there is one state that is used for both
1645	* MH and SH. Difference is that we always use the high clock index for
1646	* mclk and vddci.
1647	*/
1648	if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1649	(rdev->family >= CHIP_BARTS) &&
1650	rdev->pm.active_crtc_count &&
1651	((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1652	(rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1653	voltage = &rdev->pm.power_state[req_ps_idx].
1654	clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1655
1656	/* 0xff0x are flags rather then an actual voltage */
1657	if ((voltage->vddci & 0xff00) == 0xff00)
1658	return;
1659	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1660	radeon_atom_set_voltage(rdev, voltage_level: voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1661	rdev->pm.current_vddci = voltage->vddci;
1662	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1663	}
1664	}
1665	}
1666
1667	/**
1668	* evergreen_pm_prepare - pre-power state change callback.
1669	*
1670	* @rdev: radeon_device pointer
1671	*
1672	* Prepare for a power state change (evergreen+).
1673	*/
1674	void evergreen_pm_prepare(struct radeon_device *rdev)
1675	{
1676	struct drm_device *ddev = rdev_to_drm(rdev);
1677	struct drm_crtc *crtc;
1678	struct radeon_crtc *radeon_crtc;
1679	u32 tmp;
1680
1681	/* disable any active CRTCs */
1682	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1683	radeon_crtc = to_radeon_crtc(crtc);
1684	if (radeon_crtc->enabled) {
1685	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1686	tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1687	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1688	}
1689	}
1690	}
1691
1692	/**
1693	* evergreen_pm_finish - post-power state change callback.
1694	*
1695	* @rdev: radeon_device pointer
1696	*
1697	* Clean up after a power state change (evergreen+).
1698	*/
1699	void evergreen_pm_finish(struct radeon_device *rdev)
1700	{
1701	struct drm_device *ddev = rdev_to_drm(rdev);
1702	struct drm_crtc *crtc;
1703	struct radeon_crtc *radeon_crtc;
1704	u32 tmp;
1705
1706	/* enable any active CRTCs */
1707	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1708	radeon_crtc = to_radeon_crtc(crtc);
1709	if (radeon_crtc->enabled) {
1710	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1711	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1712	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1713	}
1714	}
1715	}
1716
1717	/**
1718	* evergreen_hpd_sense - hpd sense callback.
1719	*
1720	* @rdev: radeon_device pointer
1721	* @hpd: hpd (hotplug detect) pin
1722	*
1723	* Checks if a digital monitor is connected (evergreen+).
1724	* Returns true if connected, false if not connected.
1725	*/
1726	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1727	{
1728	if (hpd == RADEON_HPD_NONE)
1729	return false;
1730
1731	return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1732	}
1733
1734	/**
1735	* evergreen_hpd_set_polarity - hpd set polarity callback.
1736	*
1737	* @rdev: radeon_device pointer
1738	* @hpd: hpd (hotplug detect) pin
1739	*
1740	* Set the polarity of the hpd pin (evergreen+).
1741	*/
1742	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1743	enum radeon_hpd_id hpd)
1744	{
1745	bool connected = evergreen_hpd_sense(rdev, hpd);
1746
1747	if (hpd == RADEON_HPD_NONE)
1748	return;
1749
1750	if (connected)
1751	WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1752	else
1753	WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1754	}
1755
1756	/**
1757	* evergreen_hpd_init - hpd setup callback.
1758	*
1759	* @rdev: radeon_device pointer
1760	*
1761	* Setup the hpd pins used by the card (evergreen+).
1762	* Enable the pin, set the polarity, and enable the hpd interrupts.
1763	*/
1764	void evergreen_hpd_init(struct radeon_device *rdev)
1765	{
1766	struct drm_device *dev = rdev_to_drm(rdev);
1767	struct drm_connector *connector;
1768	unsigned enabled = 0;
1769	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1770	DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1771
1772	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773	enum radeon_hpd_id hpd =
1774	to_radeon_connector(connector)->hpd.hpd;
1775
1776	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1777	connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1778	/* don't try to enable hpd on eDP or LVDS avoid breaking the
1779	* aux dp channel on imac and help (but not completely fix)
1780	* https://bugzilla.redhat.com/show_bug.cgi?id=726143
1781	* also avoid interrupt storms during dpms.
1782	*/
1783	continue;
1784	}
1785
1786	if (hpd == RADEON_HPD_NONE)
1787	continue;
1788
1789	WREG32(DC_HPDx_CONTROL(hpd), tmp);
1790	enabled |= 1 << hpd;
1791
1792	radeon_hpd_set_polarity(rdev, hpd);
1793	}
1794	radeon_irq_kms_enable_hpd(rdev, hpd_mask: enabled);
1795	}
1796
1797	/**
1798	* evergreen_hpd_fini - hpd tear down callback.
1799	*
1800	* @rdev: radeon_device pointer
1801	*
1802	* Tear down the hpd pins used by the card (evergreen+).
1803	* Disable the hpd interrupts.
1804	*/
1805	void evergreen_hpd_fini(struct radeon_device *rdev)
1806	{
1807	struct drm_device *dev = rdev_to_drm(rdev);
1808	struct drm_connector *connector;
1809	unsigned disabled = 0;
1810
1811	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1812	enum radeon_hpd_id hpd =
1813	to_radeon_connector(connector)->hpd.hpd;
1814
1815	if (hpd == RADEON_HPD_NONE)
1816	continue;
1817
1818	WREG32(DC_HPDx_CONTROL(hpd), 0);
1819	disabled |= 1 << hpd;
1820	}
1821	radeon_irq_kms_disable_hpd(rdev, hpd_mask: disabled);
1822	}
1823
1824	/* watermark setup */
1825
1826	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1827	struct radeon_crtc *radeon_crtc,
1828	struct drm_display_mode *mode,
1829	struct drm_display_mode *other_mode)
1830	{
1831	u32 tmp, buffer_alloc, i;
1832	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1833	/*
1834	* Line Buffer Setup
1835	* There are 3 line buffers, each one shared by 2 display controllers.
1836	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1837	* the display controllers. The paritioning is done via one of four
1838	* preset allocations specified in bits 2:0:
1839	* first display controller
1840	* 0 - first half of lb (3840 * 2)
1841	* 1 - first 3/4 of lb (5760 * 2)
1842	* 2 - whole lb (7680 * 2), other crtc must be disabled
1843	* 3 - first 1/4 of lb (1920 * 2)
1844	* second display controller
1845	* 4 - second half of lb (3840 * 2)
1846	* 5 - second 3/4 of lb (5760 * 2)
1847	* 6 - whole lb (7680 * 2), other crtc must be disabled
1848	* 7 - last 1/4 of lb (1920 * 2)
1849	*/
1850	/* this can get tricky if we have two large displays on a paired group
1851	* of crtcs. Ideally for multiple large displays we'd assign them to
1852	* non-linked crtcs for maximum line buffer allocation.
1853	*/
1854	if (radeon_crtc->base.enabled && mode) {
1855	if (other_mode) {
1856	tmp = 0; /* 1/2 */
1857	buffer_alloc = 1;
1858	} else {
1859	tmp = 2; /* whole */
1860	buffer_alloc = 2;
1861	}
1862	} else {
1863	tmp = 0;
1864	buffer_alloc = 0;
1865	}
1866
1867	/* second controller of the pair uses second half of the lb */
1868	if (radeon_crtc->crtc_id % 2)
1869	tmp += 4;
1870	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1871
1872	if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1873	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1874	DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1875	for (i = 0; i < rdev->usec_timeout; i++) {
1876	if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1877	DMIF_BUFFERS_ALLOCATED_COMPLETED)
1878	break;
1879	udelay(usec: 1);
1880	}
1881	}
1882
1883	if (radeon_crtc->base.enabled && mode) {
1884	switch (tmp) {
1885	case 0:
1886	case 4:
1887	default:
1888	if (ASIC_IS_DCE5(rdev))
1889	return 4096 * 2;
1890	else
1891	return 3840 * 2;
1892	case 1:
1893	case 5:
1894	if (ASIC_IS_DCE5(rdev))
1895	return 6144 * 2;
1896	else
1897	return 5760 * 2;
1898	case 2:
1899	case 6:
1900	if (ASIC_IS_DCE5(rdev))
1901	return 8192 * 2;
1902	else
1903	return 7680 * 2;
1904	case 3:
1905	case 7:
1906	if (ASIC_IS_DCE5(rdev))
1907	return 2048 * 2;
1908	else
1909	return 1920 * 2;
1910	}
1911	}
1912
1913	/* controller not enabled, so no lb used */
1914	return 0;
1915	}
1916
1917	u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1918	{
1919	u32 tmp = RREG32(MC_SHARED_CHMAP);
1920
1921	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1922	case 0:
1923	default:
1924	return 1;
1925	case 1:
1926	return 2;
1927	case 2:
1928	return 4;
1929	case 3:
1930	return 8;
1931	}
1932	}
1933
1934	struct evergreen_wm_params {
1935	u32 dram_channels; /* number of dram channels */
1936	u32 yclk; /* bandwidth per dram data pin in kHz */
1937	u32 sclk; /* engine clock in kHz */
1938	u32 disp_clk; /* display clock in kHz */
1939	u32 src_width; /* viewport width */
1940	u32 active_time; /* active display time in ns */
1941	u32 blank_time; /* blank time in ns */
1942	bool interlaced; /* mode is interlaced */
1943	fixed20_12 vsc; /* vertical scale ratio */
1944	u32 num_heads; /* number of active crtcs */
1945	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1946	u32 lb_size; /* line buffer allocated to pipe */
1947	u32 vtaps; /* vertical scaler taps */
1948	};
1949
1950	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1951	{
1952	/* Calculate DRAM Bandwidth and the part allocated to display. */
1953	fixed20_12 dram_efficiency; /* 0.7 */
1954	fixed20_12 yclk, dram_channels, bandwidth;
1955	fixed20_12 a;
1956
1957	a.full = dfixed_const(1000);
1958	yclk.full = dfixed_const(wm->yclk);
1959	yclk.full = dfixed_div(A: yclk, B: a);
1960	dram_channels.full = dfixed_const(wm->dram_channels * 4);
1961	a.full = dfixed_const(10);
1962	dram_efficiency.full = dfixed_const(7);
1963	dram_efficiency.full = dfixed_div(A: dram_efficiency, B: a);
1964	bandwidth.full = dfixed_mul(dram_channels, yclk);
1965	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1966
1967	return dfixed_trunc(bandwidth);
1968	}
1969
1970	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1971	{
1972	/* Calculate DRAM Bandwidth and the part allocated to display. */
1973	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1974	fixed20_12 yclk, dram_channels, bandwidth;
1975	fixed20_12 a;
1976
1977	a.full = dfixed_const(1000);
1978	yclk.full = dfixed_const(wm->yclk);
1979	yclk.full = dfixed_div(A: yclk, B: a);
1980	dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981	a.full = dfixed_const(10);
1982	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1983	disp_dram_allocation.full = dfixed_div(A: disp_dram_allocation, B: a);
1984	bandwidth.full = dfixed_mul(dram_channels, yclk);
1985	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1986
1987	return dfixed_trunc(bandwidth);
1988	}
1989
1990	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1991	{
1992	/* Calculate the display Data return Bandwidth */
1993	fixed20_12 return_efficiency; /* 0.8 */
1994	fixed20_12 sclk, bandwidth;
1995	fixed20_12 a;
1996
1997	a.full = dfixed_const(1000);
1998	sclk.full = dfixed_const(wm->sclk);
1999	sclk.full = dfixed_div(A: sclk, B: a);
2000	a.full = dfixed_const(10);
2001	return_efficiency.full = dfixed_const(8);
2002	return_efficiency.full = dfixed_div(A: return_efficiency, B: a);
2003	a.full = dfixed_const(32);
2004	bandwidth.full = dfixed_mul(a, sclk);
2005	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2006
2007	return dfixed_trunc(bandwidth);
2008	}
2009
2010	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2011	{
2012	/* Calculate the DMIF Request Bandwidth */
2013	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2014	fixed20_12 disp_clk, bandwidth;
2015	fixed20_12 a;
2016
2017	a.full = dfixed_const(1000);
2018	disp_clk.full = dfixed_const(wm->disp_clk);
2019	disp_clk.full = dfixed_div(A: disp_clk, B: a);
2020	a.full = dfixed_const(10);
2021	disp_clk_request_efficiency.full = dfixed_const(8);
2022	disp_clk_request_efficiency.full = dfixed_div(A: disp_clk_request_efficiency, B: a);
2023	a.full = dfixed_const(32);
2024	bandwidth.full = dfixed_mul(a, disp_clk);
2025	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2026
2027	return dfixed_trunc(bandwidth);
2028	}
2029
2030	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2031	{
2032	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2033	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2034	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2035	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2036
2037	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2038	}
2039
2040	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2041	{
2042	/* Calculate the display mode Average Bandwidth
2043	* DisplayMode should contain the source and destination dimensions,
2044	* timing, etc.
2045	*/
2046	fixed20_12 bpp;
2047	fixed20_12 line_time;
2048	fixed20_12 src_width;
2049	fixed20_12 bandwidth;
2050	fixed20_12 a;
2051
2052	a.full = dfixed_const(1000);
2053	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2054	line_time.full = dfixed_div(A: line_time, B: a);
2055	bpp.full = dfixed_const(wm->bytes_per_pixel);
2056	src_width.full = dfixed_const(wm->src_width);
2057	bandwidth.full = dfixed_mul(src_width, bpp);
2058	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2059	bandwidth.full = dfixed_div(A: bandwidth, B: line_time);
2060
2061	return dfixed_trunc(bandwidth);
2062	}
2063
2064	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2065	{
2066	/* First calcualte the latency in ns */
2067	u32 mc_latency = 2000; /* 2000 ns. */
2068	u32 available_bandwidth = evergreen_available_bandwidth(wm);
2069	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2070	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2071	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2072	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2073	(wm->num_heads * cursor_line_pair_return_time);
2074	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2075	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2076	fixed20_12 a, b, c;
2077
2078	if (wm->num_heads == 0)
2079	return 0;
2080
2081	a.full = dfixed_const(2);
2082	b.full = dfixed_const(1);
2083	if ((wm->vsc.full > a.full) ||
2084	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2085	(wm->vtaps >= 5) ||
2086	((wm->vsc.full >= a.full) && wm->interlaced))
2087	max_src_lines_per_dst_line = 4;
2088	else
2089	max_src_lines_per_dst_line = 2;
2090
2091	a.full = dfixed_const(available_bandwidth);
2092	b.full = dfixed_const(wm->num_heads);
2093	a.full = dfixed_div(A: a, B: b);
2094
2095	lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2096
2097	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2098	b.full = dfixed_const(1000);
2099	c.full = dfixed_const(lb_fill_bw);
2100	b.full = dfixed_div(A: c, B: b);
2101	a.full = dfixed_div(A: a, B: b);
2102	line_fill_time = dfixed_trunc(a);
2103
2104	if (line_fill_time < wm->active_time)
2105	return latency;
2106	else
2107	return latency + (line_fill_time - wm->active_time);
2108
2109	}
2110
2111	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2112	{
2113	if (evergreen_average_bandwidth(wm) <=
2114	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2115	return true;
2116	else
2117	return false;
2118	};
2119
2120	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2121	{
2122	if (evergreen_average_bandwidth(wm) <=
2123	(evergreen_available_bandwidth(wm) / wm->num_heads))
2124	return true;
2125	else
2126	return false;
2127	};
2128
2129	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2130	{
2131	u32 lb_partitions = wm->lb_size / wm->src_width;
2132	u32 line_time = wm->active_time + wm->blank_time;
2133	u32 latency_tolerant_lines;
2134	u32 latency_hiding;
2135	fixed20_12 a;
2136
2137	a.full = dfixed_const(1);
2138	if (wm->vsc.full > a.full)
2139	latency_tolerant_lines = 1;
2140	else {
2141	if (lb_partitions <= (wm->vtaps + 1))
2142	latency_tolerant_lines = 1;
2143	else
2144	latency_tolerant_lines = 2;
2145	}
2146
2147	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2148
2149	if (evergreen_latency_watermark(wm) <= latency_hiding)
2150	return true;
2151	else
2152	return false;
2153	}
2154
2155	static void evergreen_program_watermarks(struct radeon_device *rdev,
2156	struct radeon_crtc *radeon_crtc,
2157	u32 lb_size, u32 num_heads)
2158	{
2159	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2160	struct evergreen_wm_params wm_low, wm_high;
2161	u32 dram_channels;
2162	u32 active_time;
2163	u32 line_time = 0;
2164	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2165	u32 priority_a_mark = 0, priority_b_mark = 0;
2166	u32 priority_a_cnt = PRIORITY_OFF;
2167	u32 priority_b_cnt = PRIORITY_OFF;
2168	u32 pipe_offset = radeon_crtc->crtc_id * 16;
2169	u32 tmp, arb_control3;
2170	fixed20_12 a, b, c;
2171
2172	if (radeon_crtc->base.enabled && num_heads && mode) {
2173	active_time = (u32) div_u64(dividend: (u64)mode->crtc_hdisplay * 1000000,
2174	divisor: (u32)mode->clock);
2175	line_time = (u32) div_u64(dividend: (u64)mode->crtc_htotal * 1000000,
2176	divisor: (u32)mode->clock);
2177	line_time = min(line_time, (u32)65535);
2178	priority_a_cnt = 0;
2179	priority_b_cnt = 0;
2180	dram_channels = evergreen_get_number_of_dram_channels(rdev);
2181
2182	/* watermark for high clocks */
2183	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2184	wm_high.yclk =
2185	radeon_dpm_get_mclk(rdev, false) * 10;
2186	wm_high.sclk =
2187	radeon_dpm_get_sclk(rdev, false) * 10;
2188	} else {
2189	wm_high.yclk = rdev->pm.current_mclk * 10;
2190	wm_high.sclk = rdev->pm.current_sclk * 10;
2191	}
2192
2193	wm_high.disp_clk = mode->clock;
2194	wm_high.src_width = mode->crtc_hdisplay;
2195	wm_high.active_time = active_time;
2196	wm_high.blank_time = line_time - wm_high.active_time;
2197	wm_high.interlaced = false;
2198	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2199	wm_high.interlaced = true;
2200	wm_high.vsc = radeon_crtc->vsc;
2201	wm_high.vtaps = 1;
2202	if (radeon_crtc->rmx_type != RMX_OFF)
2203	wm_high.vtaps = 2;
2204	wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2205	wm_high.lb_size = lb_size;
2206	wm_high.dram_channels = dram_channels;
2207	wm_high.num_heads = num_heads;
2208
2209	/* watermark for low clocks */
2210	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2211	wm_low.yclk =
2212	radeon_dpm_get_mclk(rdev, true) * 10;
2213	wm_low.sclk =
2214	radeon_dpm_get_sclk(rdev, true) * 10;
2215	} else {
2216	wm_low.yclk = rdev->pm.current_mclk * 10;
2217	wm_low.sclk = rdev->pm.current_sclk * 10;
2218	}
2219
2220	wm_low.disp_clk = mode->clock;
2221	wm_low.src_width = mode->crtc_hdisplay;
2222	wm_low.active_time = active_time;
2223	wm_low.blank_time = line_time - wm_low.active_time;
2224	wm_low.interlaced = false;
2225	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2226	wm_low.interlaced = true;
2227	wm_low.vsc = radeon_crtc->vsc;
2228	wm_low.vtaps = 1;
2229	if (radeon_crtc->rmx_type != RMX_OFF)
2230	wm_low.vtaps = 2;
2231	wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2232	wm_low.lb_size = lb_size;
2233	wm_low.dram_channels = dram_channels;
2234	wm_low.num_heads = num_heads;
2235
2236	/* set for high clocks */
2237	latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2238	/* set for low clocks */
2239	latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2240
2241	/* possibly force display priority to high */
2242	/* should really do this at mode validation time... */
2243	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(wm: &wm_high) ||
2244	!evergreen_average_bandwidth_vs_available_bandwidth(wm: &wm_high) ||
2245	!evergreen_check_latency_hiding(wm: &wm_high) ||
2246	(rdev->disp_priority == 2)) {
2247	DRM_DEBUG_KMS("force priority a to high\n");
2248	priority_a_cnt |= PRIORITY_ALWAYS_ON;
2249	}
2250	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(wm: &wm_low) ||
2251	!evergreen_average_bandwidth_vs_available_bandwidth(wm: &wm_low) ||
2252	!evergreen_check_latency_hiding(wm: &wm_low) ||
2253	(rdev->disp_priority == 2)) {
2254	DRM_DEBUG_KMS("force priority b to high\n");
2255	priority_b_cnt |= PRIORITY_ALWAYS_ON;
2256	}
2257
2258	a.full = dfixed_const(1000);
2259	b.full = dfixed_const(mode->clock);
2260	b.full = dfixed_div(A: b, B: a);
2261	c.full = dfixed_const(latency_watermark_a);
2262	c.full = dfixed_mul(c, b);
2263	c.full = dfixed_mul(c, radeon_crtc->hsc);
2264	c.full = dfixed_div(A: c, B: a);
2265	a.full = dfixed_const(16);
2266	c.full = dfixed_div(A: c, B: a);
2267	priority_a_mark = dfixed_trunc(c);
2268	priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2269
2270	a.full = dfixed_const(1000);
2271	b.full = dfixed_const(mode->clock);
2272	b.full = dfixed_div(A: b, B: a);
2273	c.full = dfixed_const(latency_watermark_b);
2274	c.full = dfixed_mul(c, b);
2275	c.full = dfixed_mul(c, radeon_crtc->hsc);
2276	c.full = dfixed_div(A: c, B: a);
2277	a.full = dfixed_const(16);
2278	c.full = dfixed_div(A: c, B: a);
2279	priority_b_mark = dfixed_trunc(c);
2280	priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2281
2282	/* Save number of lines the linebuffer leads before the scanout */
2283	radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2284	}
2285
2286	/* select wm A */
2287	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2288	tmp = arb_control3;
2289	tmp &= ~LATENCY_WATERMARK_MASK(3);
2290	tmp |= LATENCY_WATERMARK_MASK(1);
2291	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293	(LATENCY_LOW_WATERMARK(latency_watermark_a) |
2294	LATENCY_HIGH_WATERMARK(line_time)));
2295	/* select wm B */
2296	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2297	tmp &= ~LATENCY_WATERMARK_MASK(3);
2298	tmp |= LATENCY_WATERMARK_MASK(2);
2299	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2300	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2301	(LATENCY_LOW_WATERMARK(latency_watermark_b) |
2302	LATENCY_HIGH_WATERMARK(line_time)));
2303	/* restore original selection */
2304	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2305
2306	/* write the priority marks */
2307	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2308	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2309
2310	/* save values for DPM */
2311	radeon_crtc->line_time = line_time;
2312	radeon_crtc->wm_high = latency_watermark_a;
2313	radeon_crtc->wm_low = latency_watermark_b;
2314	}
2315
2316	/**
2317	* evergreen_bandwidth_update - update display watermarks callback.
2318	*
2319	* @rdev: radeon_device pointer
2320	*
2321	* Update the display watermarks based on the requested mode(s)
2322	* (evergreen+).
2323	*/
2324	void evergreen_bandwidth_update(struct radeon_device *rdev)
2325	{
2326	struct drm_display_mode *mode0 = NULL;
2327	struct drm_display_mode *mode1 = NULL;
2328	u32 num_heads = 0, lb_size;
2329	int i;
2330
2331	if (!rdev->mode_info.mode_config_initialized)
2332	return;
2333
2334	radeon_update_display_priority(rdev);
2335
2336	for (i = 0; i < rdev->num_crtc; i++) {
2337	if (rdev->mode_info.crtcs[i]->base.enabled)
2338	num_heads++;
2339	}
2340	for (i = 0; i < rdev->num_crtc; i += 2) {
2341	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2342	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2343	lb_size = evergreen_line_buffer_adjust(rdev, radeon_crtc: rdev->mode_info.crtcs[i], mode: mode0, other_mode: mode1);
2344	evergreen_program_watermarks(rdev, radeon_crtc: rdev->mode_info.crtcs[i], lb_size, num_heads);
2345	lb_size = evergreen_line_buffer_adjust(rdev, radeon_crtc: rdev->mode_info.crtcs[i+1], mode: mode1, other_mode: mode0);
2346	evergreen_program_watermarks(rdev, radeon_crtc: rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2347	}
2348	}
2349
2350	/**
2351	* evergreen_mc_wait_for_idle - wait for MC idle callback.
2352	*
2353	* @rdev: radeon_device pointer
2354	*
2355	* Wait for the MC (memory controller) to be idle.
2356	* (evergreen+).
2357	* Returns 0 if the MC is idle, -1 if not.
2358	*/
2359	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2360	{
2361	unsigned i;
2362	u32 tmp;
2363
2364	for (i = 0; i < rdev->usec_timeout; i++) {
2365	/* read MC_STATUS */
2366	tmp = RREG32(SRBM_STATUS) & 0x1F00;
2367	if (!tmp)
2368	return 0;
2369	udelay(usec: 1);
2370	}
2371	return -1;
2372	}
2373
2374	/*
2375	* GART
2376	*/
2377	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2378	{
2379	unsigned i;
2380	u32 tmp;
2381
2382	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2383
2384	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2385	for (i = 0; i < rdev->usec_timeout; i++) {
2386	/* read MC_STATUS */
2387	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2388	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2389	if (tmp == 2) {
2390	pr_warn("[drm] r600 flush TLB failed\n");
2391	return;
2392	}
2393	if (tmp) {
2394	return;
2395	}
2396	udelay(usec: 1);
2397	}
2398	}
2399
2400	static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2401	{
2402	u32 tmp;
2403	int r;
2404
2405	if (rdev->gart.robj == NULL) {
2406	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2407	return -EINVAL;
2408	}
2409	r = radeon_gart_table_vram_pin(rdev);
2410	if (r)
2411	return r;
2412	/* Setup L2 cache */
2413	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2414	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2415	EFFECTIVE_L2_QUEUE_SIZE(7));
2416	WREG32(VM_L2_CNTL2, 0);
2417	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2418	/* Setup TLB control */
2419	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2420	SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2421	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2422	EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2423	if (rdev->flags & RADEON_IS_IGP) {
2424	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2425	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2426	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2427	} else {
2428	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2429	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2430	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2431	if ((rdev->family == CHIP_JUNIPER) ||
2432	(rdev->family == CHIP_CYPRESS) ||
2433	(rdev->family == CHIP_HEMLOCK) ||
2434	(rdev->family == CHIP_BARTS))
2435	WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2436	}
2437	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2438	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2439	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2440	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2441	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2442	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2443	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2444	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2445	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2446	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2447	(u32)(rdev->dummy_page.addr >> 12));
2448	WREG32(VM_CONTEXT1_CNTL, 0);
2449
2450	evergreen_pcie_gart_tlb_flush(rdev);
2451	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2452	(unsigned)(rdev->mc.gtt_size >> 20),
2453	(unsigned long long)rdev->gart.table_addr);
2454	rdev->gart.ready = true;
2455	return 0;
2456	}
2457
2458	static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2459	{
2460	u32 tmp;
2461
2462	/* Disable all tables */
2463	WREG32(VM_CONTEXT0_CNTL, 0);
2464	WREG32(VM_CONTEXT1_CNTL, 0);
2465
2466	/* Setup L2 cache */
2467	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2468	EFFECTIVE_L2_QUEUE_SIZE(7));
2469	WREG32(VM_L2_CNTL2, 0);
2470	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2471	/* Setup TLB control */
2472	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2473	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2474	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2475	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2476	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2477	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2478	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2479	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2480	radeon_gart_table_vram_unpin(rdev);
2481	}
2482
2483	static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2484	{
2485	evergreen_pcie_gart_disable(rdev);
2486	radeon_gart_table_vram_free(rdev);
2487	radeon_gart_fini(rdev);
2488	}
2489
2490
2491	static void evergreen_agp_enable(struct radeon_device *rdev)
2492	{
2493	u32 tmp;
2494
2495	/* Setup L2 cache */
2496	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2497	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2498	EFFECTIVE_L2_QUEUE_SIZE(7));
2499	WREG32(VM_L2_CNTL2, 0);
2500	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2501	/* Setup TLB control */
2502	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2503	SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2504	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2505	EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2506	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2507	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2508	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2509	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2510	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2511	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2512	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2513	WREG32(VM_CONTEXT0_CNTL, 0);
2514	WREG32(VM_CONTEXT1_CNTL, 0);
2515	}
2516
2517	static const unsigned ni_dig_offsets[] = {
2518	NI_DIG0_REGISTER_OFFSET,
2519	NI_DIG1_REGISTER_OFFSET,
2520	NI_DIG2_REGISTER_OFFSET,
2521	NI_DIG3_REGISTER_OFFSET,
2522	NI_DIG4_REGISTER_OFFSET,
2523	NI_DIG5_REGISTER_OFFSET
2524	};
2525
2526	static const unsigned ni_tx_offsets[] = {
2527	NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528	NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529	NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530	NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531	NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532	NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2533	};
2534
2535	static const unsigned evergreen_dp_offsets[] = {
2536	EVERGREEN_DP0_REGISTER_OFFSET,
2537	EVERGREEN_DP1_REGISTER_OFFSET,
2538	EVERGREEN_DP2_REGISTER_OFFSET,
2539	EVERGREEN_DP3_REGISTER_OFFSET,
2540	EVERGREEN_DP4_REGISTER_OFFSET,
2541	EVERGREEN_DP5_REGISTER_OFFSET
2542	};
2543
2544	static const unsigned evergreen_disp_int_status[] = {
2545	DISP_INTERRUPT_STATUS,
2546	DISP_INTERRUPT_STATUS_CONTINUE,
2547	DISP_INTERRUPT_STATUS_CONTINUE2,
2548	DISP_INTERRUPT_STATUS_CONTINUE3,
2549	DISP_INTERRUPT_STATUS_CONTINUE4,
2550	DISP_INTERRUPT_STATUS_CONTINUE5
2551	};
2552
2553	/*
2554	* Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2555	* We go from crtc to connector and it is not relible since it
2556	* should be an opposite direction .If crtc is enable then
2557	* find the dig_fe which selects this crtc and insure that it enable.
2558	* if such dig_fe is found then find dig_be which selects found dig_be and
2559	* insure that it enable and in DP_SST mode.
2560	* if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2561	* from dp symbols clocks .
2562	*/
2563	static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2564	unsigned crtc_id, unsigned *ret_dig_fe)
2565	{
2566	unsigned i;
2567	unsigned dig_fe;
2568	unsigned dig_be;
2569	unsigned dig_en_be;
2570	unsigned uniphy_pll;
2571	unsigned digs_fe_selected;
2572	unsigned dig_be_mode;
2573	unsigned dig_fe_mask;
2574	bool is_enabled = false;
2575	bool found_crtc = false;
2576
2577	/* loop through all running dig_fe to find selected crtc */
2578	for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2579	dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2580	if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2581	crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2582	/* found running pipe */
2583	found_crtc = true;
2584	dig_fe_mask = 1 << i;
2585	dig_fe = i;
2586	break;
2587	}
2588	}
2589
2590	if (found_crtc) {
2591	/* loop through all running dig_be to find selected dig_fe */
2592	for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2593	dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2594	/* if dig_fe_selected by dig_be? */
2595	digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2596	dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2597	if (dig_fe_mask & digs_fe_selected &&
2598	/* if dig_be in sst mode? */
2599	dig_be_mode == NI_DIG_BE_DPSST) {
2600	dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2601	ni_dig_offsets[i]);
2602	uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2603	ni_tx_offsets[i]);
2604	/* dig_be enable and tx is running */
2605	if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2606	dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2607	uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2608	is_enabled = true;
2609	*ret_dig_fe = dig_fe;
2610	break;
2611	}
2612	}
2613	}
2614	}
2615
2616	return is_enabled;
2617	}
2618
2619	/*
2620	* Blank dig when in dp sst mode
2621	* Dig ignores crtc timing
2622	*/
2623	static void evergreen_blank_dp_output(struct radeon_device *rdev,
2624	unsigned dig_fe)
2625	{
2626	unsigned stream_ctrl;
2627	unsigned fifo_ctrl;
2628	unsigned counter = 0;
2629
2630	if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2631	DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2632	return;
2633	}
2634
2635	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2636	evergreen_dp_offsets[dig_fe]);
2637	if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2638	DRM_ERROR("dig %d , should be enable\n", dig_fe);
2639	return;
2640	}
2641
2642	stream_ctrl &= ~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2643	WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2644	evergreen_dp_offsets[dig_fe], stream_ctrl);
2645
2646	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2647	evergreen_dp_offsets[dig_fe]);
2648	while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2649	msleep(msecs: 1);
2650	counter++;
2651	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2652	evergreen_dp_offsets[dig_fe]);
2653	}
2654	if (counter >= 32)
2655	DRM_ERROR("counter exceeds %d\n", counter);
2656
2657	fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2658	fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2659	WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2660
2661	}
2662
2663	void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2664	{
2665	u32 crtc_enabled, tmp, frame_count, blackout;
2666	int i, j;
2667	unsigned dig_fe;
2668
2669	if (!ASIC_IS_NODCE(rdev)) {
2670	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2671	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2672
2673	/* disable VGA render */
2674	WREG32(VGA_RENDER_CONTROL, 0);
2675	}
2676	/* blank the display controllers */
2677	for (i = 0; i < rdev->num_crtc; i++) {
2678	crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2679	if (crtc_enabled) {
2680	save->crtc_enabled[i] = true;
2681	if (ASIC_IS_DCE6(rdev)) {
2682	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2683	if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2684	radeon_wait_for_vblank(rdev, i);
2685	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2686	tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2687	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2688	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2689	}
2690	} else {
2691	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2692	if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2693	radeon_wait_for_vblank(rdev, i);
2694	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2695	tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2696	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2697	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2698	}
2699	}
2700	/* wait for the next frame */
2701	frame_count = radeon_get_vblank_counter(rdev, i);
2702	for (j = 0; j < rdev->usec_timeout; j++) {
2703	if (radeon_get_vblank_counter(rdev, i) != frame_count)
2704	break;
2705	udelay(usec: 1);
2706	}
2707	/*we should disable dig if it drives dp sst*/
2708	/*but we are in radeon_device_init and the topology is unknown*/
2709	/*and it is available after radeon_modeset_init*/
2710	/*the following method radeon_atom_encoder_dpms_dig*/
2711	/*does the job if we initialize it properly*/
2712	/*for now we do it this manually*/
2713	/**/
2714	if (ASIC_IS_DCE5(rdev) &&
2715	evergreen_is_dp_sst_stream_enabled(rdev, crtc_id: i, ret_dig_fe: &dig_fe))
2716	evergreen_blank_dp_output(rdev, dig_fe);
2717	/*we could remove 6 lines below*/
2718	/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2719	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2720	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2721	tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2722	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2723	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2724	save->crtc_enabled[i] = false;
2725	/* ***** */
2726	} else {
2727	save->crtc_enabled[i] = false;
2728	}
2729	}
2730
2731	radeon_mc_wait_for_idle(rdev);
2732
2733	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2734	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2735	/* Block CPU access */
2736	WREG32(BIF_FB_EN, 0);
2737	/* blackout the MC */
2738	blackout &= ~BLACKOUT_MODE_MASK;
2739	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2740	}
2741	/* wait for the MC to settle */
2742	udelay(usec: 100);
2743
2744	/* lock double buffered regs */
2745	for (i = 0; i < rdev->num_crtc; i++) {
2746	if (save->crtc_enabled[i]) {
2747	tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2748	if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2749	tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2750	WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2751	}
2752	tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2753	if (!(tmp & 1)) {
2754	tmp |= 1;
2755	WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2756	}
2757	}
2758	}
2759	}
2760
2761	void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2762	{
2763	u32 tmp, frame_count;
2764	int i, j;
2765
2766	/* update crtc base addresses */
2767	for (i = 0; i < rdev->num_crtc; i++) {
2768	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2769	upper_32_bits(rdev->mc.vram_start));
2770	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771	upper_32_bits(rdev->mc.vram_start));
2772	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2773	(u32)rdev->mc.vram_start);
2774	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2775	(u32)rdev->mc.vram_start);
2776	}
2777
2778	if (!ASIC_IS_NODCE(rdev)) {
2779	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2780	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2781	}
2782
2783	/* unlock regs and wait for update */
2784	for (i = 0; i < rdev->num_crtc; i++) {
2785	if (save->crtc_enabled[i]) {
2786	tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2787	if ((tmp & 0x7) != 0) {
2788	tmp &= ~0x7;
2789	WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2790	}
2791	tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2792	if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2793	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2794	WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2795	}
2796	tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2797	if (tmp & 1) {
2798	tmp &= ~1;
2799	WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2800	}
2801	for (j = 0; j < rdev->usec_timeout; j++) {
2802	tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2803	if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2804	break;
2805	udelay(usec: 1);
2806	}
2807	}
2808	}
2809
2810	/* unblackout the MC */
2811	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2812	tmp &= ~BLACKOUT_MODE_MASK;
2813	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2814	/* allow CPU access */
2815	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2816
2817	for (i = 0; i < rdev->num_crtc; i++) {
2818	if (save->crtc_enabled[i]) {
2819	if (ASIC_IS_DCE6(rdev)) {
2820	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2821	tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2822	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2823	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2824	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2825	} else {
2826	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2827	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2828	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2829	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2830	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2831	}
2832	/* wait for the next frame */
2833	frame_count = radeon_get_vblank_counter(rdev, i);
2834	for (j = 0; j < rdev->usec_timeout; j++) {
2835	if (radeon_get_vblank_counter(rdev, i) != frame_count)
2836	break;
2837	udelay(usec: 1);
2838	}
2839	}
2840	}
2841	if (!ASIC_IS_NODCE(rdev)) {
2842	/* Unlock vga access */
2843	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2844	mdelay(1);
2845	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2846	}
2847	}
2848
2849	void evergreen_mc_program(struct radeon_device *rdev)
2850	{
2851	struct evergreen_mc_save save;
2852	u32 tmp;
2853	int i, j;
2854
2855	/* Initialize HDP */
2856	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2857	WREG32((0x2c14 + j), 0x00000000);
2858	WREG32((0x2c18 + j), 0x00000000);
2859	WREG32((0x2c1c + j), 0x00000000);
2860	WREG32((0x2c20 + j), 0x00000000);
2861	WREG32((0x2c24 + j), 0x00000000);
2862	}
2863	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2864
2865	evergreen_mc_stop(rdev, save: &save);
2866	if (evergreen_mc_wait_for_idle(rdev)) {
2867	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2868	}
2869	/* Lockout access through VGA aperture*/
2870	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2871	/* Update configuration */
2872	if (rdev->flags & RADEON_IS_AGP) {
2873	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2874	/* VRAM before AGP */
2875	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2876	rdev->mc.vram_start >> 12);
2877	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2878	rdev->mc.gtt_end >> 12);
2879	} else {
2880	/* VRAM after AGP */
2881	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2882	rdev->mc.gtt_start >> 12);
2883	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2884	rdev->mc.vram_end >> 12);
2885	}
2886	} else {
2887	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2888	rdev->mc.vram_start >> 12);
2889	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2890	rdev->mc.vram_end >> 12);
2891	}
2892	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2893	/* llano/ontario only */
2894	if ((rdev->family == CHIP_PALM) ||
2895	(rdev->family == CHIP_SUMO) ||
2896	(rdev->family == CHIP_SUMO2)) {
2897	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2898	tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2899	tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2900	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2901	}
2902	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2903	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2904	WREG32(MC_VM_FB_LOCATION, tmp);
2905	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2906	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2907	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2908	if (rdev->flags & RADEON_IS_AGP) {
2909	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2910	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2911	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2912	} else {
2913	WREG32(MC_VM_AGP_BASE, 0);
2914	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2915	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2916	}
2917	if (evergreen_mc_wait_for_idle(rdev)) {
2918	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2919	}
2920	evergreen_mc_resume(rdev, save: &save);
2921	/* we need to own VRAM, so turn off the VGA renderer here
2922	* to stop it overwriting our objects */
2923	rv515_vga_render_disable(rdev);
2924	}
2925
2926	/*
2927	* CP.
2928	*/
2929	void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2930	{
2931	struct radeon_ring *ring = &rdev->ring[ib->ring];
2932	u32 next_rptr;
2933
2934	/* set to DX10/11 mode */
2935	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2936	radeon_ring_write(ring, v: 1);
2937
2938	if (ring->rptr_save_reg) {
2939	next_rptr = ring->wptr + 3 + 4;
2940	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2941	radeon_ring_write(ring, v: ((ring->rptr_save_reg -
2942	PACKET3_SET_CONFIG_REG_START) >> 2));
2943	radeon_ring_write(ring, v: next_rptr);
2944	} else if (rdev->wb.enabled) {
2945	next_rptr = ring->wptr + 5 + 4;
2946	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2947	radeon_ring_write(ring, v: ring->next_rptr_gpu_addr & 0xfffffffc);
2948	radeon_ring_write(ring, v: (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2949	radeon_ring_write(ring, v: next_rptr);
2950	radeon_ring_write(ring, v: 0);
2951	}
2952
2953	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2954	radeon_ring_write(ring,
2955	#ifdef __BIG_ENDIAN
2956	(2 << 0) |
2957	#endif
2958	v: (ib->gpu_addr & 0xFFFFFFFC));
2959	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2960	radeon_ring_write(ring, v: ib->length_dw);
2961	}
2962
2963
2964	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2965	{
2966	const __be32 *fw_data;
2967	int i;
2968
2969	if (!rdev->me_fw || !rdev->pfp_fw)
2970	return -EINVAL;
2971
2972	r700_cp_stop(rdev);
2973	WREG32(CP_RB_CNTL,
2974	#ifdef __BIG_ENDIAN
2975	BUF_SWAP_32BIT |
2976	#endif
2977	RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2978
2979	fw_data = (const __be32 *)rdev->pfp_fw->data;
2980	WREG32(CP_PFP_UCODE_ADDR, 0);
2981	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2982	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2983	WREG32(CP_PFP_UCODE_ADDR, 0);
2984
2985	fw_data = (const __be32 *)rdev->me_fw->data;
2986	WREG32(CP_ME_RAM_WADDR, 0);
2987	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2988	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2989
2990	WREG32(CP_PFP_UCODE_ADDR, 0);
2991	WREG32(CP_ME_RAM_WADDR, 0);
2992	WREG32(CP_ME_RAM_RADDR, 0);
2993	return 0;
2994	}
2995
2996	static int evergreen_cp_start(struct radeon_device *rdev)
2997	{
2998	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2999	int r, i;
3000	uint32_t cp_me;
3001
3002	r = radeon_ring_lock(rdev, cp: ring, ndw: 7);
3003	if (r) {
3004	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3005	return r;
3006	}
3007	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3008	radeon_ring_write(ring, v: 0x1);
3009	radeon_ring_write(ring, v: 0x0);
3010	radeon_ring_write(ring, v: rdev->config.evergreen.max_hw_contexts - 1);
3011	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3012	radeon_ring_write(ring, v: 0);
3013	radeon_ring_write(ring, v: 0);
3014	radeon_ring_unlock_commit(rdev, cp: ring, hdp_flush: false);
3015
3016	cp_me = 0xff;
3017	WREG32(CP_ME_CNTL, cp_me);
3018
3019	r = radeon_ring_lock(rdev, cp: ring, ndw: evergreen_default_size + 19);
3020	if (r) {
3021	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3022	return r;
3023	}
3024
3025	/* setup clear context state */
3026	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3027	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3028
3029	for (i = 0; i < evergreen_default_size; i++)
3030	radeon_ring_write(ring, v: evergreen_default_state[i]);
3031
3032	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3033	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3034
3035	/* set clear context state */
3036	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3037	radeon_ring_write(ring, v: 0);
3038
3039	/* SQ_VTX_BASE_VTX_LOC */
3040	radeon_ring_write(ring, v: 0xc0026f00);
3041	radeon_ring_write(ring, v: 0x00000000);
3042	radeon_ring_write(ring, v: 0x00000000);
3043	radeon_ring_write(ring, v: 0x00000000);
3044
3045	/* Clear consts */
3046	radeon_ring_write(ring, v: 0xc0036f00);
3047	radeon_ring_write(ring, v: 0x00000bc4);
3048	radeon_ring_write(ring, v: 0xffffffff);
3049	radeon_ring_write(ring, v: 0xffffffff);
3050	radeon_ring_write(ring, v: 0xffffffff);
3051
3052	radeon_ring_write(ring, v: 0xc0026900);
3053	radeon_ring_write(ring, v: 0x00000316);
3054	radeon_ring_write(ring, v: 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3055	radeon_ring_write(ring, v: 0x00000010); /* */
3056
3057	radeon_ring_unlock_commit(rdev, cp: ring, hdp_flush: false);
3058
3059	return 0;
3060	}
3061
3062	static int evergreen_cp_resume(struct radeon_device *rdev)
3063	{
3064	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3065	u32 tmp;
3066	u32 rb_bufsz;
3067	int r;
3068
3069	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3070	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3071	SOFT_RESET_PA |
3072	SOFT_RESET_SH |
3073	SOFT_RESET_VGT |
3074	SOFT_RESET_SPI |
3075	SOFT_RESET_SX));
3076	RREG32(GRBM_SOFT_RESET);
3077	mdelay(15);
3078	WREG32(GRBM_SOFT_RESET, 0);
3079	RREG32(GRBM_SOFT_RESET);
3080
3081	/* Set ring buffer size */
3082	rb_bufsz = order_base_2(ring->ring_size / 8);
3083	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3084	#ifdef __BIG_ENDIAN
3085	tmp |= BUF_SWAP_32BIT;
3086	#endif
3087	WREG32(CP_RB_CNTL, tmp);
3088	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3089	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3090
3091	/* Set the write pointer delay */
3092	WREG32(CP_RB_WPTR_DELAY, 0);
3093
3094	/* Initialize the ring buffer's read and write pointers */
3095	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3096	WREG32(CP_RB_RPTR_WR, 0);
3097	ring->wptr = 0;
3098	WREG32(CP_RB_WPTR, ring->wptr);
3099
3100	/* set the wb address whether it's enabled or not */
3101	WREG32(CP_RB_RPTR_ADDR,
3102	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3103	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3104	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3105
3106	if (rdev->wb.enabled)
3107	WREG32(SCRATCH_UMSK, 0xff);
3108	else {
3109	tmp |= RB_NO_UPDATE;
3110	WREG32(SCRATCH_UMSK, 0);
3111	}
3112
3113	mdelay(1);
3114	WREG32(CP_RB_CNTL, tmp);
3115
3116	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3117	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3118
3119	evergreen_cp_start(rdev);
3120	ring->ready = true;
3121	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3122	if (r) {
3123	ring->ready = false;
3124	return r;
3125	}
3126	return 0;
3127	}
3128
3129	/*
3130	* Core functions
3131	*/
3132	static void evergreen_gpu_init(struct radeon_device *rdev)
3133	{
3134	u32 gb_addr_config;
3135	u32 mc_arb_ramcfg;
3136	u32 sx_debug_1;
3137	u32 smx_dc_ctl0;
3138	u32 sq_config;
3139	u32 sq_lds_resource_mgmt;
3140	u32 sq_gpr_resource_mgmt_1;
3141	u32 sq_gpr_resource_mgmt_2;
3142	u32 sq_gpr_resource_mgmt_3;
3143	u32 sq_thread_resource_mgmt;
3144	u32 sq_thread_resource_mgmt_2;
3145	u32 sq_stack_resource_mgmt_1;
3146	u32 sq_stack_resource_mgmt_2;
3147	u32 sq_stack_resource_mgmt_3;
3148	u32 vgt_cache_invalidation;
3149	u32 hdp_host_path_cntl, tmp;
3150	u32 disabled_rb_mask;
3151	int i, j, ps_thread_count;
3152
3153	switch (rdev->family) {
3154	case CHIP_CYPRESS:
3155	case CHIP_HEMLOCK:
3156	rdev->config.evergreen.num_ses = 2;
3157	rdev->config.evergreen.max_pipes = 4;
3158	rdev->config.evergreen.max_tile_pipes = 8;
3159	rdev->config.evergreen.max_simds = 10;
3160	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3161	rdev->config.evergreen.max_gprs = 256;
3162	rdev->config.evergreen.max_threads = 248;
3163	rdev->config.evergreen.max_gs_threads = 32;
3164	rdev->config.evergreen.max_stack_entries = 512;
3165	rdev->config.evergreen.sx_num_of_sets = 4;
3166	rdev->config.evergreen.sx_max_export_size = 256;
3167	rdev->config.evergreen.sx_max_export_pos_size = 64;
3168	rdev->config.evergreen.sx_max_export_smx_size = 192;
3169	rdev->config.evergreen.max_hw_contexts = 8;
3170	rdev->config.evergreen.sq_num_cf_insts = 2;
3171
3172	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3173	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3174	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3175	gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3176	break;
3177	case CHIP_JUNIPER:
3178	rdev->config.evergreen.num_ses = 1;
3179	rdev->config.evergreen.max_pipes = 4;
3180	rdev->config.evergreen.max_tile_pipes = 4;
3181	rdev->config.evergreen.max_simds = 10;
3182	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3183	rdev->config.evergreen.max_gprs = 256;
3184	rdev->config.evergreen.max_threads = 248;
3185	rdev->config.evergreen.max_gs_threads = 32;
3186	rdev->config.evergreen.max_stack_entries = 512;
3187	rdev->config.evergreen.sx_num_of_sets = 4;
3188	rdev->config.evergreen.sx_max_export_size = 256;
3189	rdev->config.evergreen.sx_max_export_pos_size = 64;
3190	rdev->config.evergreen.sx_max_export_smx_size = 192;
3191	rdev->config.evergreen.max_hw_contexts = 8;
3192	rdev->config.evergreen.sq_num_cf_insts = 2;
3193
3194	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3195	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3196	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3197	gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3198	break;
3199	case CHIP_REDWOOD:
3200	rdev->config.evergreen.num_ses = 1;
3201	rdev->config.evergreen.max_pipes = 4;
3202	rdev->config.evergreen.max_tile_pipes = 4;
3203	rdev->config.evergreen.max_simds = 5;
3204	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3205	rdev->config.evergreen.max_gprs = 256;
3206	rdev->config.evergreen.max_threads = 248;
3207	rdev->config.evergreen.max_gs_threads = 32;
3208	rdev->config.evergreen.max_stack_entries = 256;
3209	rdev->config.evergreen.sx_num_of_sets = 4;
3210	rdev->config.evergreen.sx_max_export_size = 256;
3211	rdev->config.evergreen.sx_max_export_pos_size = 64;
3212	rdev->config.evergreen.sx_max_export_smx_size = 192;
3213	rdev->config.evergreen.max_hw_contexts = 8;
3214	rdev->config.evergreen.sq_num_cf_insts = 2;
3215
3216	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3217	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3218	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3219	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3220	break;
3221	case CHIP_CEDAR:
3222	default:
3223	rdev->config.evergreen.num_ses = 1;
3224	rdev->config.evergreen.max_pipes = 2;
3225	rdev->config.evergreen.max_tile_pipes = 2;
3226	rdev->config.evergreen.max_simds = 2;
3227	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3228	rdev->config.evergreen.max_gprs = 256;
3229	rdev->config.evergreen.max_threads = 192;
3230	rdev->config.evergreen.max_gs_threads = 16;
3231	rdev->config.evergreen.max_stack_entries = 256;
3232	rdev->config.evergreen.sx_num_of_sets = 4;
3233	rdev->config.evergreen.sx_max_export_size = 128;
3234	rdev->config.evergreen.sx_max_export_pos_size = 32;
3235	rdev->config.evergreen.sx_max_export_smx_size = 96;
3236	rdev->config.evergreen.max_hw_contexts = 4;
3237	rdev->config.evergreen.sq_num_cf_insts = 1;
3238
3239	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3240	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3241	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3242	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3243	break;
3244	case CHIP_PALM:
3245	rdev->config.evergreen.num_ses = 1;
3246	rdev->config.evergreen.max_pipes = 2;
3247	rdev->config.evergreen.max_tile_pipes = 2;
3248	rdev->config.evergreen.max_simds = 2;
3249	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3250	rdev->config.evergreen.max_gprs = 256;
3251	rdev->config.evergreen.max_threads = 192;
3252	rdev->config.evergreen.max_gs_threads = 16;
3253	rdev->config.evergreen.max_stack_entries = 256;
3254	rdev->config.evergreen.sx_num_of_sets = 4;
3255	rdev->config.evergreen.sx_max_export_size = 128;
3256	rdev->config.evergreen.sx_max_export_pos_size = 32;
3257	rdev->config.evergreen.sx_max_export_smx_size = 96;
3258	rdev->config.evergreen.max_hw_contexts = 4;
3259	rdev->config.evergreen.sq_num_cf_insts = 1;
3260
3261	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3262	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3263	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3264	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3265	break;
3266	case CHIP_SUMO:
3267	rdev->config.evergreen.num_ses = 1;
3268	rdev->config.evergreen.max_pipes = 4;
3269	rdev->config.evergreen.max_tile_pipes = 4;
3270	if (rdev->pdev->device == 0x9648)
3271	rdev->config.evergreen.max_simds = 3;
3272	else if ((rdev->pdev->device == 0x9647) ||
3273	(rdev->pdev->device == 0x964a))
3274	rdev->config.evergreen.max_simds = 4;
3275	else
3276	rdev->config.evergreen.max_simds = 5;
3277	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3278	rdev->config.evergreen.max_gprs = 256;
3279	rdev->config.evergreen.max_threads = 248;
3280	rdev->config.evergreen.max_gs_threads = 32;
3281	rdev->config.evergreen.max_stack_entries = 256;
3282	rdev->config.evergreen.sx_num_of_sets = 4;
3283	rdev->config.evergreen.sx_max_export_size = 256;
3284	rdev->config.evergreen.sx_max_export_pos_size = 64;
3285	rdev->config.evergreen.sx_max_export_smx_size = 192;
3286	rdev->config.evergreen.max_hw_contexts = 8;
3287	rdev->config.evergreen.sq_num_cf_insts = 2;
3288
3289	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3290	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3291	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3292	gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3293	break;
3294	case CHIP_SUMO2:
3295	rdev->config.evergreen.num_ses = 1;
3296	rdev->config.evergreen.max_pipes = 4;
3297	rdev->config.evergreen.max_tile_pipes = 4;
3298	rdev->config.evergreen.max_simds = 2;
3299	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3300	rdev->config.evergreen.max_gprs = 256;
3301	rdev->config.evergreen.max_threads = 248;
3302	rdev->config.evergreen.max_gs_threads = 32;
3303	rdev->config.evergreen.max_stack_entries = 512;
3304	rdev->config.evergreen.sx_num_of_sets = 4;
3305	rdev->config.evergreen.sx_max_export_size = 256;
3306	rdev->config.evergreen.sx_max_export_pos_size = 64;
3307	rdev->config.evergreen.sx_max_export_smx_size = 192;
3308	rdev->config.evergreen.max_hw_contexts = 4;
3309	rdev->config.evergreen.sq_num_cf_insts = 2;
3310
3311	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3312	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3313	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3314	gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3315	break;
3316	case CHIP_BARTS:
3317	rdev->config.evergreen.num_ses = 2;
3318	rdev->config.evergreen.max_pipes = 4;
3319	rdev->config.evergreen.max_tile_pipes = 8;
3320	rdev->config.evergreen.max_simds = 7;
3321	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3322	rdev->config.evergreen.max_gprs = 256;
3323	rdev->config.evergreen.max_threads = 248;
3324	rdev->config.evergreen.max_gs_threads = 32;
3325	rdev->config.evergreen.max_stack_entries = 512;
3326	rdev->config.evergreen.sx_num_of_sets = 4;
3327	rdev->config.evergreen.sx_max_export_size = 256;
3328	rdev->config.evergreen.sx_max_export_pos_size = 64;
3329	rdev->config.evergreen.sx_max_export_smx_size = 192;
3330	rdev->config.evergreen.max_hw_contexts = 8;
3331	rdev->config.evergreen.sq_num_cf_insts = 2;
3332
3333	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3334	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3335	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3336	gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3337	break;
3338	case CHIP_TURKS:
3339	rdev->config.evergreen.num_ses = 1;
3340	rdev->config.evergreen.max_pipes = 4;
3341	rdev->config.evergreen.max_tile_pipes = 4;
3342	rdev->config.evergreen.max_simds = 6;
3343	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3344	rdev->config.evergreen.max_gprs = 256;
3345	rdev->config.evergreen.max_threads = 248;
3346	rdev->config.evergreen.max_gs_threads = 32;
3347	rdev->config.evergreen.max_stack_entries = 256;
3348	rdev->config.evergreen.sx_num_of_sets = 4;
3349	rdev->config.evergreen.sx_max_export_size = 256;
3350	rdev->config.evergreen.sx_max_export_pos_size = 64;
3351	rdev->config.evergreen.sx_max_export_smx_size = 192;
3352	rdev->config.evergreen.max_hw_contexts = 8;
3353	rdev->config.evergreen.sq_num_cf_insts = 2;
3354
3355	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3356	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3357	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3358	gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3359	break;
3360	case CHIP_CAICOS:
3361	rdev->config.evergreen.num_ses = 1;
3362	rdev->config.evergreen.max_pipes = 2;
3363	rdev->config.evergreen.max_tile_pipes = 2;
3364	rdev->config.evergreen.max_simds = 2;
3365	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3366	rdev->config.evergreen.max_gprs = 256;
3367	rdev->config.evergreen.max_threads = 192;
3368	rdev->config.evergreen.max_gs_threads = 16;
3369	rdev->config.evergreen.max_stack_entries = 256;
3370	rdev->config.evergreen.sx_num_of_sets = 4;
3371	rdev->config.evergreen.sx_max_export_size = 128;
3372	rdev->config.evergreen.sx_max_export_pos_size = 32;
3373	rdev->config.evergreen.sx_max_export_smx_size = 96;
3374	rdev->config.evergreen.max_hw_contexts = 4;
3375	rdev->config.evergreen.sq_num_cf_insts = 1;
3376
3377	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3378	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3379	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3380	gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3381	break;
3382	}
3383
3384	/* Initialize HDP */
3385	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3386	WREG32((0x2c14 + j), 0x00000000);
3387	WREG32((0x2c18 + j), 0x00000000);
3388	WREG32((0x2c1c + j), 0x00000000);
3389	WREG32((0x2c20 + j), 0x00000000);
3390	WREG32((0x2c24 + j), 0x00000000);
3391	}
3392
3393	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3394	WREG32(SRBM_INT_CNTL, 0x1);
3395	WREG32(SRBM_INT_ACK, 0x1);
3396
3397	evergreen_fix_pci_max_read_req_size(rdev);
3398
3399	RREG32(MC_SHARED_CHMAP);
3400	if ((rdev->family == CHIP_PALM) ||
3401	(rdev->family == CHIP_SUMO) ||
3402	(rdev->family == CHIP_SUMO2))
3403	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3404	else
3405	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3406
3407	/* setup tiling info dword. gb_addr_config is not adequate since it does
3408	* not have bank info, so create a custom tiling dword.
3409	* bits 3:0 num_pipes
3410	* bits 7:4 num_banks
3411	* bits 11:8 group_size
3412	* bits 15:12 row_size
3413	*/
3414	rdev->config.evergreen.tile_config = 0;
3415	switch (rdev->config.evergreen.max_tile_pipes) {
3416	case 1:
3417	default:
3418	rdev->config.evergreen.tile_config |= (0 << 0);
3419	break;
3420	case 2:
3421	rdev->config.evergreen.tile_config |= (1 << 0);
3422	break;
3423	case 4:
3424	rdev->config.evergreen.tile_config |= (2 << 0);
3425	break;
3426	case 8:
3427	rdev->config.evergreen.tile_config |= (3 << 0);
3428	break;
3429	}
3430	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3431	if (rdev->flags & RADEON_IS_IGP)
3432	rdev->config.evergreen.tile_config |= 1 << 4;
3433	else {
3434	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3435	case 0: /* four banks */
3436	rdev->config.evergreen.tile_config |= 0 << 4;
3437	break;
3438	case 1: /* eight banks */
3439	rdev->config.evergreen.tile_config |= 1 << 4;
3440	break;
3441	case 2: /* sixteen banks */
3442	default:
3443	rdev->config.evergreen.tile_config |= 2 << 4;
3444	break;
3445	}
3446	}
3447	rdev->config.evergreen.tile_config |= 0 << 8;
3448	rdev->config.evergreen.tile_config |=
3449	((gb_addr_config & 0x30000000) >> 28) << 12;
3450
3451	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3452	u32 efuse_straps_4;
3453	u32 efuse_straps_3;
3454
3455	efuse_straps_4 = RREG32_RCU(0x204);
3456	efuse_straps_3 = RREG32_RCU(0x203);
3457	tmp = (((efuse_straps_4 & 0xf) << 4) |
3458	((efuse_straps_3 & 0xf0000000) >> 28));
3459	} else {
3460	tmp = 0;
3461	for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3462	u32 rb_disable_bitmap;
3463
3464	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3465	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3467	tmp <<= 4;
3468	tmp |= rb_disable_bitmap;
3469	}
3470	}
3471	/* enabled rb are just the one not disabled :) */
3472	disabled_rb_mask = tmp;
3473	tmp = 0;
3474	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3475	tmp |= (1 << i);
3476	/* if all the backends are disabled, fix it up here */
3477	if ((disabled_rb_mask & tmp) == tmp) {
3478	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3479	disabled_rb_mask &= ~(1 << i);
3480	}
3481
3482	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3483	u32 simd_disable_bitmap;
3484
3485	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3486	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487	simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3488	simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3489	tmp <<= 16;
3490	tmp |= simd_disable_bitmap;
3491	}
3492	rdev->config.evergreen.active_simds = hweight32(~tmp);
3493
3494	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3495	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496
3497	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3498	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3499	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3500	WREG32(DMA_TILING_CONFIG, gb_addr_config);
3501	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3502	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3503	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3504
3505	if ((rdev->config.evergreen.max_backends == 1) &&
3506	(rdev->flags & RADEON_IS_IGP)) {
3507	if ((disabled_rb_mask & 3) == 1) {
3508	/* RB0 disabled, RB1 enabled */
3509	tmp = 0x11111111;
3510	} else {
3511	/* RB1 disabled, RB0 enabled */
3512	tmp = 0x00000000;
3513	}
3514	} else {
3515	tmp = gb_addr_config & NUM_PIPES_MASK;
3516	tmp = r6xx_remap_render_backend(rdev, tiling_pipe_num: tmp, max_rb_num: rdev->config.evergreen.max_backends,
3517	EVERGREEN_MAX_BACKENDS, enabled_rb_mask: disabled_rb_mask);
3518	}
3519	rdev->config.evergreen.backend_map = tmp;
3520	WREG32(GB_BACKEND_MAP, tmp);
3521
3522	WREG32(CGTS_SYS_TCC_DISABLE, 0);
3523	WREG32(CGTS_TCC_DISABLE, 0);
3524	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3525	WREG32(CGTS_USER_TCC_DISABLE, 0);
3526
3527	/* set HW defaults for 3D engine */
3528	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3529	ROQ_IB2_START(0x2b)));
3530
3531	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3532
3533	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3534	SYNC_GRADIENT |
3535	SYNC_WALKER |
3536	SYNC_ALIGNER));
3537
3538	sx_debug_1 = RREG32(SX_DEBUG_1);
3539	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3540	WREG32(SX_DEBUG_1, sx_debug_1);
3541
3542
3543	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3544	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3545	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3546	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3547
3548	if (rdev->family <= CHIP_SUMO2)
3549	WREG32(SMX_SAR_CTL0, 0x00010000);
3550
3551	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3552	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3553	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3554
3555	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3556	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3557	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3558
3559	WREG32(VGT_NUM_INSTANCES, 1);
3560	WREG32(SPI_CONFIG_CNTL, 0);
3561	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3562	WREG32(CP_PERFMON_CNTL, 0);
3563
3564	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3565	FETCH_FIFO_HIWATER(0x4) |
3566	DONE_FIFO_HIWATER(0xe0) |
3567	ALU_UPDATE_FIFO_HIWATER(0x8)));
3568
3569	sq_config = RREG32(SQ_CONFIG);
3570	sq_config &= ~(PS_PRIO(3) |
3571	VS_PRIO(3) |
3572	GS_PRIO(3) |
3573	ES_PRIO(3));
3574	sq_config |= (VC_ENABLE |
3575	EXPORT_SRC_C |
3576	PS_PRIO(0) |
3577	VS_PRIO(1) |
3578	GS_PRIO(2) |
3579	ES_PRIO(3));
3580
3581	switch (rdev->family) {
3582	case CHIP_CEDAR:
3583	case CHIP_PALM:
3584	case CHIP_SUMO:
3585	case CHIP_SUMO2:
3586	case CHIP_CAICOS:
3587	/* no vertex cache */
3588	sq_config &= ~VC_ENABLE;
3589	break;
3590	default:
3591	break;
3592	}
3593
3594	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3595
3596	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 12 / 32);
3597	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3598	sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3599	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3600	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3602	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603
3604	switch (rdev->family) {
3605	case CHIP_CEDAR:
3606	case CHIP_PALM:
3607	case CHIP_SUMO:
3608	case CHIP_SUMO2:
3609	ps_thread_count = 96;
3610	break;
3611	default:
3612	ps_thread_count = 128;
3613	break;
3614	}
3615
3616	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3617	sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618	sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619	sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621	sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622
3623	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624	sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626	sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628	sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629
3630	WREG32(SQ_CONFIG, sq_config);
3631	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3632	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3633	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3634	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3635	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3636	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3637	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3638	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3639	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3640	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3641
3642	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3643	FORCE_EOV_MAX_REZ_CNT(255)));
3644
3645	switch (rdev->family) {
3646	case CHIP_CEDAR:
3647	case CHIP_PALM:
3648	case CHIP_SUMO:
3649	case CHIP_SUMO2:
3650	case CHIP_CAICOS:
3651	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3652	break;
3653	default:
3654	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3655	break;
3656	}
3657	vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3658	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3659
3660	WREG32(VGT_GS_VERTEX_REUSE, 16);
3661	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3662	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3663
3664	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3665	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3666
3667	WREG32(CB_PERF_CTR0_SEL_0, 0);
3668	WREG32(CB_PERF_CTR0_SEL_1, 0);
3669	WREG32(CB_PERF_CTR1_SEL_0, 0);
3670	WREG32(CB_PERF_CTR1_SEL_1, 0);
3671	WREG32(CB_PERF_CTR2_SEL_0, 0);
3672	WREG32(CB_PERF_CTR2_SEL_1, 0);
3673	WREG32(CB_PERF_CTR3_SEL_0, 0);
3674	WREG32(CB_PERF_CTR3_SEL_1, 0);
3675
3676	/* clear render buffer base addresses */
3677	WREG32(CB_COLOR0_BASE, 0);
3678	WREG32(CB_COLOR1_BASE, 0);
3679	WREG32(CB_COLOR2_BASE, 0);
3680	WREG32(CB_COLOR3_BASE, 0);
3681	WREG32(CB_COLOR4_BASE, 0);
3682	WREG32(CB_COLOR5_BASE, 0);
3683	WREG32(CB_COLOR6_BASE, 0);
3684	WREG32(CB_COLOR7_BASE, 0);
3685	WREG32(CB_COLOR8_BASE, 0);
3686	WREG32(CB_COLOR9_BASE, 0);
3687	WREG32(CB_COLOR10_BASE, 0);
3688	WREG32(CB_COLOR11_BASE, 0);
3689
3690	/* set the shader const cache sizes to 0 */
3691	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3692	WREG32(i, 0);
3693	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3694	WREG32(i, 0);
3695
3696	tmp = RREG32(HDP_MISC_CNTL);
3697	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3698	WREG32(HDP_MISC_CNTL, tmp);
3699
3700	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3701	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3702
3703	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3704
3705	udelay(usec: 50);
3706
3707	}
3708
3709	int evergreen_mc_init(struct radeon_device *rdev)
3710	{
3711	u32 tmp;
3712	int chansize, numchan;
3713
3714	/* Get VRAM informations */
3715	rdev->mc.vram_is_ddr = true;
3716	if ((rdev->family == CHIP_PALM) ||
3717	(rdev->family == CHIP_SUMO) ||
3718	(rdev->family == CHIP_SUMO2))
3719	tmp = RREG32(FUS_MC_ARB_RAMCFG);
3720	else
3721	tmp = RREG32(MC_ARB_RAMCFG);
3722	if (tmp & CHANSIZE_OVERRIDE) {
3723	chansize = 16;
3724	} else if (tmp & CHANSIZE_MASK) {
3725	chansize = 64;
3726	} else {
3727	chansize = 32;
3728	}
3729	tmp = RREG32(MC_SHARED_CHMAP);
3730	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3731	case 0:
3732	default:
3733	numchan = 1;
3734	break;
3735	case 1:
3736	numchan = 2;
3737	break;
3738	case 2:
3739	numchan = 4;
3740	break;
3741	case 3:
3742	numchan = 8;
3743	break;
3744	}
3745	rdev->mc.vram_width = numchan * chansize;
3746	/* Could aper size report 0 ? */
3747	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3748	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3749	/* Setup GPU memory space */
3750	if ((rdev->family == CHIP_PALM) ||
3751	(rdev->family == CHIP_SUMO) ||
3752	(rdev->family == CHIP_SUMO2)) {
3753	/* size in bytes on fusion */
3754	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3755	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3756	} else {
3757	/* size in MB on evergreen/cayman/tn */
3758	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3759	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760	}
3761	rdev->mc.visible_vram_size = rdev->mc.aper_size;
3762	r700_vram_gtt_location(rdev, mc: &rdev->mc);
3763	radeon_update_bandwidth_info(rdev);
3764
3765	return 0;
3766	}
3767
3768	void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3769	{
3770	dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3771	RREG32(GRBM_STATUS));
3772	dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3773	RREG32(GRBM_STATUS_SE0));
3774	dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3775	RREG32(GRBM_STATUS_SE1));
3776	dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3777	RREG32(SRBM_STATUS));
3778	dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3779	RREG32(SRBM_STATUS2));
3780	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3781	RREG32(CP_STALLED_STAT1));
3782	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3783	RREG32(CP_STALLED_STAT2));
3784	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3785	RREG32(CP_BUSY_STAT));
3786	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3787	RREG32(CP_STAT));
3788	dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3789	RREG32(DMA_STATUS_REG));
3790	if (rdev->family >= CHIP_CAYMAN) {
3791	dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3792	RREG32(DMA_STATUS_REG + 0x800));
3793	}
3794	}
3795
3796	bool evergreen_is_display_hung(struct radeon_device *rdev)
3797	{
3798	u32 crtc_hung = 0;
3799	u32 crtc_status[6];
3800	u32 i, j, tmp;
3801
3802	for (i = 0; i < rdev->num_crtc; i++) {
3803	if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3804	crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3805	crtc_hung |= (1 << i);
3806	}
3807	}
3808
3809	for (j = 0; j < 10; j++) {
3810	for (i = 0; i < rdev->num_crtc; i++) {
3811	if (crtc_hung & (1 << i)) {
3812	tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3813	if (tmp != crtc_status[i])
3814	crtc_hung &= ~(1 << i);
3815	}
3816	}
3817	if (crtc_hung == 0)
3818	return false;
3819	udelay(usec: 100);
3820	}
3821
3822	return true;
3823	}
3824
3825	u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3826	{
3827	u32 reset_mask = 0;
3828	u32 tmp;
3829
3830	/* GRBM_STATUS */
3831	tmp = RREG32(GRBM_STATUS);
3832	if (tmp & (PA_BUSY | SC_BUSY |
3833	SH_BUSY | SX_BUSY |
3834	TA_BUSY | VGT_BUSY |
3835	DB_BUSY | CB_BUSY |
3836	SPI_BUSY | VGT_BUSY_NO_DMA))
3837	reset_mask |= RADEON_RESET_GFX;
3838
3839	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3840	CP_BUSY | CP_COHERENCY_BUSY))
3841	reset_mask |= RADEON_RESET_CP;
3842
3843	if (tmp & GRBM_EE_BUSY)
3844	reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3845
3846	/* DMA_STATUS_REG */
3847	tmp = RREG32(DMA_STATUS_REG);
3848	if (!(tmp & DMA_IDLE))
3849	reset_mask |= RADEON_RESET_DMA;
3850
3851	/* SRBM_STATUS2 */
3852	tmp = RREG32(SRBM_STATUS2);
3853	if (tmp & DMA_BUSY)
3854	reset_mask |= RADEON_RESET_DMA;
3855
3856	/* SRBM_STATUS */
3857	tmp = RREG32(SRBM_STATUS);
3858	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3859	reset_mask |= RADEON_RESET_RLC;
3860
3861	if (tmp & IH_BUSY)
3862	reset_mask |= RADEON_RESET_IH;
3863
3864	if (tmp & SEM_BUSY)
3865	reset_mask |= RADEON_RESET_SEM;
3866
3867	if (tmp & GRBM_RQ_PENDING)
3868	reset_mask |= RADEON_RESET_GRBM;
3869
3870	if (tmp & VMC_BUSY)
3871	reset_mask |= RADEON_RESET_VMC;
3872
3873	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3874	MCC_BUSY | MCD_BUSY))
3875	reset_mask |= RADEON_RESET_MC;
3876
3877	if (evergreen_is_display_hung(rdev))
3878	reset_mask |= RADEON_RESET_DISPLAY;
3879
3880	/* VM_L2_STATUS */
3881	tmp = RREG32(VM_L2_STATUS);
3882	if (tmp & L2_BUSY)
3883	reset_mask |= RADEON_RESET_VMC;
3884
3885	/* Skip MC reset as it's mostly likely not hung, just busy */
3886	if (reset_mask & RADEON_RESET_MC) {
3887	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3888	reset_mask &= ~RADEON_RESET_MC;
3889	}
3890
3891	return reset_mask;
3892	}
3893
3894	static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3895	{
3896	struct evergreen_mc_save save;
3897	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3898	u32 tmp;
3899
3900	if (reset_mask == 0)
3901	return;
3902
3903	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3904
3905	evergreen_print_gpu_status_regs(rdev);
3906
3907	/* Disable CP parsing/prefetching */
3908	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3909
3910	if (reset_mask & RADEON_RESET_DMA) {
3911	/* Disable DMA */
3912	tmp = RREG32(DMA_RB_CNTL);
3913	tmp &= ~DMA_RB_ENABLE;
3914	WREG32(DMA_RB_CNTL, tmp);
3915	}
3916
3917	udelay(usec: 50);
3918
3919	evergreen_mc_stop(rdev, save: &save);
3920	if (evergreen_mc_wait_for_idle(rdev)) {
3921	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3922	}
3923
3924	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3925	grbm_soft_reset |= SOFT_RESET_DB |
3926	SOFT_RESET_CB |
3927	SOFT_RESET_PA |
3928	SOFT_RESET_SC |
3929	SOFT_RESET_SPI |
3930	SOFT_RESET_SX |
3931	SOFT_RESET_SH |
3932	SOFT_RESET_TC |
3933	SOFT_RESET_TA |
3934	SOFT_RESET_VC |
3935	SOFT_RESET_VGT;
3936	}
3937
3938	if (reset_mask & RADEON_RESET_CP) {
3939	grbm_soft_reset |= SOFT_RESET_CP |
3940	SOFT_RESET_VGT;
3941
3942	srbm_soft_reset |= SOFT_RESET_GRBM;
3943	}
3944
3945	if (reset_mask & RADEON_RESET_DMA)
3946	srbm_soft_reset |= SOFT_RESET_DMA;
3947
3948	if (reset_mask & RADEON_RESET_DISPLAY)
3949	srbm_soft_reset |= SOFT_RESET_DC;
3950
3951	if (reset_mask & RADEON_RESET_RLC)
3952	srbm_soft_reset |= SOFT_RESET_RLC;
3953
3954	if (reset_mask & RADEON_RESET_SEM)
3955	srbm_soft_reset |= SOFT_RESET_SEM;
3956
3957	if (reset_mask & RADEON_RESET_IH)
3958	srbm_soft_reset |= SOFT_RESET_IH;
3959
3960	if (reset_mask & RADEON_RESET_GRBM)
3961	srbm_soft_reset |= SOFT_RESET_GRBM;
3962
3963	if (reset_mask & RADEON_RESET_VMC)
3964	srbm_soft_reset |= SOFT_RESET_VMC;
3965
3966	if (!(rdev->flags & RADEON_IS_IGP)) {
3967	if (reset_mask & RADEON_RESET_MC)
3968	srbm_soft_reset |= SOFT_RESET_MC;
3969	}
3970
3971	if (grbm_soft_reset) {
3972	tmp = RREG32(GRBM_SOFT_RESET);
3973	tmp |= grbm_soft_reset;
3974	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3975	WREG32(GRBM_SOFT_RESET, tmp);
3976	tmp = RREG32(GRBM_SOFT_RESET);
3977
3978	udelay(usec: 50);
3979
3980	tmp &= ~grbm_soft_reset;
3981	WREG32(GRBM_SOFT_RESET, tmp);
3982	tmp = RREG32(GRBM_SOFT_RESET);
3983	}
3984
3985	if (srbm_soft_reset) {
3986	tmp = RREG32(SRBM_SOFT_RESET);
3987	tmp |= srbm_soft_reset;
3988	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3989	WREG32(SRBM_SOFT_RESET, tmp);
3990	tmp = RREG32(SRBM_SOFT_RESET);
3991
3992	udelay(usec: 50);
3993
3994	tmp &= ~srbm_soft_reset;
3995	WREG32(SRBM_SOFT_RESET, tmp);
3996	tmp = RREG32(SRBM_SOFT_RESET);
3997	}
3998
3999	/* Wait a little for things to settle down */
4000	udelay(usec: 50);
4001
4002	evergreen_mc_resume(rdev, save: &save);
4003	udelay(usec: 50);
4004
4005	evergreen_print_gpu_status_regs(rdev);
4006	}
4007
4008	void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4009	{
4010	struct evergreen_mc_save save;
4011	u32 tmp, i;
4012
4013	dev_info(rdev->dev, "GPU pci config reset\n");
4014
4015	/* disable dpm? */
4016
4017	/* Disable CP parsing/prefetching */
4018	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4019	udelay(usec: 50);
4020	/* Disable DMA */
4021	tmp = RREG32(DMA_RB_CNTL);
4022	tmp &= ~DMA_RB_ENABLE;
4023	WREG32(DMA_RB_CNTL, tmp);
4024	/* XXX other engines? */
4025
4026	/* halt the rlc */
4027	r600_rlc_stop(rdev);
4028
4029	udelay(usec: 50);
4030
4031	/* set mclk/sclk to bypass */
4032	rv770_set_clk_bypass_mode(rdev);
4033	/* disable BM */
4034	pci_clear_master(dev: rdev->pdev);
4035	/* disable mem access */
4036	evergreen_mc_stop(rdev, save: &save);
4037	if (evergreen_mc_wait_for_idle(rdev)) {
4038	dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4039	}
4040	/* reset */
4041	radeon_pci_config_reset(rdev);
4042	/* wait for asic to come out of reset */
4043	for (i = 0; i < rdev->usec_timeout; i++) {
4044	if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4045	break;
4046	udelay(usec: 1);
4047	}
4048	}
4049
4050	int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4051	{
4052	u32 reset_mask;
4053
4054	if (hard) {
4055	evergreen_gpu_pci_config_reset(rdev);
4056	return 0;
4057	}
4058
4059	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4060
4061	if (reset_mask)
4062	r600_set_bios_scratch_engine_hung(rdev, hung: true);
4063
4064	/* try soft reset */
4065	evergreen_gpu_soft_reset(rdev, reset_mask);
4066
4067	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4068
4069	/* try pci config reset */
4070	if (reset_mask && radeon_hard_reset)
4071	evergreen_gpu_pci_config_reset(rdev);
4072
4073	reset_mask = evergreen_gpu_check_soft_reset(rdev);
4074
4075	if (!reset_mask)
4076	r600_set_bios_scratch_engine_hung(rdev, hung: false);
4077
4078	return 0;
4079	}
4080
4081	/**
4082	* evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4083	*
4084	* @rdev: radeon_device pointer
4085	* @ring: radeon_ring structure holding ring information
4086	*
4087	* Check if the GFX engine is locked up.
4088	* Returns true if the engine appears to be locked up, false if not.
4089	*/
4090	bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4091	{
4092	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4093
4094	if (!(reset_mask & (RADEON_RESET_GFX |
4095	RADEON_RESET_COMPUTE |
4096	RADEON_RESET_CP))) {
4097	radeon_ring_lockup_update(rdev, ring);
4098	return false;
4099	}
4100	return radeon_ring_test_lockup(rdev, ring);
4101	}
4102
4103	/*
4104	* RLC
4105	*/
4106	#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4107	#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4108
4109	void sumo_rlc_fini(struct radeon_device *rdev)
4110	{
4111	int r;
4112
4113	/* save restore block */
4114	if (rdev->rlc.save_restore_obj) {
4115	r = radeon_bo_reserve(bo: rdev->rlc.save_restore_obj, no_intr: false);
4116	if (unlikely(r != 0))
4117	dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4118	radeon_bo_unpin(bo: rdev->rlc.save_restore_obj);
4119	radeon_bo_unreserve(bo: rdev->rlc.save_restore_obj);
4120
4121	radeon_bo_unref(bo: &rdev->rlc.save_restore_obj);
4122	rdev->rlc.save_restore_obj = NULL;
4123	}
4124
4125	/* clear state block */
4126	if (rdev->rlc.clear_state_obj) {
4127	r = radeon_bo_reserve(bo: rdev->rlc.clear_state_obj, no_intr: false);
4128	if (unlikely(r != 0))
4129	dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4130	radeon_bo_unpin(bo: rdev->rlc.clear_state_obj);
4131	radeon_bo_unreserve(bo: rdev->rlc.clear_state_obj);
4132
4133	radeon_bo_unref(bo: &rdev->rlc.clear_state_obj);
4134	rdev->rlc.clear_state_obj = NULL;
4135	}
4136
4137	/* clear state block */
4138	if (rdev->rlc.cp_table_obj) {
4139	r = radeon_bo_reserve(bo: rdev->rlc.cp_table_obj, no_intr: false);
4140	if (unlikely(r != 0))
4141	dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4142	radeon_bo_unpin(bo: rdev->rlc.cp_table_obj);
4143	radeon_bo_unreserve(bo: rdev->rlc.cp_table_obj);
4144
4145	radeon_bo_unref(bo: &rdev->rlc.cp_table_obj);
4146	rdev->rlc.cp_table_obj = NULL;
4147	}
4148	}
4149
4150	#define CP_ME_TABLE_SIZE 96
4151
4152	int sumo_rlc_init(struct radeon_device *rdev)
4153	{
4154	const u32 *src_ptr;
4155	volatile u32 *dst_ptr;
4156	u32 dws, data, i, j, k, reg_num;
4157	u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4158	u64 reg_list_mc_addr;
4159	const struct cs_section_def *cs_data;
4160	int r;
4161
4162	src_ptr = rdev->rlc.reg_list;
4163	dws = rdev->rlc.reg_list_size;
4164	if (rdev->family >= CHIP_BONAIRE) {
4165	dws += (5 * 16) + 48 + 48 + 64;
4166	}
4167	cs_data = rdev->rlc.cs_data;
4168
4169	if (src_ptr) {
4170	/* save restore block */
4171	if (rdev->rlc.save_restore_obj == NULL) {
4172	r = radeon_bo_create(rdev, size: dws * 4, PAGE_SIZE, kernel: true,
4173	RADEON_GEM_DOMAIN_VRAM, flags: 0, NULL,
4174	NULL, bo_ptr: &rdev->rlc.save_restore_obj);
4175	if (r) {
4176	dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4177	return r;
4178	}
4179	}
4180
4181	r = radeon_bo_reserve(bo: rdev->rlc.save_restore_obj, no_intr: false);
4182	if (unlikely(r != 0)) {
4183	sumo_rlc_fini(rdev);
4184	return r;
4185	}
4186	r = radeon_bo_pin(bo: rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4187	gpu_addr: &rdev->rlc.save_restore_gpu_addr);
4188	if (r) {
4189	radeon_bo_unreserve(bo: rdev->rlc.save_restore_obj);
4190	dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4191	sumo_rlc_fini(rdev);
4192	return r;
4193	}
4194
4195	r = radeon_bo_kmap(bo: rdev->rlc.save_restore_obj, ptr: (void **)&rdev->rlc.sr_ptr);
4196	if (r) {
4197	dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4198	sumo_rlc_fini(rdev);
4199	return r;
4200	}
4201	/* write the sr buffer */
4202	dst_ptr = rdev->rlc.sr_ptr;
4203	if (rdev->family >= CHIP_TAHITI) {
4204	/* SI */
4205	for (i = 0; i < rdev->rlc.reg_list_size; i++)
4206	dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4207	} else {
4208	/* ON/LN/TN */
4209	/* format:
4210	* dw0: (reg2 << 16) | reg1
4211	* dw1: reg1 save space
4212	* dw2: reg2 save space
4213	*/
4214	for (i = 0; i < dws; i++) {
4215	data = src_ptr[i] >> 2;
4216	i++;
4217	if (i < dws)
4218	data |= (src_ptr[i] >> 2) << 16;
4219	j = (((i - 1) * 3) / 2);
4220	dst_ptr[j] = cpu_to_le32(data);
4221	}
4222	j = ((i * 3) / 2);
4223	dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4224	}
4225	radeon_bo_kunmap(bo: rdev->rlc.save_restore_obj);
4226	radeon_bo_unreserve(bo: rdev->rlc.save_restore_obj);
4227	}
4228
4229	if (cs_data) {
4230	/* clear state block */
4231	if (rdev->family >= CHIP_BONAIRE) {
4232	rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4233	} else if (rdev->family >= CHIP_TAHITI) {
4234	rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4235	dws = rdev->rlc.clear_state_size + (256 / 4);
4236	} else {
4237	reg_list_num = 0;
4238	dws = 0;
4239	for (i = 0; cs_data[i].section != NULL; i++) {
4240	for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4241	reg_list_num++;
4242	dws += cs_data[i].section[j].reg_count;
4243	}
4244	}
4245	reg_list_blk_index = (3 * reg_list_num + 2);
4246	dws += reg_list_blk_index;
4247	rdev->rlc.clear_state_size = dws;
4248	}
4249
4250	if (rdev->rlc.clear_state_obj == NULL) {
4251	r = radeon_bo_create(rdev, size: dws * 4, PAGE_SIZE, kernel: true,
4252	RADEON_GEM_DOMAIN_VRAM, flags: 0, NULL,
4253	NULL, bo_ptr: &rdev->rlc.clear_state_obj);
4254	if (r) {
4255	dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4256	sumo_rlc_fini(rdev);
4257	return r;
4258	}
4259	}
4260	r = radeon_bo_reserve(bo: rdev->rlc.clear_state_obj, no_intr: false);
4261	if (unlikely(r != 0)) {
4262	sumo_rlc_fini(rdev);
4263	return r;
4264	}
4265	r = radeon_bo_pin(bo: rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4266	gpu_addr: &rdev->rlc.clear_state_gpu_addr);
4267	if (r) {
4268	radeon_bo_unreserve(bo: rdev->rlc.clear_state_obj);
4269	dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4270	sumo_rlc_fini(rdev);
4271	return r;
4272	}
4273
4274	r = radeon_bo_kmap(bo: rdev->rlc.clear_state_obj, ptr: (void **)&rdev->rlc.cs_ptr);
4275	if (r) {
4276	dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4277	sumo_rlc_fini(rdev);
4278	return r;
4279	}
4280	/* set up the cs buffer */
4281	dst_ptr = rdev->rlc.cs_ptr;
4282	if (rdev->family >= CHIP_BONAIRE) {
4283	cik_get_csb_buffer(rdev, buffer: dst_ptr);
4284	} else if (rdev->family >= CHIP_TAHITI) {
4285	reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4286	dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4287	dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4288	dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4289	si_get_csb_buffer(rdev, buffer: &dst_ptr[(256/4)]);
4290	} else {
4291	reg_list_hdr_blk_index = 0;
4292	reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4293	data = upper_32_bits(reg_list_mc_addr);
4294	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4295	reg_list_hdr_blk_index++;
4296	for (i = 0; cs_data[i].section != NULL; i++) {
4297	for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4298	reg_num = cs_data[i].section[j].reg_count;
4299	data = reg_list_mc_addr & 0xffffffff;
4300	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301	reg_list_hdr_blk_index++;
4302
4303	data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4304	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305	reg_list_hdr_blk_index++;
4306
4307	data = 0x08000000 | (reg_num * 4);
4308	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309	reg_list_hdr_blk_index++;
4310
4311	for (k = 0; k < reg_num; k++) {
4312	data = cs_data[i].section[j].extent[k];
4313	dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4314	}
4315	reg_list_mc_addr += reg_num * 4;
4316	reg_list_blk_index += reg_num;
4317	}
4318	}
4319	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4320	}
4321	radeon_bo_kunmap(bo: rdev->rlc.clear_state_obj);
4322	radeon_bo_unreserve(bo: rdev->rlc.clear_state_obj);
4323	}
4324
4325	if (rdev->rlc.cp_table_size) {
4326	if (rdev->rlc.cp_table_obj == NULL) {
4327	r = radeon_bo_create(rdev, size: rdev->rlc.cp_table_size,
4328	PAGE_SIZE, kernel: true,
4329	RADEON_GEM_DOMAIN_VRAM, flags: 0, NULL,
4330	NULL, bo_ptr: &rdev->rlc.cp_table_obj);
4331	if (r) {
4332	dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4333	sumo_rlc_fini(rdev);
4334	return r;
4335	}
4336	}
4337
4338	r = radeon_bo_reserve(bo: rdev->rlc.cp_table_obj, no_intr: false);
4339	if (unlikely(r != 0)) {
4340	dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4341	sumo_rlc_fini(rdev);
4342	return r;
4343	}
4344	r = radeon_bo_pin(bo: rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4345	gpu_addr: &rdev->rlc.cp_table_gpu_addr);
4346	if (r) {
4347	radeon_bo_unreserve(bo: rdev->rlc.cp_table_obj);
4348	dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4349	sumo_rlc_fini(rdev);
4350	return r;
4351	}
4352	r = radeon_bo_kmap(bo: rdev->rlc.cp_table_obj, ptr: (void **)&rdev->rlc.cp_table_ptr);
4353	if (r) {
4354	dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4355	sumo_rlc_fini(rdev);
4356	return r;
4357	}
4358
4359	cik_init_cp_pg_table(rdev);
4360
4361	radeon_bo_kunmap(bo: rdev->rlc.cp_table_obj);
4362	radeon_bo_unreserve(bo: rdev->rlc.cp_table_obj);
4363
4364	}
4365
4366	return 0;
4367	}
4368
4369	static void evergreen_rlc_start(struct radeon_device *rdev)
4370	{
4371	u32 mask = RLC_ENABLE;
4372
4373	if (rdev->flags & RADEON_IS_IGP) {
4374	mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4375	}
4376
4377	WREG32(RLC_CNTL, mask);
4378	}
4379
4380	int evergreen_rlc_resume(struct radeon_device *rdev)
4381	{
4382	u32 i;
4383	const __be32 *fw_data;
4384
4385	if (!rdev->rlc_fw)
4386	return -EINVAL;
4387
4388	r600_rlc_stop(rdev);
4389
4390	WREG32(RLC_HB_CNTL, 0);
4391
4392	if (rdev->flags & RADEON_IS_IGP) {
4393	if (rdev->family == CHIP_ARUBA) {
4394	u32 always_on_bitmap =
4395	3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4396	/* find out the number of active simds */
4397	u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4398	tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4399	tmp = hweight32(~tmp);
4400	if (tmp == rdev->config.cayman.max_simds_per_se) {
4401	WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4402	WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4403	WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4404	WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4405	WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4406	}
4407	} else {
4408	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4409	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4410	}
4411	WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4412	WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4413	} else {
4414	WREG32(RLC_HB_BASE, 0);
4415	WREG32(RLC_HB_RPTR, 0);
4416	WREG32(RLC_HB_WPTR, 0);
4417	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4418	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4419	}
4420	WREG32(RLC_MC_CNTL, 0);
4421	WREG32(RLC_UCODE_CNTL, 0);
4422
4423	fw_data = (const __be32 *)rdev->rlc_fw->data;
4424	if (rdev->family >= CHIP_ARUBA) {
4425	for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4426	WREG32(RLC_UCODE_ADDR, i);
4427	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4428	}
4429	} else if (rdev->family >= CHIP_CAYMAN) {
4430	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4431	WREG32(RLC_UCODE_ADDR, i);
4432	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4433	}
4434	} else {
4435	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4436	WREG32(RLC_UCODE_ADDR, i);
4437	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4438	}
4439	}
4440	WREG32(RLC_UCODE_ADDR, 0);
4441
4442	evergreen_rlc_start(rdev);
4443
4444	return 0;
4445	}
4446
4447	/* Interrupts */
4448
4449	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4450	{
4451	if (crtc >= rdev->num_crtc)
4452	return 0;
4453	else
4454	return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4455	}
4456
4457	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4458	{
4459	int i;
4460	u32 tmp;
4461
4462	if (rdev->family >= CHIP_CAYMAN) {
4463	cayman_cp_int_cntl_setup(rdev, ring: 0,
4464	CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4465	cayman_cp_int_cntl_setup(rdev, ring: 1, cp_int_cntl: 0);
4466	cayman_cp_int_cntl_setup(rdev, ring: 2, cp_int_cntl: 0);
4467	tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4468	WREG32(CAYMAN_DMA1_CNTL, tmp);
4469	} else
4470	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4471	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4472	WREG32(DMA_CNTL, tmp);
4473	WREG32(GRBM_INT_CNTL, 0);
4474	WREG32(SRBM_INT_CNTL, 0);
4475	for (i = 0; i < rdev->num_crtc; i++)
4476	WREG32(INT_MASK + crtc_offsets[i], 0);
4477	for (i = 0; i < rdev->num_crtc; i++)
4478	WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4479
4480	/* only one DAC on DCE5 */
4481	if (!ASIC_IS_DCE5(rdev))
4482	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4483	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4484
4485	for (i = 0; i < 6; i++)
4486	WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4487	}
4488
4489	/* Note that the order we write back regs here is important */
4490	int evergreen_irq_set(struct radeon_device *rdev)
4491	{
4492	int i;
4493	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4494	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4495	u32 grbm_int_cntl = 0;
4496	u32 dma_cntl, dma_cntl1 = 0;
4497	u32 thermal_int = 0;
4498
4499	if (!rdev->irq.installed) {
4500	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4501	return -EINVAL;
4502	}
4503	/* don't enable anything if the ih is disabled */
4504	if (!rdev->ih.enabled) {
4505	r600_disable_interrupts(rdev);
4506	/* force the active interrupt state to all disabled */
4507	evergreen_disable_interrupt_state(rdev);
4508	return 0;
4509	}
4510
4511	if (rdev->family == CHIP_ARUBA)
4512	thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4513	~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4514	else
4515	thermal_int = RREG32(CG_THERMAL_INT) &
4516	~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4517
4518	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4519
4520	if (rdev->family >= CHIP_CAYMAN) {
4521	/* enable CP interrupts on all rings */
4522	if (atomic_read(v: &rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4523	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4524	cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4525	}
4526	if (atomic_read(v: &rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4527	DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4528	cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4529	}
4530	if (atomic_read(v: &rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4531	DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4532	cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4533	}
4534	} else {
4535	if (atomic_read(v: &rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4536	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4537	cp_int_cntl |= RB_INT_ENABLE;
4538	cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4539	}
4540	}
4541
4542	if (atomic_read(v: &rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4543	DRM_DEBUG("r600_irq_set: sw int dma\n");
4544	dma_cntl |= TRAP_ENABLE;
4545	}
4546
4547	if (rdev->family >= CHIP_CAYMAN) {
4548	dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4549	if (atomic_read(v: &rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4550	DRM_DEBUG("r600_irq_set: sw int dma1\n");
4551	dma_cntl1 |= TRAP_ENABLE;
4552	}
4553	}
4554
4555	if (rdev->irq.dpm_thermal) {
4556	DRM_DEBUG("dpm thermal\n");
4557	thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4558	}
4559
4560	if (rdev->family >= CHIP_CAYMAN) {
4561	cayman_cp_int_cntl_setup(rdev, ring: 0, cp_int_cntl);
4562	cayman_cp_int_cntl_setup(rdev, ring: 1, cp_int_cntl: cp_int_cntl1);
4563	cayman_cp_int_cntl_setup(rdev, ring: 2, cp_int_cntl: cp_int_cntl2);
4564	} else
4565	WREG32(CP_INT_CNTL, cp_int_cntl);
4566
4567	WREG32(DMA_CNTL, dma_cntl);
4568
4569	if (rdev->family >= CHIP_CAYMAN)
4570	WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4571
4572	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4573
4574	for (i = 0; i < rdev->num_crtc; i++) {
4575	radeon_irq_kms_set_irq_n_enabled(
4576	rdev, INT_MASK + crtc_offsets[i],
4577	VBLANK_INT_MASK,
4578	enable: rdev->irq.crtc_vblank_int[i] ||
4579	atomic_read(v: &rdev->irq.pflip[i]), name: "vblank", n: i);
4580	}
4581
4582	for (i = 0; i < rdev->num_crtc; i++)
4583	WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4584
4585	for (i = 0; i < 6; i++) {
4586	radeon_irq_kms_set_irq_n_enabled(
4587	rdev, DC_HPDx_INT_CONTROL(i),
4588	DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4589	enable: rdev->irq.hpd[i], name: "HPD", n: i);
4590	}
4591
4592	if (rdev->family == CHIP_ARUBA)
4593	WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4594	else
4595	WREG32(CG_THERMAL_INT, thermal_int);
4596
4597	for (i = 0; i < 6; i++) {
4598	radeon_irq_kms_set_irq_n_enabled(
4599	rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4600	AFMT_AZ_FORMAT_WTRIG_MASK,
4601	enable: rdev->irq.afmt[i], name: "HDMI", n: i);
4602	}
4603
4604	/* posting read */
4605	RREG32(SRBM_STATUS);
4606
4607	return 0;
4608	}
4609
4610	/* Note that the order we write back regs here is important */
4611	static void evergreen_irq_ack(struct radeon_device *rdev)
4612	{
4613	int i, j;
4614	u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4615	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4616	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4617
4618	for (i = 0; i < 6; i++) {
4619	disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4620	afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4621	if (i < rdev->num_crtc)
4622	grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4623	}
4624
4625	/* We write back each interrupt register in pairs of two */
4626	for (i = 0; i < rdev->num_crtc; i += 2) {
4627	for (j = i; j < (i + 2); j++) {
4628	if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4629	WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4630	GRPH_PFLIP_INT_CLEAR);
4631	}
4632
4633	for (j = i; j < (i + 2); j++) {
4634	if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4635	WREG32(VBLANK_STATUS + crtc_offsets[j],
4636	VBLANK_ACK);
4637	if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4638	WREG32(VLINE_STATUS + crtc_offsets[j],
4639	VLINE_ACK);
4640	}
4641	}
4642
4643	for (i = 0; i < 6; i++) {
4644	if (disp_int[i] & DC_HPD1_INTERRUPT)
4645	WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4646	}
4647
4648	for (i = 0; i < 6; i++) {
4649	if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4650	WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4651	}
4652
4653	for (i = 0; i < 6; i++) {
4654	if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4655	WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4656	AFMT_AZ_FORMAT_WTRIG_ACK);
4657	}
4658	}
4659
4660	static void evergreen_irq_disable(struct radeon_device *rdev)
4661	{
4662	r600_disable_interrupts(rdev);
4663	/* Wait and acknowledge irq */
4664	mdelay(1);
4665	evergreen_irq_ack(rdev);
4666	evergreen_disable_interrupt_state(rdev);
4667	}
4668
4669	void evergreen_irq_suspend(struct radeon_device *rdev)
4670	{
4671	evergreen_irq_disable(rdev);
4672	r600_rlc_stop(rdev);
4673	}
4674
4675	static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4676	{
4677	u32 wptr, tmp;
4678
4679	if (rdev->wb.enabled)
4680	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4681	else
4682	wptr = RREG32(IH_RB_WPTR);
4683
4684	if (wptr & RB_OVERFLOW) {
4685	wptr &= ~RB_OVERFLOW;
4686	/* When a ring buffer overflow happen start parsing interrupt
4687	* from the last not overwritten vector (wptr + 16). Hopefully
4688	* this should allow us to catchup.
4689	*/
4690	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4691	wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4692	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4693	tmp = RREG32(IH_RB_CNTL);
4694	tmp |= IH_WPTR_OVERFLOW_CLEAR;
4695	WREG32(IH_RB_CNTL, tmp);
4696	}
4697	return (wptr & rdev->ih.ptr_mask);
4698	}
4699
4700	int evergreen_irq_process(struct radeon_device *rdev)
4701	{
4702	u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4703	u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4704	u32 crtc_idx, hpd_idx, afmt_idx;
4705	u32 mask;
4706	u32 wptr;
4707	u32 rptr;
4708	u32 src_id, src_data;
4709	u32 ring_index;
4710	bool queue_hotplug = false;
4711	bool queue_hdmi = false;
4712	bool queue_dp = false;
4713	bool queue_thermal = false;
4714	u32 status, addr;
4715	const char *event_name;
4716
4717	if (!rdev->ih.enabled || rdev->shutdown)
4718	return IRQ_NONE;
4719
4720	wptr = evergreen_get_ih_wptr(rdev);
4721
4722	restart_ih:
4723	/* is somebody else already processing irqs? */
4724	if (atomic_xchg(v: &rdev->ih.lock, new: 1))
4725	return IRQ_NONE;
4726
4727	rptr = rdev->ih.rptr;
4728	DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4729
4730	/* Order reading of wptr vs. reading of IH ring data */
4731	rmb();
4732
4733	/* display interrupts */
4734	evergreen_irq_ack(rdev);
4735
4736	while (rptr != wptr) {
4737	/* wptr/rptr are in bytes! */
4738	ring_index = rptr / 4;
4739	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4740	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4741
4742	switch (src_id) {
4743	case 1: /* D1 vblank/vline */
4744	case 2: /* D2 vblank/vline */
4745	case 3: /* D3 vblank/vline */
4746	case 4: /* D4 vblank/vline */
4747	case 5: /* D5 vblank/vline */
4748	case 6: /* D6 vblank/vline */
4749	crtc_idx = src_id - 1;
4750
4751	if (src_data == 0) { /* vblank */
4752	mask = LB_D1_VBLANK_INTERRUPT;
4753	event_name = "vblank";
4754
4755	if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4756	drm_handle_vblank(dev: rdev_to_drm(rdev), pipe: crtc_idx);
4757	rdev->pm.vblank_sync = true;
4758	wake_up(&rdev->irq.vblank_queue);
4759	}
4760	if (atomic_read(v: &rdev->irq.pflip[crtc_idx])) {
4761	radeon_crtc_handle_vblank(rdev,
4762	crtc_id: crtc_idx);
4763	}
4764
4765	} else if (src_data == 1) { /* vline */
4766	mask = LB_D1_VLINE_INTERRUPT;
4767	event_name = "vline";
4768	} else {
4769	DRM_DEBUG("Unhandled interrupt: %d %d\n",
4770	src_id, src_data);
4771	break;
4772	}
4773
4774	if (!(disp_int[crtc_idx] & mask)) {
4775	DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4776	crtc_idx + 1, event_name);
4777	}
4778
4779	disp_int[crtc_idx] &= ~mask;
4780	DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4781
4782	break;
4783	case 8: /* D1 page flip */
4784	case 10: /* D2 page flip */
4785	case 12: /* D3 page flip */
4786	case 14: /* D4 page flip */
4787	case 16: /* D5 page flip */
4788	case 18: /* D6 page flip */
4789	DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4790	if (radeon_use_pflipirq > 0)
4791	radeon_crtc_handle_flip(rdev, crtc_id: (src_id - 8) >> 1);
4792	break;
4793	case 42: /* HPD hotplug */
4794	if (src_data <= 5) {
4795	hpd_idx = src_data;
4796	mask = DC_HPD1_INTERRUPT;
4797	queue_hotplug = true;
4798	event_name = "HPD";
4799
4800	} else if (src_data <= 11) {
4801	hpd_idx = src_data - 6;
4802	mask = DC_HPD1_RX_INTERRUPT;
4803	queue_dp = true;
4804	event_name = "HPD_RX";
4805
4806	} else {
4807	DRM_DEBUG("Unhandled interrupt: %d %d\n",
4808	src_id, src_data);
4809	break;
4810	}
4811
4812	if (!(disp_int[hpd_idx] & mask))
4813	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4814
4815	disp_int[hpd_idx] &= ~mask;
4816	DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4817
4818	break;
4819	case 44: /* hdmi */
4820	afmt_idx = src_data;
4821	if (afmt_idx > 5) {
4822	DRM_ERROR("Unhandled interrupt: %d %d\n",
4823	src_id, src_data);
4824	break;
4825	}
4826
4827	if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4828	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4829
4830	afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831	queue_hdmi = true;
4832	DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833	break;
4834	case 96:
4835	DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836	WREG32(SRBM_INT_ACK, 0x1);
4837	break;
4838	case 124: /* UVD */
4839	DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840	radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841	break;
4842	case 146:
4843	case 147:
4844	addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845	status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846	/* reset addr and status */
4847	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848	if (addr == 0x0 && status == 0x0)
4849	break;
4850	dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4852	addr);
4853	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854	status);
4855	cayman_vm_decode_fault(rdev, status, addr);
4856	break;
4857	case 176: /* CP_INT in ring buffer */
4858	case 177: /* CP_INT in IB1 */
4859	case 178: /* CP_INT in IB2 */
4860	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862	break;
4863	case 181: /* CP EOP event */
4864	DRM_DEBUG("IH: CP EOP\n");
4865	if (rdev->family >= CHIP_CAYMAN) {
4866	switch (src_data) {
4867	case 0:
4868	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869	break;
4870	case 1:
4871	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872	break;
4873	case 2:
4874	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875	break;
4876	}
4877	} else
4878	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879	break;
4880	case 224: /* DMA trap event */
4881	DRM_DEBUG("IH: DMA trap\n");
4882	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883	break;
4884	case 230: /* thermal low to high */
4885	DRM_DEBUG("IH: thermal low to high\n");
4886	rdev->pm.dpm.thermal.high_to_low = false;
4887	queue_thermal = true;
4888	break;
4889	case 231: /* thermal high to low */
4890	DRM_DEBUG("IH: thermal high to low\n");
4891	rdev->pm.dpm.thermal.high_to_low = true;
4892	queue_thermal = true;
4893	break;
4894	case 233: /* GUI IDLE */
4895	DRM_DEBUG("IH: GUI idle\n");
4896	break;
4897	case 244: /* DMA trap event */
4898	if (rdev->family >= CHIP_CAYMAN) {
4899	DRM_DEBUG("IH: DMA1 trap\n");
4900	radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901	}
4902	break;
4903	default:
4904	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905	break;
4906	}
4907
4908	/* wptr/rptr are in bytes! */
4909	rptr += 16;
4910	rptr &= rdev->ih.ptr_mask;
4911	WREG32(IH_RB_RPTR, rptr);
4912	}
4913	if (queue_dp)
4914	schedule_work(work: &rdev->dp_work);
4915	if (queue_hotplug)
4916	schedule_delayed_work(dwork: &rdev->hotplug_work, delay: 0);
4917	if (queue_hdmi)
4918	schedule_work(work: &rdev->audio_work);
4919	if (queue_thermal && rdev->pm.dpm_enabled)
4920	schedule_work(work: &rdev->pm.dpm.thermal.work);
4921	rdev->ih.rptr = rptr;
4922	atomic_set(v: &rdev->ih.lock, i: 0);
4923
4924	/* make sure wptr hasn't changed while processing */
4925	wptr = evergreen_get_ih_wptr(rdev);
4926	if (wptr != rptr)
4927	goto restart_ih;
4928
4929	return IRQ_HANDLED;
4930	}
4931
4932	static void evergreen_uvd_init(struct radeon_device *rdev)
4933	{
4934	int r;
4935
4936	if (!rdev->has_uvd)
4937	return;
4938
4939	r = radeon_uvd_init(rdev);
4940	if (r) {
4941	dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942	/*
4943	* At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944	* to early fails uvd_v2_2_resume() and thus nothing happens
4945	* there. So it is pointless to try to go through that code
4946	* hence why we disable uvd here.
4947	*/
4948	rdev->has_uvd = false;
4949	return;
4950	}
4951	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952	r600_ring_init(rdev, cp: &rdev->ring[R600_RING_TYPE_UVD_INDEX], ring_size: 4096);
4953	}
4954
4955	static void evergreen_uvd_start(struct radeon_device *rdev)
4956	{
4957	int r;
4958
4959	if (!rdev->has_uvd)
4960	return;
4961
4962	r = uvd_v2_2_resume(rdev);
4963	if (r) {
4964	dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965	goto error;
4966	}
4967	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968	if (r) {
4969	dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970	goto error;
4971	}
4972	return;
4973
4974	error:
4975	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976	}
4977
4978	static void evergreen_uvd_resume(struct radeon_device *rdev)
4979	{
4980	struct radeon_ring *ring;
4981	int r;
4982
4983	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984	return;
4985
4986	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987	r = radeon_ring_init(rdev, cp: ring, ring_size: ring->ring_size, rptr_offs: 0, PACKET0(UVD_NO_OP, 0));
4988	if (r) {
4989	dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990	return;
4991	}
4992	r = uvd_v1_0_init(rdev);
4993	if (r) {
4994	dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995	return;
4996	}
4997	}
4998
4999	static int evergreen_startup(struct radeon_device *rdev)
5000	{
5001	struct radeon_ring *ring;
5002	int r;
5003
5004	/* enable pcie gen2 link */
5005	evergreen_pcie_gen2_enable(rdev);
5006	/* enable aspm */
5007	evergreen_program_aspm(rdev);
5008
5009	/* scratch needs to be initialized before MC */
5010	r = r600_vram_scratch_init(rdev);
5011	if (r)
5012	return r;
5013
5014	evergreen_mc_program(rdev);
5015
5016	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017	r = ni_mc_load_microcode(rdev);
5018	if (r) {
5019	DRM_ERROR("Failed to load MC firmware!\n");
5020	return r;
5021	}
5022	}
5023
5024	if (rdev->flags & RADEON_IS_AGP) {
5025	evergreen_agp_enable(rdev);
5026	} else {
5027	r = evergreen_pcie_gart_enable(rdev);
5028	if (r)
5029	return r;
5030	}
5031	evergreen_gpu_init(rdev);
5032
5033	/* allocate rlc buffers */
5034	if (rdev->flags & RADEON_IS_IGP) {
5035	rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036	rdev->rlc.reg_list_size =
5037	(u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038	rdev->rlc.cs_data = evergreen_cs_data;
5039	r = sumo_rlc_init(rdev);
5040	if (r) {
5041	DRM_ERROR("Failed to init rlc BOs!\n");
5042	return r;
5043	}
5044	}
5045
5046	/* allocate wb buffer */
5047	r = radeon_wb_init(rdev);
5048	if (r)
5049	return r;
5050
5051	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052	if (r) {
5053	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054	return r;
5055	}
5056
5057	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058	if (r) {
5059	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060	return r;
5061	}
5062
5063	evergreen_uvd_start(rdev);
5064
5065	/* Enable IRQ */
5066	if (!rdev->irq.installed) {
5067	r = radeon_irq_kms_init(rdev);
5068	if (r)
5069	return r;
5070	}
5071
5072	r = r600_irq_init(rdev);
5073	if (r) {
5074	DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075	radeon_irq_kms_fini(rdev);
5076	return r;
5077	}
5078	evergreen_irq_set(rdev);
5079
5080	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081	r = radeon_ring_init(rdev, cp: ring, ring_size: ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082	RADEON_CP_PACKET2);
5083	if (r)
5084	return r;
5085
5086	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087	r = radeon_ring_init(rdev, cp: ring, ring_size: ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088	DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089	if (r)
5090	return r;
5091
5092	r = evergreen_cp_load_microcode(rdev);
5093	if (r)
5094	return r;
5095	r = evergreen_cp_resume(rdev);
5096	if (r)
5097	return r;
5098	r = r600_dma_resume(rdev);
5099	if (r)
5100	return r;
5101
5102	evergreen_uvd_resume(rdev);
5103
5104	r = radeon_ib_pool_init(rdev);
5105	if (r) {
5106	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107	return r;
5108	}
5109
5110	r = radeon_audio_init(rdev);
5111	if (r) {
5112	DRM_ERROR("radeon: audio init failed\n");
5113	return r;
5114	}
5115
5116	return 0;
5117	}
5118
5119	int evergreen_resume(struct radeon_device *rdev)
5120	{
5121	int r;
5122
5123	/* reset the asic, the gfx blocks are often in a bad state
5124	* after the driver is unloaded or after a resume
5125	*/
5126	if (radeon_asic_reset(rdev))
5127	dev_warn(rdev->dev, "GPU reset failed !\n");
5128	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129	* posting will perform necessary task to bring back GPU into good
5130	* shape.
5131	*/
5132	/* post card */
5133	atom_asic_init(rdev->mode_info.atom_context);
5134
5135	/* init golden registers */
5136	evergreen_init_golden_registers(rdev);
5137
5138	if (rdev->pm.pm_method == PM_METHOD_DPM)
5139	radeon_pm_resume(rdev);
5140
5141	rdev->accel_working = true;
5142	r = evergreen_startup(rdev);
5143	if (r) {
5144	DRM_ERROR("evergreen startup failed on resume\n");
5145	rdev->accel_working = false;
5146	return r;
5147	}
5148
5149	return r;
5150
5151	}
5152
5153	int evergreen_suspend(struct radeon_device *rdev)
5154	{
5155	radeon_pm_suspend(rdev);
5156	radeon_audio_fini(rdev);
5157	if (rdev->has_uvd) {
5158	radeon_uvd_suspend(rdev);
5159	uvd_v1_0_fini(rdev);
5160	}
5161	r700_cp_stop(rdev);
5162	r600_dma_stop(rdev);
5163	evergreen_irq_suspend(rdev);
5164	radeon_wb_disable(rdev);
5165	evergreen_pcie_gart_disable(rdev);
5166
5167	return 0;
5168	}
5169
5170	/* Plan is to move initialization in that function and use
5171	* helper function so that radeon_device_init pretty much
5172	* do nothing more than calling asic specific function. This
5173	* should also allow to remove a bunch of callback function
5174	* like vram_info.
5175	*/
5176	int evergreen_init(struct radeon_device *rdev)
5177	{
5178	int r;
5179
5180	/* Read BIOS */
5181	if (!radeon_get_bios(rdev)) {
5182	if (ASIC_IS_AVIVO(rdev))
5183	return -EINVAL;
5184	}
5185	/* Must be an ATOMBIOS */
5186	if (!rdev->is_atom_bios) {
5187	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188	return -EINVAL;
5189	}
5190	r = radeon_atombios_init(rdev);
5191	if (r)
5192	return r;
5193	/* reset the asic, the gfx blocks are often in a bad state
5194	* after the driver is unloaded or after a resume
5195	*/
5196	if (radeon_asic_reset(rdev))
5197	dev_warn(rdev->dev, "GPU reset failed !\n");
5198	/* Post card if necessary */
5199	if (!radeon_card_posted(rdev)) {
5200	if (!rdev->bios) {
5201	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202	return -EINVAL;
5203	}
5204	DRM_INFO("GPU not posted. posting now...\n");
5205	atom_asic_init(rdev->mode_info.atom_context);
5206	}
5207	/* init golden registers */
5208	evergreen_init_golden_registers(rdev);
5209	/* Initialize scratch registers */
5210	r600_scratch_init(rdev);
5211	/* Initialize surface registers */
5212	radeon_surface_init(rdev);
5213	/* Initialize clocks */
5214	radeon_get_clock_info(dev: rdev_to_drm(rdev));
5215	/* Fence driver */
5216	radeon_fence_driver_init(rdev);
5217	/* initialize AGP */
5218	if (rdev->flags & RADEON_IS_AGP) {
5219	r = radeon_agp_init(rdev);
5220	if (r)
5221	radeon_agp_disable(rdev);
5222	}
5223	/* initialize memory controller */
5224	r = evergreen_mc_init(rdev);
5225	if (r)
5226	return r;
5227	/* Memory manager */
5228	r = radeon_bo_init(rdev);
5229	if (r)
5230	return r;
5231
5232	if (ASIC_IS_DCE5(rdev)) {
5233	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5234	r = ni_init_microcode(rdev);
5235	if (r) {
5236	DRM_ERROR("Failed to load firmware!\n");
5237	return r;
5238	}
5239	}
5240	} else {
5241	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5242	r = r600_init_microcode(rdev);
5243	if (r) {
5244	DRM_ERROR("Failed to load firmware!\n");
5245	return r;
5246	}
5247	}
5248	}
5249
5250	/* Initialize power management */
5251	radeon_pm_init(rdev);
5252
5253	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5254	r600_ring_init(rdev, cp: &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], ring_size: 1024 * 1024);
5255
5256	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5257	r600_ring_init(rdev, cp: &rdev->ring[R600_RING_TYPE_DMA_INDEX], ring_size: 64 * 1024);
5258
5259	evergreen_uvd_init(rdev);
5260
5261	rdev->ih.ring_obj = NULL;
5262	r600_ih_ring_init(rdev, ring_size: 64 * 1024);
5263
5264	r = r600_pcie_gart_init(rdev);
5265	if (r)
5266	return r;
5267
5268	rdev->accel_working = true;
5269	r = evergreen_startup(rdev);
5270	if (r) {
5271	dev_err(rdev->dev, "disabling GPU acceleration\n");
5272	r700_cp_fini(rdev);
5273	r600_dma_fini(rdev);
5274	r600_irq_fini(rdev);
5275	if (rdev->flags & RADEON_IS_IGP)
5276	sumo_rlc_fini(rdev);
5277	radeon_wb_fini(rdev);
5278	radeon_ib_pool_fini(rdev);
5279	radeon_irq_kms_fini(rdev);
5280	evergreen_pcie_gart_fini(rdev);
5281	rdev->accel_working = false;
5282	}
5283
5284	/* Don't start up if the MC ucode is missing on BTC parts.
5285	* The default clocks and voltages before the MC ucode
5286	* is loaded are not suffient for advanced operations.
5287	*/
5288	if (ASIC_IS_DCE5(rdev)) {
5289	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5290	DRM_ERROR("radeon: MC ucode required for NI+.\n");
5291	return -EINVAL;
5292	}
5293	}
5294
5295	return 0;
5296	}
5297
5298	void evergreen_fini(struct radeon_device *rdev)
5299	{
5300	radeon_pm_fini(rdev);
5301	radeon_audio_fini(rdev);
5302	r700_cp_fini(rdev);
5303	r600_dma_fini(rdev);
5304	r600_irq_fini(rdev);
5305	if (rdev->flags & RADEON_IS_IGP)
5306	sumo_rlc_fini(rdev);
5307	radeon_wb_fini(rdev);
5308	radeon_ib_pool_fini(rdev);
5309	radeon_irq_kms_fini(rdev);
5310	uvd_v1_0_fini(rdev);
5311	radeon_uvd_fini(rdev);
5312	evergreen_pcie_gart_fini(rdev);
5313	r600_vram_scratch_fini(rdev);
5314	radeon_gem_fini(rdev);
5315	radeon_fence_driver_fini(rdev);
5316	radeon_agp_fini(rdev);
5317	radeon_bo_fini(rdev);
5318	radeon_atombios_fini(rdev);
5319	kfree(objp: rdev->bios);
5320	rdev->bios = NULL;
5321	}
5322
5323	void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5324	{
5325	u32 link_width_cntl, speed_cntl;
5326
5327	if (radeon_pcie_gen2 == 0)
5328	return;
5329
5330	if (rdev->flags & RADEON_IS_IGP)
5331	return;
5332
5333	if (!(rdev->flags & RADEON_IS_PCIE))
5334	return;
5335
5336	/* x2 cards have a special sequence */
5337	if (ASIC_IS_X2(rdev))
5338	return;
5339
5340	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5341	(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5342	return;
5343
5344	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5345	if (speed_cntl & LC_CURRENT_DATA_RATE) {
5346	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5347	return;
5348	}
5349
5350	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5351
5352	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5353	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5354
5355	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5356	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5357	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5358
5359	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5360	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5361	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5362
5363	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5364	speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5365	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5366
5367	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5368	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5369	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5370
5371	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372	speed_cntl |= LC_GEN2_EN_STRAP;
5373	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5374
5375	} else {
5376	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5377	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5378	if (1)
5379	link_width_cntl |= LC_UPCONFIGURE_DIS;
5380	else
5381	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5382	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5383	}
5384	}
5385
5386	void evergreen_program_aspm(struct radeon_device *rdev)
5387	{
5388	u32 data, orig;
5389	u32 pcie_lc_cntl, pcie_lc_cntl_old;
5390	bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5391	/* fusion_platform = true
5392	* if the system is a fusion system
5393	* (APU or DGPU in a fusion system).
5394	* todo: check if the system is a fusion platform.
5395	*/
5396	bool fusion_platform = false;
5397
5398	if (radeon_aspm == 0)
5399	return;
5400
5401	if (!(rdev->flags & RADEON_IS_PCIE))
5402	return;
5403
5404	switch (rdev->family) {
5405	case CHIP_CYPRESS:
5406	case CHIP_HEMLOCK:
5407	case CHIP_JUNIPER:
5408	case CHIP_REDWOOD:
5409	case CHIP_CEDAR:
5410	case CHIP_SUMO:
5411	case CHIP_SUMO2:
5412	case CHIP_PALM:
5413	case CHIP_ARUBA:
5414	disable_l0s = true;
5415	break;
5416	default:
5417	disable_l0s = false;
5418	break;
5419	}
5420
5421	if (rdev->flags & RADEON_IS_IGP)
5422	fusion_platform = true; /* XXX also dGPUs in a fusion system */
5423
5424	data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5425	if (fusion_platform)
5426	data &= ~MULTI_PIF;
5427	else
5428	data |= MULTI_PIF;
5429	if (data != orig)
5430	WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5431
5432	data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5433	if (fusion_platform)
5434	data &= ~MULTI_PIF;
5435	else
5436	data |= MULTI_PIF;
5437	if (data != orig)
5438	WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5439
5440	pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5441	pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5442	if (!disable_l0s) {
5443	if (rdev->family >= CHIP_BARTS)
5444	pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5445	else
5446	pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5447	}
5448
5449	if (!disable_l1) {
5450	if (rdev->family >= CHIP_BARTS)
5451	pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5452	else
5453	pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5454
5455	if (!disable_plloff_in_l1) {
5456	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5457	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5458	data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5459	if (data != orig)
5460	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5461
5462	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5463	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5464	data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5465	if (data != orig)
5466	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5467
5468	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5469	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5470	data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5471	if (data != orig)
5472	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5473
5474	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5475	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5476	data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5477	if (data != orig)
5478	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5479
5480	if (rdev->family >= CHIP_BARTS) {
5481	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5482	data &= ~PLL_RAMP_UP_TIME_0_MASK;
5483	data |= PLL_RAMP_UP_TIME_0(4);
5484	if (data != orig)
5485	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5486
5487	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5488	data &= ~PLL_RAMP_UP_TIME_1_MASK;
5489	data |= PLL_RAMP_UP_TIME_1(4);
5490	if (data != orig)
5491	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5492
5493	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5494	data &= ~PLL_RAMP_UP_TIME_0_MASK;
5495	data |= PLL_RAMP_UP_TIME_0(4);
5496	if (data != orig)
5497	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5498
5499	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5500	data &= ~PLL_RAMP_UP_TIME_1_MASK;
5501	data |= PLL_RAMP_UP_TIME_1(4);
5502	if (data != orig)
5503	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5504	}
5505
5506	data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5507	data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5508	data |= LC_DYN_LANES_PWR_STATE(3);
5509	if (data != orig)
5510	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5511
5512	if (rdev->family >= CHIP_BARTS) {
5513	data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5514	data &= ~LS2_EXIT_TIME_MASK;
5515	data |= LS2_EXIT_TIME(1);
5516	if (data != orig)
5517	WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5518
5519	data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5520	data &= ~LS2_EXIT_TIME_MASK;
5521	data |= LS2_EXIT_TIME(1);
5522	if (data != orig)
5523	WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5524	}
5525	}
5526	}
5527
5528	/* evergreen parts only */
5529	if (rdev->family < CHIP_BARTS)
5530	pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5531
5532	if (pcie_lc_cntl != pcie_lc_cntl_old)
5533	WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5534	}
5535