1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
3
4
5	#include "msm_gem.h"
6	#include "msm_mmu.h"
7	#include "msm_gpu_trace.h"
8	#include "a6xx_gpu.h"
9	#include "a6xx_gmu.xml.h"
10
11	#include <linux/bitfield.h>
12	#include <linux/devfreq.h>
13	#include <linux/pm_domain.h>
14	#include <linux/soc/qcom/llcc-qcom.h>
15
16	#define GPU_PAS_ID 13
17
18	static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
19	{
20	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
21	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
22
23	/* Check that the GMU is idle */
24	if (!adreno_has_gmu_wrapper(gpu: adreno_gpu) && !a6xx_gmu_isidle(gmu: &a6xx_gpu->gmu))
25	return false;
26
27	/* Check tha the CX master is idle */
28	if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
29	~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
30	return false;
31
32	return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
33	A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
34	}
35
36	static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
37	{
38	/* wait for CP to drain ringbuffer: */
39	if (!adreno_idle(gpu, ring))
40	return false;
41
42	if (spin_until(_a6xx_check_idle(gpu))) {
43	DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
44	gpu->name, __builtin_return_address(0),
45	gpu_read(gpu, REG_A6XX_RBBM_STATUS),
46	gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
47	gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
48	gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
49	return false;
50	}
51
52	return true;
53	}
54
55	static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
56	{
57	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
58	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
59
60	/* Expanded APRIV doesn't need to issue the WHERE_AM_I opcode */
61	if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) {
62	OUT_PKT7(ring, opcode: CP_WHERE_AM_I, cnt: 2);
63	OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring)));
64	OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring)));
65	}
66	}
67
68	static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
69	{
70	uint32_t wptr;
71	unsigned long flags;
72
73	update_shadow_rptr(gpu, ring);
74
75	spin_lock_irqsave(&ring->preempt_lock, flags);
76
77	/* Copy the shadow to the actual register */
78	ring->cur = ring->next;
79
80	/* Make sure to wrap wptr if we need to */
81	wptr = get_wptr(ring);
82
83	spin_unlock_irqrestore(lock: &ring->preempt_lock, flags);
84
85	/* Make sure everything is posted before making a decision */
86	mb();
87
88	gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
89	}
90
91	static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter,
92	u64 iova)
93	{
94	OUT_PKT7(ring, opcode: CP_REG_TO_MEM, cnt: 3);
95	OUT_RING(ring, CP_REG_TO_MEM_0_REG(val: counter) |
96	CP_REG_TO_MEM_0_CNT(val: 2) |
97	CP_REG_TO_MEM_0_64B);
98	OUT_RING(ring, lower_32_bits(iova));
99	OUT_RING(ring, upper_32_bits(iova));
100	}
101
102	static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
103	struct msm_ringbuffer *ring, struct msm_file_private *ctx)
104	{
105	bool sysprof = refcount_read(r: &a6xx_gpu->base.base.sysprof_active) > 1;
106	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
107	phys_addr_t ttbr;
108	u32 asid;
109	u64 memptr = rbmemptr(ring, ttbr0);
110
111	if (ctx->seqno == a6xx_gpu->base.base.cur_ctx_seqno)
112	return;
113
114	if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid))
115	return;
116
117	if (!sysprof) {
118	if (!adreno_is_a7xx(gpu: adreno_gpu)) {
119	/* Turn off protected mode to write to special registers */
120	OUT_PKT7(ring, opcode: CP_SET_PROTECTED_MODE, cnt: 1);
121	OUT_RING(ring, 0);
122	}
123
124	OUT_PKT4(ring, REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, cnt: 1);
125	OUT_RING(ring, 1);
126	}
127
128	/* Execute the table update */
129	OUT_PKT7(ring, opcode: CP_SMMU_TABLE_UPDATE, cnt: 4);
130	OUT_RING(ring, CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(lower_32_bits(ttbr)));
131
132	OUT_RING(ring,
133	CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(upper_32_bits(ttbr)) |
134	CP_SMMU_TABLE_UPDATE_1_ASID(val: asid));
135	OUT_RING(ring, CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(val: 0));
136	OUT_RING(ring, CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(val: 0));
137
138	/*
139	* Write the new TTBR0 to the memstore. This is good for debugging.
140	*/
141	OUT_PKT7(ring, opcode: CP_MEM_WRITE, cnt: 4);
142	OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr)));
143	OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr)));
144	OUT_RING(ring, lower_32_bits(ttbr));
145	OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr));
146
147	/*
148	* Sync both threads after switching pagetables and enable BR only
149	* to make sure BV doesn't race ahead while BR is still switching
150	* pagetables.
151	*/
152	if (adreno_is_a7xx(gpu: &a6xx_gpu->base)) {
153	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
154	OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR);
155	}
156
157	/*
158	* And finally, trigger a uche flush to be sure there isn't anything
159	* lingering in that part of the GPU
160	*/
161
162	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 1);
163	OUT_RING(ring, CACHE_INVALIDATE);
164
165	if (!sysprof) {
166	/*
167	* Wait for SRAM clear after the pgtable update, so the
168	* two can happen in parallel:
169	*/
170	OUT_PKT7(ring, opcode: CP_WAIT_REG_MEM, cnt: 6);
171	OUT_RING(ring, CP_WAIT_REG_MEM_0_FUNCTION(val: WRITE_EQ));
172	OUT_RING(ring, CP_WAIT_REG_MEM_1_POLL_ADDR_LO(
173	REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS));
174	OUT_RING(ring, CP_WAIT_REG_MEM_2_POLL_ADDR_HI(val: 0));
175	OUT_RING(ring, CP_WAIT_REG_MEM_3_REF(val: 0x1));
176	OUT_RING(ring, CP_WAIT_REG_MEM_4_MASK(val: 0x1));
177	OUT_RING(ring, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(val: 0));
178
179	if (!adreno_is_a7xx(gpu: adreno_gpu)) {
180	/* Re-enable protected mode: */
181	OUT_PKT7(ring, opcode: CP_SET_PROTECTED_MODE, cnt: 1);
182	OUT_RING(ring, 1);
183	}
184	}
185	}
186
187	static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
188	{
189	unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
190	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
191	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
192	struct msm_ringbuffer *ring = submit->ring;
193	unsigned int i, ibs = 0;
194
195	a6xx_set_pagetable(a6xx_gpu, ring, ctx: submit->queue->ctx);
196
197	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
198	iova: rbmemptr_stats(ring, index, cpcycles_start));
199
200	/*
201	* For PM4 the GMU register offsets are calculated from the base of the
202	* GPU registers so we need to add 0x1a800 to the register value on A630
203	* to get the right value from PM4.
204	*/
205	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
206	iova: rbmemptr_stats(ring, index, alwayson_start));
207
208	/* Invalidate CCU depth and color */
209	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 1);
210	OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(val: PC_CCU_INVALIDATE_DEPTH));
211
212	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 1);
213	OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(val: PC_CCU_INVALIDATE_COLOR));
214
215	/* Submit the commands */
216	for (i = 0; i < submit->nr_cmds; i++) {
217	switch (submit->cmd[i].type) {
218	case MSM_SUBMIT_CMD_IB_TARGET_BUF:
219	break;
220	case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
221	if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
222	break;
223	fallthrough;
224	case MSM_SUBMIT_CMD_BUF:
225	OUT_PKT7(ring, opcode: CP_INDIRECT_BUFFER_PFE, cnt: 3);
226	OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
227	OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
228	OUT_RING(ring, submit->cmd[i].size);
229	ibs++;
230	break;
231	}
232
233	/*
234	* Periodically update shadow-wptr if needed, so that we
235	* can see partial progress of submits with large # of
236	* cmds.. otherwise we could needlessly stall waiting for
237	* ringbuffer state, simply due to looking at a shadow
238	* rptr value that has not been updated
239	*/
240	if ((ibs % 32) == 0)
241	update_shadow_rptr(gpu, ring);
242	}
243
244	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
245	iova: rbmemptr_stats(ring, index, cpcycles_end));
246	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
247	iova: rbmemptr_stats(ring, index, alwayson_end));
248
249	/* Write the fence to the scratch register */
250	OUT_PKT4(ring, regindx: REG_A6XX_CP_SCRATCH_REG(i0: 2), cnt: 1);
251	OUT_RING(ring, submit->seqno);
252
253	/*
254	* Execute a CACHE_FLUSH_TS event. This will ensure that the
255	* timestamp is written to the memory and then triggers the interrupt
256	*/
257	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 4);
258	OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(val: CACHE_FLUSH_TS) |
259	CP_EVENT_WRITE_0_IRQ);
260	OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
261	OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
262	OUT_RING(ring, submit->seqno);
263
264	trace_msm_gpu_submit_flush(submit,
265	gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
266
267	a6xx_flush(gpu, ring);
268	}
269
270	static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
271	{
272	unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
273	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
274	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
275	struct msm_ringbuffer *ring = submit->ring;
276	unsigned int i, ibs = 0;
277
278	/*
279	* Toggle concurrent binning for pagetable switch and set the thread to
280	* BR since only it can execute the pagetable switch packets.
281	*/
282	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
283	OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR);
284
285	a6xx_set_pagetable(a6xx_gpu, ring, ctx: submit->queue->ctx);
286
287	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
288	iova: rbmemptr_stats(ring, index, cpcycles_start));
289	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
290	iova: rbmemptr_stats(ring, index, alwayson_start));
291
292	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
293	OUT_RING(ring, CP_SET_THREAD_BOTH);
294
295	OUT_PKT7(ring, opcode: CP_SET_MARKER, cnt: 1);
296	OUT_RING(ring, 0x101); /* IFPC disable */
297
298	OUT_PKT7(ring, opcode: CP_SET_MARKER, cnt: 1);
299	OUT_RING(ring, 0x00d); /* IB1LIST start */
300
301	/* Submit the commands */
302	for (i = 0; i < submit->nr_cmds; i++) {
303	switch (submit->cmd[i].type) {
304	case MSM_SUBMIT_CMD_IB_TARGET_BUF:
305	break;
306	case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
307	if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
308	break;
309	fallthrough;
310	case MSM_SUBMIT_CMD_BUF:
311	OUT_PKT7(ring, opcode: CP_INDIRECT_BUFFER_PFE, cnt: 3);
312	OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
313	OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
314	OUT_RING(ring, submit->cmd[i].size);
315	ibs++;
316	break;
317	}
318
319	/*
320	* Periodically update shadow-wptr if needed, so that we
321	* can see partial progress of submits with large # of
322	* cmds.. otherwise we could needlessly stall waiting for
323	* ringbuffer state, simply due to looking at a shadow
324	* rptr value that has not been updated
325	*/
326	if ((ibs % 32) == 0)
327	update_shadow_rptr(gpu, ring);
328	}
329
330	OUT_PKT7(ring, opcode: CP_SET_MARKER, cnt: 1);
331	OUT_RING(ring, 0x00e); /* IB1LIST end */
332
333	get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
334	iova: rbmemptr_stats(ring, index, cpcycles_end));
335	get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
336	iova: rbmemptr_stats(ring, index, alwayson_end));
337
338	/* Write the fence to the scratch register */
339	OUT_PKT4(ring, regindx: REG_A6XX_CP_SCRATCH_REG(i0: 2), cnt: 1);
340	OUT_RING(ring, submit->seqno);
341
342	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
343	OUT_RING(ring, CP_SET_THREAD_BR);
344
345	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 1);
346	OUT_RING(ring, CCU_INVALIDATE_DEPTH);
347
348	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 1);
349	OUT_RING(ring, CCU_INVALIDATE_COLOR);
350
351	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
352	OUT_RING(ring, CP_SET_THREAD_BV);
353
354	/*
355	* Make sure the timestamp is committed once BV pipe is
356	* completely done with this submission.
357	*/
358	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 4);
359	OUT_RING(ring, CACHE_CLEAN | BIT(27));
360	OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence)));
361	OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence)));
362	OUT_RING(ring, submit->seqno);
363
364	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
365	OUT_RING(ring, CP_SET_THREAD_BR);
366
367	/*
368	* This makes sure that BR doesn't race ahead and commit
369	* timestamp to memstore while BV is still processing
370	* this submission.
371	*/
372	OUT_PKT7(ring, opcode: CP_WAIT_TIMESTAMP, cnt: 4);
373	OUT_RING(ring, 0);
374	OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence)));
375	OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence)));
376	OUT_RING(ring, submit->seqno);
377
378	/* write the ringbuffer timestamp */
379	OUT_PKT7(ring, opcode: CP_EVENT_WRITE, cnt: 4);
380	OUT_RING(ring, CACHE_CLEAN | CP_EVENT_WRITE_0_IRQ | BIT(27));
381	OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
382	OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
383	OUT_RING(ring, submit->seqno);
384
385	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
386	OUT_RING(ring, CP_SET_THREAD_BOTH);
387
388	OUT_PKT7(ring, opcode: CP_SET_MARKER, cnt: 1);
389	OUT_RING(ring, 0x100); /* IFPC enable */
390
391	trace_msm_gpu_submit_flush(submit,
392	gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
393
394	a6xx_flush(gpu, ring);
395	}
396
397	const struct adreno_reglist a612_hwcg[] = {
398	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
399	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
400	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081},
401	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
402	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
403	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
404	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
405	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
406	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
407	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
408	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
409	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
410	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
411	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
412	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
413	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
414	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
415	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222},
416	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
417	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
418	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022},
419	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
420	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
421	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
422	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
423	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
424	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
425	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
426	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
427	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
428	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
429	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
430	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
431	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
432	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
433	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
434	{REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
435	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
436	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
437	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
438	{REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
439	{REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
440	{REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
441	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
442	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
443	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
444	{},
445	};
446
447	/* For a615 family (a615, a616, a618 and a619) */
448	const struct adreno_reglist a615_hwcg[] = {
449	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
450	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
451	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
452	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
453	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
454	{REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
455	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
456	{REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
457	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
458	{REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
459	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
460	{REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
461	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
462	{REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
463	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
464	{REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
465	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
466	{REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
467	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
468	{REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
469	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
470	{REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
471	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
472	{REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
473	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
474	{REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
475	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
476	{REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
477	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
478	{REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
479	{REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
480	{REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
481	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
482	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
483	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
484	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
485	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002020},
486	{REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
487	{REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
488	{REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
489	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
490	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00},
491	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00},
492	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00},
493	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
494	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
495	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
496	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
497	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
498	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
499	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
500	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
501	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
502	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
503	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
504	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
505	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
506	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
507	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
508	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
509	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
510	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
511	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
512	{},
513	};
514
515	const struct adreno_reglist a630_hwcg[] = {
516	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
517	{REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
518	{REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
519	{REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
520	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
521	{REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
522	{REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
523	{REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
524	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
525	{REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
526	{REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
527	{REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
528	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
529	{REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
530	{REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
531	{REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
532	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
533	{REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
534	{REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
535	{REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
536	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
537	{REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
538	{REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
539	{REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
540	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
541	{REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
542	{REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
543	{REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
544	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
545	{REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
546	{REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
547	{REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
548	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
549	{REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
550	{REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
551	{REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
552	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
553	{REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
554	{REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
555	{REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
556	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
557	{REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
558	{REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
559	{REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
560	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
561	{REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
562	{REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
563	{REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
564	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
565	{REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
566	{REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
567	{REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
568	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
569	{REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
570	{REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
571	{REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
572	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
573	{REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
574	{REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
575	{REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
576	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
577	{REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
578	{REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
579	{REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
580	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
581	{REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
582	{REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
583	{REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
584	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
585	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
586	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
587	{REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
588	{REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
589	{REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
590	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
591	{REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
592	{REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
593	{REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
594	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
595	{REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
596	{REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
597	{REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
598	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
599	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
600	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
601	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
602	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
603	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
604	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
605	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
606	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
607	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
608	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
609	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
610	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
611	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
612	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
613	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
614	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
615	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
616	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
617	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
618	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
619	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
620	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
621	{},
622	};
623
624	const struct adreno_reglist a640_hwcg[] = {
625	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
626	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
627	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
628	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
629	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
630	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
631	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
632	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
633	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
634	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
635	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
636	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
637	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
638	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
639	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
640	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
641	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
642	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
643	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
644	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
645	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022},
646	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
647	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
648	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
649	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
650	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
651	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
652	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
653	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
654	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
655	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
656	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
657	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
658	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
659	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
660	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
661	{REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
662	{REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
663	{REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
664	{REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
665	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
666	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
667	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
668	{REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
669	{REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
670	{REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
671	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
672	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
673	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
674	{},
675	};
676
677	const struct adreno_reglist a650_hwcg[] = {
678	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
679	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
680	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
681	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
682	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
683	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
684	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
685	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
686	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
687	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
688	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
689	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
690	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
691	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
692	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
693	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
694	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
695	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
696	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
697	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
698	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
699	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
700	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
701	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
702	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
703	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
704	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
705	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
706	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
707	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
708	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
709	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
710	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
711	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
712	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
713	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
714	{REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
715	{REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
716	{REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
717	{REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777},
718	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
719	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
720	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
721	{REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
722	{REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
723	{REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
724	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
725	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
726	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
727	{},
728	};
729
730	const struct adreno_reglist a660_hwcg[] = {
731	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
732	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
733	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
734	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
735	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
736	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
737	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
738	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
739	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
740	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
741	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
742	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
743	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
744	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
745	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
746	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
747	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
748	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
749	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
750	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
751	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
752	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
753	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
754	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
755	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
756	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
757	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
758	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
759	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
760	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
761	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
762	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
763	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
764	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
765	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
766	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
767	{REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
768	{REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
769	{REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
770	{REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
771	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
772	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
773	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
774	{REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
775	{REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
776	{REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
777	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
778	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
779	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
780	{},
781	};
782
783	const struct adreno_reglist a690_hwcg[] = {
784	{REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
785	{REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
786	{REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
787	{REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
788	{REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
789	{REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
790	{REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
791	{REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
792	{REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
793	{REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
794	{REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
795	{REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
796	{REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
797	{REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
798	{REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
799	{REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
800	{REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
801	{REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
802	{REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
803	{REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
804	{REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
805	{REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
806	{REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
807	{REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
808	{REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
809	{REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
810	{REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
811	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
812	{REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
813	{REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
814	{REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
815	{REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
816	{REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
817	{REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
818	{REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
819	{REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
820	{REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
821	{REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
822	{REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
823	{REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
824	{REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
825	{REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
826	{REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
827	{REG_A6XX_RBBM_CLOCK_CNTL, 0x8AA8AA82},
828	{REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
829	{REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
830	{REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
831	{REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
832	{REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
833	{REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
834	{REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 0x20200},
835	{REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x10111},
836	{REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x5555},
837	{}
838	};
839
840	const struct adreno_reglist a702_hwcg[] = {
841	{ REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222 },
842	{ REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220 },
843	{ REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081 },
844	{ REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf },
845	{ REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222 },
846	{ REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
847	{ REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
848	{ REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222 },
849	{ REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
850	{ REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
851	{ REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
852	{ REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
853	{ REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
854	{ REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
855	{ REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
856	{ REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
857	{ REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
858	{ REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222 },
859	{ REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
860	{ REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00 },
861	{ REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022 },
862	{ REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555 },
863	{ REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
864	{ REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044 },
865	{ REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
866	{ REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 },
867	{ REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222 },
868	{ REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 },
869	{ REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
870	{ REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 },
871	{ REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 },
872	{ REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
873	{ REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
874	{ REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
875	{ REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
876	{ REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
877	{ REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
878	{ REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
879	{ REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 },
880	{ REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 },
881	{ REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
882	{ REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
883	{ REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
884	{ REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
885	{ REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
886	{ REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
887	{ REG_A6XX_RBBM_CLOCK_CNTL_FCHE, 0x00000222 },
888	{ REG_A6XX_RBBM_CLOCK_DELAY_FCHE, 0x00000000 },
889	{ REG_A6XX_RBBM_CLOCK_HYST_FCHE, 0x00000000 },
890	{ REG_A6XX_RBBM_CLOCK_CNTL_GLC, 0x00222222 },
891	{ REG_A6XX_RBBM_CLOCK_DELAY_GLC, 0x00000000 },
892	{ REG_A6XX_RBBM_CLOCK_HYST_GLC, 0x00000000 },
893	{ REG_A6XX_RBBM_CLOCK_CNTL_MHUB, 0x00000002 },
894	{ REG_A6XX_RBBM_CLOCK_DELAY_MHUB, 0x00000000 },
895	{ REG_A6XX_RBBM_CLOCK_HYST_MHUB, 0x00000000 },
896	{}
897	};
898
899	const struct adreno_reglist a730_hwcg[] = {
900	{ REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 },
901	{ REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022222 },
902	{ REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf },
903	{ REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 },
904	{ REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 },
905	{ REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
906	{ REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
907	{ REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 },
908	{ REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
909	{ REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
910	{ REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
911	{ REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
912	{ REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
913	{ REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
914	{ REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
915	{ REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
916	{ REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
917	{ REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 },
918	{ REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 },
919	{ REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
920	{ REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 },
921	{ REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
922	{ REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 },
923	{ REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 },
924	{ REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 },
925	{ REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
926	{ REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 },
927	{ REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
928	{ REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 },
929	{ REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 },
930	{ REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 },
931	{ REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 },
932	{ REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 },
933	{ REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 },
934	{ REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
935	{ REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
936	{ REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
937	{ REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 },
938	{ REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
939	{ REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 },
940	{ REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
941	{ REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
942	{ REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
943	{ REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 },
944	{ REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 },
945	{ REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000223 },
946	{ REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 },
947	{ REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
948	{ REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
949	{ REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
950	{ REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
951	{ REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
952	{ REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
953	{},
954	};
955
956	const struct adreno_reglist a740_hwcg[] = {
957	{ REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 },
958	{ REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x22022222 },
959	{ REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x003cf3cf },
960	{ REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 },
961	{ REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 },
962	{ REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
963	{ REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
964	{ REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 },
965	{ REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
966	{ REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
967	{ REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
968	{ REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
969	{ REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
970	{ REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
971	{ REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
972	{ REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
973	{ REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
974	{ REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x00222222 },
975	{ REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000444 },
976	{ REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000222 },
977	{ REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
978	{ REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 },
979	{ REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
980	{ REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 },
981	{ REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 },
982	{ REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 },
983	{ REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
984	{ REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 },
985	{ REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
986	{ REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 },
987	{ REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 },
988	{ REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 },
989	{ REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00222222 },
990	{ REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 },
991	{ REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 },
992	{ REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
993	{ REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
994	{ REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
995	{ REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000000 },
996	{ REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
997	{ REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00000000 },
998	{ REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
999	{ REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
1000	{ REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
1001	{ REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 },
1002	{ REG_A7XX_RBBM_CLOCK_HYST2_VFD, 0x00000000 },
1003	{ REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000222 },
1004	{ REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 },
1005	{ REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
1006	{ REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
1007	{ REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
1008	{ REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
1009	{ REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
1010	{ REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
1011	{},
1012	};
1013
1014	static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
1015	{
1016	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1017	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1018	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1019	const struct adreno_reglist *reg;
1020	unsigned int i;
1021	u32 val, clock_cntl_on, cgc_mode;
1022
1023	if (!(adreno_gpu->info->hwcg || adreno_is_a7xx(gpu: adreno_gpu)))
1024	return;
1025
1026	if (adreno_is_a630(gpu: adreno_gpu))
1027	clock_cntl_on = 0x8aa8aa02;
1028	else if (adreno_is_a610(gpu: adreno_gpu))
1029	clock_cntl_on = 0xaaa8aa82;
1030	else if (adreno_is_a702(gpu: adreno_gpu))
1031	clock_cntl_on = 0xaaaaaa82;
1032	else
1033	clock_cntl_on = 0x8aa8aa82;
1034
1035	if (adreno_is_a7xx(gpu: adreno_gpu)) {
1036	cgc_mode = adreno_is_a740_family(gpu: adreno_gpu) ? 0x20222 : 0x20000;
1037
1038	gmu_write(gmu: &a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL,
1039	value: state ? cgc_mode : 0);
1040	gmu_write(gmu: &a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL,
1041	value: state ? 0x10111 : 0);
1042	gmu_write(gmu: &a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL,
1043	value: state ? 0x5555 : 0);
1044	}
1045
1046	if (!adreno_gpu->info->hwcg) {
1047	gpu_write(gpu, REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL, 1);
1048	gpu_write(gpu, REG_A7XX_RBBM_CGC_GLOBAL_LOAD_CMD, state ? 1 : 0);
1049
1050	if (state) {
1051	gpu_write(gpu, REG_A7XX_RBBM_CGC_P2S_TRIG_CMD, 1);
1052
1053	if (gpu_poll_timeout(gpu, REG_A7XX_RBBM_CGC_P2S_STATUS, val,
1054	val & A7XX_RBBM_CGC_P2S_STATUS_TXDONE, 1, 10)) {
1055	dev_err(&gpu->pdev->dev, "RBBM_CGC_P2S_STATUS TXDONE Poll failed\n");
1056	return;
1057	}
1058
1059	gpu_write(gpu, REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL, 0);
1060	}
1061
1062	return;
1063	}
1064
1065	val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
1066
1067	/* Don't re-program the registers if they are already correct */
1068	if ((!state && !val) || (state && (val == clock_cntl_on)))
1069	return;
1070
1071	/* Disable SP clock before programming HWCG registers */
1072	if (!adreno_is_a610_family(gpu: adreno_gpu) && !adreno_is_a7xx(gpu: adreno_gpu))
1073	gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, mask: 1, or: 0);
1074
1075	for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++)
1076	gpu_write(gpu, reg->offset, state ? reg->value : 0);
1077
1078	/* Enable SP clock */
1079	if (!adreno_is_a610_family(gpu: adreno_gpu) && !adreno_is_a7xx(gpu: adreno_gpu))
1080	gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, mask: 0, or: 1);
1081
1082	gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? clock_cntl_on : 0);
1083	}
1084
1085	/* For a615, a616, a618, a619, a630, a640 and a680 */
1086	static const u32 a6xx_protect[] = {
1087	A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1088	A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1089	A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1090	A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1091	A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1092	A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1093	A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1094	A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1095	A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1096	A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1097	A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1098	A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1099	A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1100	A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1101	A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1102	A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1103	A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1104	A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1105	A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1106	A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1107	A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1108	A6XX_PROTECT_NORDWR(0x09e70, 0x0001),
1109	A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1110	A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1111	A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1112	A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
1113	A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1114	A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1115	A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
1116	A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1117	A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1118	A6XX_PROTECT_NORDWR(0x11c00, 0x0000), /* note: infinite range */
1119	};
1120
1121	/* These are for a620 and a650 */
1122	static const u32 a650_protect[] = {
1123	A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1124	A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1125	A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1126	A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1127	A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1128	A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1129	A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1130	A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1131	A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1132	A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1133	A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1134	A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1135	A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1136	A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1137	A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1138	A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1139	A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1140	A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1141	A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1142	A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1143	A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
1144	A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1145	A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
1146	A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1147	A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1148	A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1149	A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
1150	A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1151	A6XX_PROTECT_NORDWR(0x0b608, 0x0007),
1152	A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1153	A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
1154	A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1155	A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1156	A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
1157	A6XX_PROTECT_NORDWR(0x1a800, 0x1fff),
1158	A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1159	A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1160	A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
1161	A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
1162	};
1163
1164	/* These are for a635 and a660 */
1165	static const u32 a660_protect[] = {
1166	A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1167	A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1168	A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1169	A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1170	A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1171	A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1172	A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1173	A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1174	A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1175	A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1176	A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1177	A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1178	A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1179	A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1180	A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1181	A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1182	A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1183	A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1184	A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1185	A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1186	A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
1187	A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1188	A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
1189	A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1190	A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1191	A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1192	A6XX_PROTECT_NORDWR(0x0ae50, 0x012f),
1193	A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1194	A6XX_PROTECT_NORDWR(0x0b608, 0x0006),
1195	A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1196	A6XX_PROTECT_NORDWR(0x0be20, 0x015f),
1197	A6XX_PROTECT_NORDWR(0x0d000, 0x05ff),
1198	A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1199	A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1200	A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
1201	A6XX_PROTECT_NORDWR(0x1a400, 0x1fff),
1202	A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1203	A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1204	A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
1205	A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
1206	A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
1207	};
1208
1209	/* These are for a690 */
1210	static const u32 a690_protect[] = {
1211	A6XX_PROTECT_RDONLY(0x00000, 0x004ff),
1212	A6XX_PROTECT_RDONLY(0x00501, 0x00001),
1213	A6XX_PROTECT_RDONLY(0x0050b, 0x002f4),
1214	A6XX_PROTECT_NORDWR(0x0050e, 0x00000),
1215	A6XX_PROTECT_NORDWR(0x00510, 0x00000),
1216	A6XX_PROTECT_NORDWR(0x00534, 0x00000),
1217	A6XX_PROTECT_NORDWR(0x00800, 0x00082),
1218	A6XX_PROTECT_NORDWR(0x008a0, 0x00008),
1219	A6XX_PROTECT_NORDWR(0x008ab, 0x00024),
1220	A6XX_PROTECT_RDONLY(0x008de, 0x000ae),
1221	A6XX_PROTECT_NORDWR(0x00900, 0x0004d),
1222	A6XX_PROTECT_NORDWR(0x0098d, 0x00272),
1223	A6XX_PROTECT_NORDWR(0x00e00, 0x00001),
1224	A6XX_PROTECT_NORDWR(0x00e03, 0x0000c),
1225	A6XX_PROTECT_NORDWR(0x03c00, 0x000c3),
1226	A6XX_PROTECT_RDONLY(0x03cc4, 0x01fff),
1227	A6XX_PROTECT_NORDWR(0x08630, 0x001cf),
1228	A6XX_PROTECT_NORDWR(0x08e00, 0x00000),
1229	A6XX_PROTECT_NORDWR(0x08e08, 0x00007),
1230	A6XX_PROTECT_NORDWR(0x08e50, 0x0001f),
1231	A6XX_PROTECT_NORDWR(0x08e80, 0x0027f),
1232	A6XX_PROTECT_NORDWR(0x09624, 0x001db),
1233	A6XX_PROTECT_NORDWR(0x09e60, 0x00011),
1234	A6XX_PROTECT_NORDWR(0x09e78, 0x00187),
1235	A6XX_PROTECT_NORDWR(0x0a630, 0x001cf),
1236	A6XX_PROTECT_NORDWR(0x0ae02, 0x00000),
1237	A6XX_PROTECT_NORDWR(0x0ae50, 0x0012f),
1238	A6XX_PROTECT_NORDWR(0x0b604, 0x00000),
1239	A6XX_PROTECT_NORDWR(0x0b608, 0x00006),
1240	A6XX_PROTECT_NORDWR(0x0be02, 0x00001),
1241	A6XX_PROTECT_NORDWR(0x0be20, 0x0015f),
1242	A6XX_PROTECT_NORDWR(0x0d000, 0x005ff),
1243	A6XX_PROTECT_NORDWR(0x0f000, 0x00bff),
1244	A6XX_PROTECT_RDONLY(0x0fc00, 0x01fff),
1245	A6XX_PROTECT_NORDWR(0x11c00, 0x00000), /*note: infiite range */
1246	};
1247
1248	static const u32 a730_protect[] = {
1249	A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1250	A6XX_PROTECT_RDONLY(0x0050b, 0x0058),
1251	A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1252	A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1253	A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1254	A6XX_PROTECT_RDONLY(0x005fb, 0x009d),
1255	A6XX_PROTECT_NORDWR(0x00699, 0x01e9),
1256	A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1257	A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1258	/* 0x008d0-0x008dd are unprotected on purpose for tools like perfetto */
1259	A6XX_PROTECT_RDONLY(0x008de, 0x0154),
1260	A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1261	A6XX_PROTECT_NORDWR(0x0098d, 0x00b2),
1262	A6XX_PROTECT_NORDWR(0x00a41, 0x01be),
1263	A6XX_PROTECT_NORDWR(0x00df0, 0x0001),
1264	A6XX_PROTECT_NORDWR(0x00e01, 0x0000),
1265	A6XX_PROTECT_NORDWR(0x00e07, 0x0008),
1266	A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1267	A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1268	A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1269	A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1270	A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1271	A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1272	A6XX_PROTECT_NORDWR(0x08e80, 0x0280),
1273	A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1274	A6XX_PROTECT_NORDWR(0x09e40, 0x0000),
1275	A6XX_PROTECT_NORDWR(0x09e64, 0x000d),
1276	A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1277	A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1278	A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1279	A6XX_PROTECT_NORDWR(0x0ae50, 0x000f),
1280	A6XX_PROTECT_NORDWR(0x0ae66, 0x0003),
1281	A6XX_PROTECT_NORDWR(0x0ae6f, 0x0003),
1282	A6XX_PROTECT_NORDWR(0x0b604, 0x0003),
1283	A6XX_PROTECT_NORDWR(0x0ec00, 0x0fff),
1284	A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1285	A6XX_PROTECT_NORDWR(0x18400, 0x0053),
1286	A6XX_PROTECT_RDONLY(0x18454, 0x0004),
1287	A6XX_PROTECT_NORDWR(0x18459, 0x1fff),
1288	A6XX_PROTECT_NORDWR(0x1a459, 0x1fff),
1289	A6XX_PROTECT_NORDWR(0x1c459, 0x1fff),
1290	A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1291	A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1292	A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
1293	A6XX_PROTECT_NORDWR(0x1f878, 0x002a),
1294	/* CP_PROTECT_REG[44, 46] are left untouched! */
1295	0,
1296	0,
1297	0,
1298	A6XX_PROTECT_NORDWR(0x1f8c0, 0x00000),
1299	};
1300
1301	static void a6xx_set_cp_protect(struct msm_gpu *gpu)
1302	{
1303	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1304	const u32 *regs = a6xx_protect;
1305	unsigned i, count, count_max;
1306
1307	if (adreno_is_a650(gpu: adreno_gpu) || adreno_is_a702(gpu: adreno_gpu)) {
1308	regs = a650_protect;
1309	count = ARRAY_SIZE(a650_protect);
1310	count_max = 48;
1311	BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
1312	} else if (adreno_is_a690(gpu: adreno_gpu)) {
1313	regs = a690_protect;
1314	count = ARRAY_SIZE(a690_protect);
1315	count_max = 48;
1316	BUILD_BUG_ON(ARRAY_SIZE(a690_protect) > 48);
1317	} else if (adreno_is_a660_family(gpu: adreno_gpu)) {
1318	regs = a660_protect;
1319	count = ARRAY_SIZE(a660_protect);
1320	count_max = 48;
1321	BUILD_BUG_ON(ARRAY_SIZE(a660_protect) > 48);
1322	} else if (adreno_is_a730(gpu: adreno_gpu) ||
1323	adreno_is_a740(gpu: adreno_gpu) ||
1324	adreno_is_a750(gpu: adreno_gpu)) {
1325	regs = a730_protect;
1326	count = ARRAY_SIZE(a730_protect);
1327	count_max = 48;
1328	BUILD_BUG_ON(ARRAY_SIZE(a730_protect) > 48);
1329	} else {
1330	regs = a6xx_protect;
1331	count = ARRAY_SIZE(a6xx_protect);
1332	count_max = 32;
1333	BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
1334	}
1335
1336	/*
1337	* Enable access protection to privileged registers, fault on an access
1338	* protect violation and select the last span to protect from the start
1339	* address all the way to the end of the register address space
1340	*/
1341	gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL,
1342	A6XX_CP_PROTECT_CNTL_ACCESS_PROT_EN |
1343	A6XX_CP_PROTECT_CNTL_ACCESS_FAULT_ON_VIOL_EN |
1344	A6XX_CP_PROTECT_CNTL_LAST_SPAN_INF_RANGE);
1345
1346	for (i = 0; i < count - 1; i++) {
1347	/* Intentionally skip writing to some registers */
1348	if (regs[i])
1349	gpu_write(gpu, REG_A6XX_CP_PROTECT(i), regs[i]);
1350	}
1351	/* last CP_PROTECT to have "infinite" length on the last entry */
1352	gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]);
1353	}
1354
1355	static void a6xx_calc_ubwc_config(struct adreno_gpu *gpu)
1356	{
1357	/* Unknown, introduced with A650 family, related to UBWC mode/ver 4 */
1358	gpu->ubwc_config.rgb565_predicator = 0;
1359	/* Unknown, introduced with A650 family */
1360	gpu->ubwc_config.uavflagprd_inv = 0;
1361	/* Whether the minimum access length is 64 bits */
1362	gpu->ubwc_config.min_acc_len = 0;
1363	/* Entirely magic, per-GPU-gen value */
1364	gpu->ubwc_config.ubwc_mode = 0;
1365	/*
1366	* The Highest Bank Bit value represents the bit of the highest DDR bank.
1367	* This should ideally use DRAM type detection.
1368	*/
1369	gpu->ubwc_config.highest_bank_bit = 15;
1370
1371	if (adreno_is_a610(gpu)) {
1372	gpu->ubwc_config.highest_bank_bit = 13;
1373	gpu->ubwc_config.min_acc_len = 1;
1374	gpu->ubwc_config.ubwc_mode = 1;
1375	}
1376
1377	if (adreno_is_a618(gpu))
1378	gpu->ubwc_config.highest_bank_bit = 14;
1379
1380	if (adreno_is_a619(gpu))
1381	/* TODO: Should be 14 but causes corruption at e.g. 1920x1200 on DP */
1382	gpu->ubwc_config.highest_bank_bit = 13;
1383
1384	if (adreno_is_a619_holi(gpu))
1385	gpu->ubwc_config.highest_bank_bit = 13;
1386
1387	if (adreno_is_a640_family(gpu))
1388	gpu->ubwc_config.amsbc = 1;
1389
1390	if (adreno_is_a650(gpu) ||
1391	adreno_is_a660(gpu) ||
1392	adreno_is_a690(gpu) ||
1393	adreno_is_a730(gpu) ||
1394	adreno_is_a740_family(gpu)) {
1395	/* TODO: get ddr type from bootloader and use 2 for LPDDR4 */
1396	gpu->ubwc_config.highest_bank_bit = 16;
1397	gpu->ubwc_config.amsbc = 1;
1398	gpu->ubwc_config.rgb565_predicator = 1;
1399	gpu->ubwc_config.uavflagprd_inv = 2;
1400	}
1401
1402	if (adreno_is_7c3(gpu)) {
1403	gpu->ubwc_config.highest_bank_bit = 14;
1404	gpu->ubwc_config.amsbc = 1;
1405	gpu->ubwc_config.rgb565_predicator = 1;
1406	gpu->ubwc_config.uavflagprd_inv = 2;
1407	}
1408
1409	if (adreno_is_a702(gpu)) {
1410	gpu->ubwc_config.highest_bank_bit = 14;
1411	gpu->ubwc_config.min_acc_len = 1;
1412	gpu->ubwc_config.ubwc_mode = 2;
1413	}
1414	}
1415
1416	static void a6xx_set_ubwc_config(struct msm_gpu *gpu)
1417	{
1418	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1419	/*
1420	* We subtract 13 from the highest bank bit (13 is the minimum value
1421	* allowed by hw) and write the lowest two bits of the remaining value
1422	* as hbb_lo and the one above it as hbb_hi to the hardware.
1423	*/
1424	BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13);
1425	u32 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13;
1426	u32 hbb_hi = hbb >> 2;
1427	u32 hbb_lo = hbb & 3;
1428
1429	gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL,
1430	adreno_gpu->ubwc_config.rgb565_predicator << 11 |
1431	hbb_hi << 10 | adreno_gpu->ubwc_config.amsbc << 4 |
1432	adreno_gpu->ubwc_config.min_acc_len << 3 |
1433	hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1434
1435	gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, hbb_hi << 4 |
1436	adreno_gpu->ubwc_config.min_acc_len << 3 |
1437	hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1438
1439	gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, hbb_hi << 10 |
1440	adreno_gpu->ubwc_config.uavflagprd_inv << 4 |
1441	adreno_gpu->ubwc_config.min_acc_len << 3 |
1442	hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1443
1444	if (adreno_is_a7xx(gpu: adreno_gpu))
1445	gpu_write(gpu, REG_A7XX_GRAS_NC_MODE_CNTL,
1446	FIELD_PREP(GENMASK(8, 5), hbb_lo));
1447
1448	gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL,
1449	adreno_gpu->ubwc_config.min_acc_len << 23 | hbb_lo << 21);
1450	}
1451
1452	static int a6xx_cp_init(struct msm_gpu *gpu)
1453	{
1454	struct msm_ringbuffer *ring = gpu->rb[0];
1455
1456	OUT_PKT7(ring, opcode: CP_ME_INIT, cnt: 8);
1457
1458	OUT_RING(ring, 0x0000002f);
1459
1460	/* Enable multiple hardware contexts */
1461	OUT_RING(ring, 0x00000003);
1462
1463	/* Enable error detection */
1464	OUT_RING(ring, 0x20000000);
1465
1466	/* Don't enable header dump */
1467	OUT_RING(ring, 0x00000000);
1468	OUT_RING(ring, 0x00000000);
1469
1470	/* No workarounds enabled */
1471	OUT_RING(ring, 0x00000000);
1472
1473	/* Pad rest of the cmds with 0's */
1474	OUT_RING(ring, 0x00000000);
1475	OUT_RING(ring, 0x00000000);
1476
1477	a6xx_flush(gpu, ring);
1478	return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
1479	}
1480
1481	static int a7xx_cp_init(struct msm_gpu *gpu)
1482	{
1483	struct msm_ringbuffer *ring = gpu->rb[0];
1484	u32 mask;
1485
1486	/* Disable concurrent binning before sending CP init */
1487	OUT_PKT7(ring, opcode: CP_THREAD_CONTROL, cnt: 1);
1488	OUT_RING(ring, BIT(27));
1489
1490	OUT_PKT7(ring, opcode: CP_ME_INIT, cnt: 7);
1491
1492	/* Use multiple HW contexts */
1493	mask = BIT(0);
1494
1495	/* Enable error detection */
1496	mask |= BIT(1);
1497
1498	/* Set default reset state */
1499	mask |= BIT(3);
1500
1501	/* Disable save/restore of performance counters across preemption */
1502	mask |= BIT(6);
1503
1504	/* Enable the register init list with the spinlock */
1505	mask |= BIT(8);
1506
1507	OUT_RING(ring, mask);
1508
1509	/* Enable multiple hardware contexts */
1510	OUT_RING(ring, 0x00000003);
1511
1512	/* Enable error detection */
1513	OUT_RING(ring, 0x20000000);
1514
1515	/* Operation mode mask */
1516	OUT_RING(ring, 0x00000002);
1517
1518	/* *Don't* send a power up reg list for concurrent binning (TODO) */
1519	/* Lo address */
1520	OUT_RING(ring, 0x00000000);
1521	/* Hi address */
1522	OUT_RING(ring, 0x00000000);
1523	/* BIT(31) set => read the regs from the list */
1524	OUT_RING(ring, 0x00000000);
1525
1526	a6xx_flush(gpu, ring);
1527	return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
1528	}
1529
1530	/*
1531	* Check that the microcode version is new enough to include several key
1532	* security fixes. Return true if the ucode is safe.
1533	*/
1534	static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu,
1535	struct drm_gem_object *obj)
1536	{
1537	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1538	struct msm_gpu *gpu = &adreno_gpu->base;
1539	const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE];
1540	u32 *buf = msm_gem_get_vaddr(obj);
1541	bool ret = false;
1542
1543	if (IS_ERR(ptr: buf))
1544	return false;
1545
1546	/* A7xx is safe! */
1547	if (adreno_is_a7xx(gpu: adreno_gpu) || adreno_is_a702(gpu: adreno_gpu))
1548	return true;
1549
1550	/*
1551	* Targets up to a640 (a618, a630 and a640) need to check for a
1552	* microcode version that is patched to support the whereami opcode or
1553	* one that is new enough to include it by default.
1554	*
1555	* a650 tier targets don't need whereami but still need to be
1556	* equal to or newer than 0.95 for other security fixes
1557	*
1558	* a660 targets have all the critical security fixes from the start
1559	*/
1560	if (!strcmp(sqe_name, "a630_sqe.fw")) {
1561	/*
1562	* If the lowest nibble is 0xa that is an indication that this
1563	* microcode has been patched. The actual version is in dword
1564	* [3] but we only care about the patchlevel which is the lowest
1565	* nibble of dword [3]
1566	*
1567	* Otherwise check that the firmware is greater than or equal
1568	* to 1.90 which was the first version that had this fix built
1569	* in
1570	*/
1571	if ((((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) ||
1572	(buf[0] & 0xfff) >= 0x190) {
1573	a6xx_gpu->has_whereami = true;
1574	ret = true;
1575	goto out;
1576	}
1577
1578	DRM_DEV_ERROR(&gpu->pdev->dev,
1579	"a630 SQE ucode is too old. Have version %x need at least %x\n",
1580	buf[0] & 0xfff, 0x190);
1581	} else if (!strcmp(sqe_name, "a650_sqe.fw")) {
1582	if ((buf[0] & 0xfff) >= 0x095) {
1583	ret = true;
1584	goto out;
1585	}
1586
1587	DRM_DEV_ERROR(&gpu->pdev->dev,
1588	"a650 SQE ucode is too old. Have version %x need at least %x\n",
1589	buf[0] & 0xfff, 0x095);
1590	} else if (!strcmp(sqe_name, "a660_sqe.fw")) {
1591	ret = true;
1592	} else {
1593	DRM_DEV_ERROR(&gpu->pdev->dev,
1594	"unknown GPU, add it to a6xx_ucode_check_version()!!\n");
1595	}
1596	out:
1597	msm_gem_put_vaddr(obj);
1598	return ret;
1599	}
1600
1601	static int a6xx_ucode_load(struct msm_gpu *gpu)
1602	{
1603	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1604	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1605
1606	if (!a6xx_gpu->sqe_bo) {
1607	a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
1608	fw: adreno_gpu->fw[ADRENO_FW_SQE], iova: &a6xx_gpu->sqe_iova);
1609
1610	if (IS_ERR(ptr: a6xx_gpu->sqe_bo)) {
1611	int ret = PTR_ERR(ptr: a6xx_gpu->sqe_bo);
1612
1613	a6xx_gpu->sqe_bo = NULL;
1614	DRM_DEV_ERROR(&gpu->pdev->dev,
1615	"Could not allocate SQE ucode: %d\n", ret);
1616
1617	return ret;
1618	}
1619
1620	msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw");
1621	if (!a6xx_ucode_check_version(a6xx_gpu, obj: a6xx_gpu->sqe_bo)) {
1622	msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
1623	drm_gem_object_put(a6xx_gpu->sqe_bo);
1624
1625	a6xx_gpu->sqe_bo = NULL;
1626	return -EPERM;
1627	}
1628	}
1629
1630	/*
1631	* Expanded APRIV and targets that support WHERE_AM_I both need a
1632	* privileged buffer to store the RPTR shadow
1633	*/
1634	if ((adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) &&
1635	!a6xx_gpu->shadow_bo) {
1636	a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
1637	sizeof(u32) * gpu->nr_rings,
1638	MSM_BO_WC | MSM_BO_MAP_PRIV,
1639	gpu->aspace, &a6xx_gpu->shadow_bo,
1640	&a6xx_gpu->shadow_iova);
1641
1642	if (IS_ERR(ptr: a6xx_gpu->shadow))
1643	return PTR_ERR(ptr: a6xx_gpu->shadow);
1644
1645	msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");
1646	}
1647
1648	return 0;
1649	}
1650
1651	static int a6xx_zap_shader_init(struct msm_gpu *gpu)
1652	{
1653	static bool loaded;
1654	int ret;
1655
1656	if (loaded)
1657	return 0;
1658
1659	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
1660
1661	loaded = !ret;
1662	return ret;
1663	}
1664
1665	#define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
1666	A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
1667	A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
1668	A6XX_RBBM_INT_0_MASK_CP_IB2 | \
1669	A6XX_RBBM_INT_0_MASK_CP_IB1 | \
1670	A6XX_RBBM_INT_0_MASK_CP_RB | \
1671	A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
1672	A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
1673	A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
1674	A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
1675	A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
1676
1677	#define A7XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
1678	A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
1679	A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR | \
1680	A6XX_RBBM_INT_0_MASK_CP_SW | \
1681	A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
1682	A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \
1683	A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \
1684	A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
1685	A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
1686	A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
1687	A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
1688	A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR | \
1689	A6XX_RBBM_INT_0_MASK_TSBWRITEERROR)
1690
1691	#define A7XX_APRIV_MASK (A6XX_CP_APRIV_CNTL_ICACHE | \
1692	A6XX_CP_APRIV_CNTL_RBFETCH | \
1693	A6XX_CP_APRIV_CNTL_RBPRIVLEVEL | \
1694	A6XX_CP_APRIV_CNTL_RBRPWB)
1695
1696	#define A7XX_BR_APRIVMASK (A7XX_APRIV_MASK | \
1697	A6XX_CP_APRIV_CNTL_CDREAD | \
1698	A6XX_CP_APRIV_CNTL_CDWRITE)
1699
1700	static int hw_init(struct msm_gpu *gpu)
1701	{
1702	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1703	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1704	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1705	u64 gmem_range_min;
1706	int ret;
1707
1708	if (!adreno_has_gmu_wrapper(gpu: adreno_gpu)) {
1709	/* Make sure the GMU keeps the GPU on while we set it up */
1710	ret = a6xx_gmu_set_oob(gmu: &a6xx_gpu->gmu, state: GMU_OOB_GPU_SET);
1711	if (ret)
1712	return ret;
1713	}
1714
1715	/* Clear GBIF halt in case GX domain was not collapsed */
1716	if (adreno_is_a619_holi(gpu: adreno_gpu)) {
1717	gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
1718	gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, 0);
1719	/* Let's make extra sure that the GPU can access the memory.. */
1720	mb();
1721	} else if (a6xx_has_gbif(gpu: adreno_gpu)) {
1722	gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
1723	gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0);
1724	/* Let's make extra sure that the GPU can access the memory.. */
1725	mb();
1726	}
1727
1728	/* Some GPUs are stubborn and take their sweet time to unhalt GBIF! */
1729	if (adreno_is_a7xx(gpu: adreno_gpu) && a6xx_has_gbif(gpu: adreno_gpu))
1730	spin_until(!gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK));
1731
1732	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
1733
1734	if (adreno_is_a619_holi(gpu: adreno_gpu))
1735	a6xx_sptprac_enable(gmu);
1736
1737	/*
1738	* Disable the trusted memory range - we don't actually supported secure
1739	* memory rendering at this point in time and we don't want to block off
1740	* part of the virtual memory space.
1741	*/
1742	gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000);
1743	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
1744
1745	if (!adreno_is_a7xx(gpu: adreno_gpu)) {
1746	/* Turn on 64 bit addressing for all blocks */
1747	gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1);
1748	gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1);
1749	gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1);
1750	gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1);
1751	gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1);
1752	gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1);
1753	gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1);
1754	gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1);
1755	gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1);
1756	gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1);
1757	gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1);
1758	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
1759	}
1760
1761	/* enable hardware clockgating */
1762	a6xx_set_hwcg(gpu, state: true);
1763
1764	/* VBIF/GBIF start*/
1765	if (adreno_is_a610_family(gpu: adreno_gpu) ||
1766	adreno_is_a640_family(gpu: adreno_gpu) ||
1767	adreno_is_a650_family(gpu: adreno_gpu) ||
1768	adreno_is_a7xx(gpu: adreno_gpu)) {
1769	gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620);
1770	gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620);
1771	gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620);
1772	gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
1773	gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL,
1774	adreno_is_a7xx(gpu: adreno_gpu) ? 0x2120212 : 0x3);
1775	} else {
1776	gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
1777	}
1778
1779	if (adreno_is_a630(gpu: adreno_gpu))
1780	gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
1781
1782	if (adreno_is_a7xx(gpu: adreno_gpu))
1783	gpu_write(gpu, REG_A6XX_UCHE_GBIF_GX_CONFIG, 0x10240e0);
1784
1785	/* Make all blocks contribute to the GPU BUSY perf counter */
1786	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
1787
1788	/* Disable L2 bypass in the UCHE */
1789	if (adreno_is_a7xx(gpu: adreno_gpu)) {
1790	gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
1791	gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
1792	} else {
1793	gpu_write64(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX, 0x0001ffffffffffc0llu);
1794	gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
1795	gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
1796	}
1797
1798	if (!(adreno_is_a650_family(gpu: adreno_gpu) ||
1799	adreno_is_a702(gpu: adreno_gpu) ||
1800	adreno_is_a730(gpu: adreno_gpu))) {
1801	gmem_range_min = adreno_is_a740_family(adreno_gpu) ? SZ_16M : SZ_1M;
1802
1803	/* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
1804	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN, gmem_range_min);
1805
1806	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX,
1807	gmem_range_min + adreno_gpu->info->gmem - 1);
1808	}
1809
1810	if (adreno_is_a7xx(gpu: adreno_gpu))
1811	gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, BIT(23));
1812	else {
1813	gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
1814	gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
1815	}
1816
1817	if (adreno_is_a640_family(gpu: adreno_gpu) || adreno_is_a650_family(gpu: adreno_gpu)) {
1818	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
1819	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
1820	} else if (adreno_is_a610_family(gpu: adreno_gpu)) {
1821	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x00800060);
1822	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x40201b16);
1823	} else if (!adreno_is_a7xx(gpu: adreno_gpu)) {
1824	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
1825	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
1826	}
1827
1828	if (adreno_is_a660_family(gpu: adreno_gpu))
1829	gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020);
1830
1831	/* Setting the mem pool size */
1832	if (adreno_is_a610(gpu: adreno_gpu)) {
1833	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 48);
1834	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 47);
1835	} else if (adreno_is_a702(gpu: adreno_gpu)) {
1836	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 64);
1837	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 63);
1838	} else if (!adreno_is_a7xx(gpu: adreno_gpu))
1839	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
1840
1841	/* Setting the primFifo thresholds default values,
1842	* and vccCacheSkipDis=1 bit (0x200) for A640 and newer
1843	*/
1844	if (adreno_is_a702(gpu: adreno_gpu))
1845	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x0000c000);
1846	else if (adreno_is_a690(gpu: adreno_gpu))
1847	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00800200);
1848	else if (adreno_is_a650(gpu: adreno_gpu) || adreno_is_a660(gpu: adreno_gpu))
1849	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
1850	else if (adreno_is_a640_family(gpu: adreno_gpu) || adreno_is_7c3(gpu: adreno_gpu))
1851	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200);
1852	else if (adreno_is_a650(gpu: adreno_gpu) || adreno_is_a660(gpu: adreno_gpu))
1853	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
1854	else if (adreno_is_a619(gpu: adreno_gpu))
1855	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00018000);
1856	else if (adreno_is_a610(gpu: adreno_gpu))
1857	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00080000);
1858	else if (!adreno_is_a7xx(gpu: adreno_gpu))
1859	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000);
1860
1861	/* Set the AHB default slave response to "ERROR" */
1862	gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
1863
1864	/* Turn on performance counters */
1865	gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
1866
1867	if (adreno_is_a7xx(gpu: adreno_gpu)) {
1868	/* Turn on the IFPC counter (countable 4 on XOCLK4) */
1869	gmu_write(gmu: &a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1,
1870	FIELD_PREP(GENMASK(7, 0), 0x4));
1871	}
1872
1873	/* Select CP0 to always count cycles */
1874	gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL(0), PERF_CP_ALWAYS_COUNT);
1875
1876	a6xx_set_ubwc_config(gpu);
1877
1878	/* Enable fault detection */
1879	if (adreno_is_a730(gpu: adreno_gpu) ||
1880	adreno_is_a740_family(gpu: adreno_gpu))
1881	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0xcfffff);
1882	else if (adreno_is_a690(gpu: adreno_gpu))
1883	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x4fffff);
1884	else if (adreno_is_a619(gpu: adreno_gpu))
1885	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3fffff);
1886	else if (adreno_is_a610(gpu: adreno_gpu) || adreno_is_a702(gpu: adreno_gpu))
1887	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3ffff);
1888	else
1889	gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x1fffff);
1890
1891	gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, BIT(7) | 0x1);
1892
1893	/* Set weights for bicubic filtering */
1894	if (adreno_is_a650_family(gpu: adreno_gpu)) {
1895	gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0);
1896	gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1,
1897	0x3fe05ff4);
1898	gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2,
1899	0x3fa0ebee);
1900	gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3,
1901	0x3f5193ed);
1902	gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4,
1903	0x3f0243f0);
1904	}
1905
1906	/* Set up the CX GMU counter 0 to count busy ticks */
1907	gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, value: 0xff000000);
1908
1909	/* Enable the power counter */
1910	gmu_rmw(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, mask: 0xff, BIT(5));
1911	gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, value: 1);
1912
1913	/* Protect registers from the CP */
1914	a6xx_set_cp_protect(gpu);
1915
1916	if (adreno_is_a660_family(gpu: adreno_gpu)) {
1917	if (adreno_is_a690(gpu: adreno_gpu))
1918	gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x00028801);
1919	else
1920	gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1);
1921	gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0);
1922	} else if (adreno_is_a702(gpu: adreno_gpu)) {
1923	/* Something to do with the HLSQ cluster */
1924	gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, BIT(24));
1925	}
1926
1927	if (adreno_is_a690(gpu: adreno_gpu))
1928	gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x90);
1929	/* Set dualQ + disable afull for A660 GPU */
1930	else if (adreno_is_a660(gpu: adreno_gpu))
1931	gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906);
1932	else if (adreno_is_a7xx(gpu: adreno_gpu))
1933	gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG,
1934	FIELD_PREP(GENMASK(19, 16), 6) |
1935	FIELD_PREP(GENMASK(15, 12), 6) |
1936	FIELD_PREP(GENMASK(11, 8), 9) |
1937	BIT(3) | BIT(2) |
1938	FIELD_PREP(GENMASK(1, 0), 2));
1939
1940	/* Enable expanded apriv for targets that support it */
1941	if (gpu->hw_apriv) {
1942	if (adreno_is_a7xx(gpu: adreno_gpu)) {
1943	gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1944	A7XX_BR_APRIVMASK);
1945	gpu_write(gpu, REG_A7XX_CP_BV_APRIV_CNTL,
1946	A7XX_APRIV_MASK);
1947	gpu_write(gpu, REG_A7XX_CP_LPAC_APRIV_CNTL,
1948	A7XX_APRIV_MASK);
1949	} else
1950	gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1951	BIT(6) | BIT(5) | BIT(3) | BIT(2) | BIT(1));
1952	}
1953
1954	/* Enable interrupts */
1955	gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK,
1956	adreno_is_a7xx(gpu: adreno_gpu) ? A7XX_INT_MASK : A6XX_INT_MASK);
1957
1958	ret = adreno_hw_init(gpu);
1959	if (ret)
1960	goto out;
1961
1962	gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova);
1963
1964	/* Set the ringbuffer address */
1965	gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova);
1966
1967	/* Targets that support extended APRIV can use the RPTR shadow from
1968	* hardware but all the other ones need to disable the feature. Targets
1969	* that support the WHERE_AM_I opcode can use that instead
1970	*/
1971	if (adreno_gpu->base.hw_apriv)
1972	gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT);
1973	else
1974	gpu_write(gpu, REG_A6XX_CP_RB_CNTL,
1975	MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
1976
1977	/* Configure the RPTR shadow if needed: */
1978	if (a6xx_gpu->shadow_bo) {
1979	gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR,
1980	shadowptr(a6xx_gpu, gpu->rb[0]));
1981	}
1982
1983	/* ..which means "always" on A7xx, also for BV shadow */
1984	if (adreno_is_a7xx(gpu: adreno_gpu)) {
1985	gpu_write64(gpu, REG_A7XX_CP_BV_RB_RPTR_ADDR,
1986	rbmemptr(gpu->rb[0], bv_fence));
1987	}
1988
1989	/* Always come up on rb 0 */
1990	a6xx_gpu->cur_ring = gpu->rb[0];
1991
1992	gpu->cur_ctx_seqno = 0;
1993
1994	/* Enable the SQE_to start the CP engine */
1995	gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
1996
1997	ret = adreno_is_a7xx(gpu: adreno_gpu) ? a7xx_cp_init(gpu) : a6xx_cp_init(gpu);
1998	if (ret)
1999	goto out;
2000
2001	/*
2002	* Try to load a zap shader into the secure world. If successful
2003	* we can use the CP to switch out of secure mode. If not then we
2004	* have no resource but to try to switch ourselves out manually. If we
2005	* guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
2006	* be blocked and a permissions violation will soon follow.
2007	*/
2008	ret = a6xx_zap_shader_init(gpu);
2009	if (!ret) {
2010	OUT_PKT7(ring: gpu->rb[0], opcode: CP_SET_SECURE_MODE, cnt: 1);
2011	OUT_RING(gpu->rb[0], 0x00000000);
2012
2013	a6xx_flush(gpu, ring: gpu->rb[0]);
2014	if (!a6xx_idle(gpu, ring: gpu->rb[0]))
2015	return -EINVAL;
2016	} else if (ret == -ENODEV) {
2017	/*
2018	* This device does not use zap shader (but print a warning
2019	* just in case someone got their dt wrong.. hopefully they
2020	* have a debug UART to realize the error of their ways...
2021	* if you mess this up you are about to crash horribly)
2022	*/
2023	dev_warn_once(gpu->dev->dev,
2024	"Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
2025	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
2026	ret = 0;
2027	} else {
2028	return ret;
2029	}
2030
2031	out:
2032	if (adreno_has_gmu_wrapper(gpu: adreno_gpu))
2033	return ret;
2034	/*
2035	* Tell the GMU that we are done touching the GPU and it can start power
2036	* management
2037	*/
2038	a6xx_gmu_clear_oob(gmu: &a6xx_gpu->gmu, state: GMU_OOB_GPU_SET);
2039
2040	if (a6xx_gpu->gmu.legacy) {
2041	/* Take the GMU out of its special boot mode */
2042	a6xx_gmu_clear_oob(gmu: &a6xx_gpu->gmu, state: GMU_OOB_BOOT_SLUMBER);
2043	}
2044
2045	return ret;
2046	}
2047
2048	static int a6xx_hw_init(struct msm_gpu *gpu)
2049	{
2050	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2051	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2052	int ret;
2053
2054	mutex_lock(&a6xx_gpu->gmu.lock);
2055	ret = hw_init(gpu);
2056	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2057
2058	return ret;
2059	}
2060
2061	static void a6xx_dump(struct msm_gpu *gpu)
2062	{
2063	DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
2064	gpu_read(gpu, REG_A6XX_RBBM_STATUS));
2065	adreno_dump(gpu);
2066	}
2067
2068	static void a6xx_recover(struct msm_gpu *gpu)
2069	{
2070	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2071	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2072	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2073	int i, active_submits;
2074
2075	adreno_dump_info(gpu);
2076
2077	for (i = 0; i < 8; i++)
2078	DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
2079	gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i0: i)));
2080
2081	if (hang_debug)
2082	a6xx_dump(gpu);
2083
2084	/*
2085	* To handle recovery specific sequences during the rpm suspend we are
2086	* about to trigger
2087	*/
2088	a6xx_gpu->hung = true;
2089
2090	/* Halt SQE first */
2091	gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3);
2092
2093	pm_runtime_dont_use_autosuspend(&gpu->pdev->dev);
2094
2095	/* active_submit won't change until we make a submission */
2096	mutex_lock(&gpu->active_lock);
2097	active_submits = gpu->active_submits;
2098
2099	/*
2100	* Temporarily clear active_submits count to silence a WARN() in the
2101	* runtime suspend cb
2102	*/
2103	gpu->active_submits = 0;
2104
2105	if (adreno_has_gmu_wrapper(gpu: adreno_gpu)) {
2106	/* Drain the outstanding traffic on memory buses */
2107	a6xx_bus_clear_pending_transactions(adreno_gpu, gx_off: true);
2108
2109	/* Reset the GPU to a clean state */
2110	a6xx_gpu_sw_reset(gpu, assert: true);
2111	a6xx_gpu_sw_reset(gpu, assert: false);
2112	}
2113
2114	reinit_completion(x: &gmu->pd_gate);
2115	dev_pm_genpd_add_notifier(dev: gmu->cxpd, nb: &gmu->pd_nb);
2116	dev_pm_genpd_synced_poweroff(dev: gmu->cxpd);
2117
2118	/* Drop the rpm refcount from active submits */
2119	if (active_submits)
2120	pm_runtime_put(&gpu->pdev->dev);
2121
2122	/* And the final one from recover worker */
2123	pm_runtime_put_sync(&gpu->pdev->dev);
2124
2125	if (!wait_for_completion_timeout(x: &gmu->pd_gate, timeout: msecs_to_jiffies(m: 1000)))
2126	DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n");
2127
2128	dev_pm_genpd_remove_notifier(dev: gmu->cxpd);
2129
2130	pm_runtime_use_autosuspend(&gpu->pdev->dev);
2131
2132	if (active_submits)
2133	pm_runtime_get(&gpu->pdev->dev);
2134
2135	pm_runtime_get_sync(&gpu->pdev->dev);
2136
2137	gpu->active_submits = active_submits;
2138	mutex_unlock(lock: &gpu->active_lock);
2139
2140	msm_gpu_hw_init(gpu);
2141	a6xx_gpu->hung = false;
2142	}
2143
2144	static const char *a6xx_uche_fault_block(struct msm_gpu *gpu, u32 mid)
2145	{
2146	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2147	static const char *uche_clients[7] = {
2148	"VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ",
2149	};
2150	u32 val;
2151
2152	if (adreno_is_a7xx(gpu: adreno_gpu)) {
2153	if (mid != 1 && mid != 2 && mid != 3 && mid != 8)
2154	return "UNKNOWN";
2155	} else {
2156	if (mid < 1 || mid > 3)
2157	return "UNKNOWN";
2158	}
2159
2160	/*
2161	* The source of the data depends on the mid ID read from FSYNR1.
2162	* and the client ID read from the UCHE block
2163	*/
2164	val = gpu_read(gpu, REG_A6XX_UCHE_CLIENT_PF);
2165
2166	if (adreno_is_a7xx(gpu: adreno_gpu)) {
2167	/* Bit 3 for mid=3 indicates BR or BV */
2168	static const char *uche_clients_a7xx[16] = {
2169	"BR_VFD", "BR_SP", "BR_VSC", "BR_VPC",
2170	"BR_HLSQ", "BR_PC", "BR_LRZ", "BR_TP",
2171	"BV_VFD", "BV_SP", "BV_VSC", "BV_VPC",
2172	"BV_HLSQ", "BV_PC", "BV_LRZ", "BV_TP",
2173	};
2174
2175	/* LPAC has the same clients as BR and BV, but because it is
2176	* compute-only some of them do not exist and there are holes
2177	* in the array.
2178	*/
2179	static const char *uche_clients_lpac_a7xx[8] = {
2180	"-", "LPAC_SP", "-", "-",
2181	"LPAC_HLSQ", "-", "-", "LPAC_TP",
2182	};
2183
2184	val &= GENMASK(6, 0);
2185
2186	/* mid=3 refers to BR or BV */
2187	if (mid == 3) {
2188	if (val < ARRAY_SIZE(uche_clients_a7xx))
2189	return uche_clients_a7xx[val];
2190	else
2191	return "UCHE";
2192	}
2193
2194	/* mid=8 refers to LPAC */
2195	if (mid == 8) {
2196	if (val < ARRAY_SIZE(uche_clients_lpac_a7xx))
2197	return uche_clients_lpac_a7xx[val];
2198	else
2199	return "UCHE_LPAC";
2200	}
2201
2202	/* mid=2 is a catchall for everything else in LPAC */
2203	if (mid == 2)
2204	return "UCHE_LPAC";
2205
2206	/* mid=1 is a catchall for everything else in BR/BV */
2207	return "UCHE";
2208	} else if (adreno_is_a660_family(gpu: adreno_gpu)) {
2209	static const char *uche_clients_a660[8] = {
2210	"VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ", "TP",
2211	};
2212
2213	static const char *uche_clients_a660_not[8] = {
2214	"not VFD", "not SP", "not VSC", "not VPC",
2215	"not HLSQ", "not PC", "not LRZ", "not TP",
2216	};
2217
2218	val &= GENMASK(6, 0);
2219
2220	if (mid == 3 && val < ARRAY_SIZE(uche_clients_a660))
2221	return uche_clients_a660[val];
2222
2223	if (mid == 1 && val < ARRAY_SIZE(uche_clients_a660_not))
2224	return uche_clients_a660_not[val];
2225
2226	return "UCHE";
2227	} else {
2228	/* mid = 3 is most precise and refers to only one block per client */
2229	if (mid == 3)
2230	return uche_clients[val & 7];
2231
2232	/* For mid=2 the source is TP or VFD except when the client id is 0 */
2233	if (mid == 2)
2234	return ((val & 7) == 0) ? "TP" : "TP|VFD";
2235
2236	/* For mid=1 just return "UCHE" as a catchall for everything else */
2237	return "UCHE";
2238	}
2239	}
2240
2241	static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id)
2242	{
2243	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2244
2245	if (id == 0)
2246	return "CP";
2247	else if (id == 4)
2248	return "CCU";
2249	else if (id == 6)
2250	return "CDP Prefetch";
2251	else if (id == 7)
2252	return "GMU";
2253	else if (id == 5 && adreno_is_a7xx(gpu: adreno_gpu))
2254	return "Flag cache";
2255
2256	return a6xx_uche_fault_block(gpu, mid: id);
2257	}
2258
2259	static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
2260	{
2261	struct msm_gpu *gpu = arg;
2262	struct adreno_smmu_fault_info *info = data;
2263	const char *block = "unknown";
2264
2265	u32 scratch[] = {
2266	gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i0: 4)),
2267	gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i0: 5)),
2268	gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i0: 6)),
2269	gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i0: 7)),
2270	};
2271
2272	if (info)
2273	block = a6xx_fault_block(gpu, id: info->fsynr1 & 0xff);
2274
2275	return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
2276	}
2277
2278	static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
2279	{
2280	u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
2281
2282	if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
2283	u32 val;
2284
2285	gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
2286	val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
2287	dev_err_ratelimited(&gpu->pdev->dev,
2288	"CP | opcode error | possible opcode=0x%8.8X\n",
2289	val);
2290	}
2291
2292	if (status & A6XX_CP_INT_CP_UCODE_ERROR)
2293	dev_err_ratelimited(&gpu->pdev->dev,
2294	"CP ucode error interrupt\n");
2295
2296	if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
2297	dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
2298	gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
2299
2300	if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
2301	u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
2302
2303	dev_err_ratelimited(&gpu->pdev->dev,
2304	"CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
2305	val & (1 << 20) ? "READ" : "WRITE",
2306	(val & 0x3ffff), val);
2307	}
2308
2309	if (status & A6XX_CP_INT_CP_AHB_ERROR && !adreno_is_a7xx(to_adreno_gpu(gpu)))
2310	dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
2311
2312	if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
2313	dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
2314
2315	if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
2316	dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
2317
2318	}
2319
2320	static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
2321	{
2322	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2323	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2324	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
2325
2326	/*
2327	* If stalled on SMMU fault, we could trip the GPU's hang detection,
2328	* but the fault handler will trigger the devcore dump, and we want
2329	* to otherwise resume normally rather than killing the submit, so
2330	* just bail.
2331	*/
2332	if (gpu_read(gpu, REG_A6XX_RBBM_STATUS3) & A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT)
2333	return;
2334
2335	/*
2336	* Force the GPU to stay on until after we finish
2337	* collecting information
2338	*/
2339	if (!adreno_has_gmu_wrapper(gpu: adreno_gpu))
2340	gmu_write(gmu: &a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, value: 1);
2341
2342	DRM_DEV_ERROR(&gpu->pdev->dev,
2343	"gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
2344	ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
2345	gpu_read(gpu, REG_A6XX_RBBM_STATUS),
2346	gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
2347	gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
2348	gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
2349	gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
2350	gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
2351	gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
2352
2353	/* Turn off the hangcheck timer to keep it from bothering us */
2354	del_timer(timer: &gpu->hangcheck_timer);
2355
2356	kthread_queue_work(gpu->worker, &gpu->recover_work);
2357	}
2358
2359	static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
2360	{
2361	struct msm_drm_private *priv = gpu->dev->dev_private;
2362	u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
2363
2364	gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
2365
2366	if (priv->disable_err_irq)
2367	status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
2368
2369	if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
2370	a6xx_fault_detect_irq(gpu);
2371
2372	if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
2373	dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
2374
2375	if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
2376	a6xx_cp_hw_err_irq(gpu);
2377
2378	if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
2379	dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
2380
2381	if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
2382	dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
2383
2384	if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
2385	dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
2386
2387	if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
2388	msm_gpu_retire(gpu);
2389
2390	return IRQ_HANDLED;
2391	}
2392
2393	static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu)
2394	{
2395	llcc_slice_deactivate(desc: a6xx_gpu->llc_slice);
2396	llcc_slice_deactivate(desc: a6xx_gpu->htw_llc_slice);
2397	}
2398
2399	static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
2400	{
2401	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2402	struct msm_gpu *gpu = &adreno_gpu->base;
2403	u32 cntl1_regval = 0;
2404
2405	if (IS_ERR(ptr: a6xx_gpu->llc_mmio))
2406	return;
2407
2408	if (!llcc_slice_activate(desc: a6xx_gpu->llc_slice)) {
2409	u32 gpu_scid = llcc_get_slice_id(desc: a6xx_gpu->llc_slice);
2410
2411	gpu_scid &= 0x1f;
2412	cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) |
2413	(gpu_scid << 15) | (gpu_scid << 20);
2414
2415	/* On A660, the SCID programming for UCHE traffic is done in
2416	* A6XX_GBIF_SCACHE_CNTL0[14:10]
2417	*/
2418	if (adreno_is_a660_family(gpu: adreno_gpu))
2419	gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
2420	(1 << 8), (gpu_scid << 10) | (1 << 8));
2421	}
2422
2423	/*
2424	* For targets with a MMU500, activate the slice but don't program the
2425	* register. The XBL will take care of that.
2426	*/
2427	if (!llcc_slice_activate(desc: a6xx_gpu->htw_llc_slice)) {
2428	if (!a6xx_gpu->have_mmu500) {
2429	u32 gpuhtw_scid = llcc_get_slice_id(desc: a6xx_gpu->htw_llc_slice);
2430
2431	gpuhtw_scid &= 0x1f;
2432	cntl1_regval |= FIELD_PREP(GENMASK(29, 25), gpuhtw_scid);
2433	}
2434	}
2435
2436	if (!cntl1_regval)
2437	return;
2438
2439	/*
2440	* Program the slice IDs for the various GPU blocks and GPU MMU
2441	* pagetables
2442	*/
2443	if (!a6xx_gpu->have_mmu500) {
2444	a6xx_llc_write(a6xx_gpu,
2445	REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, value: cntl1_regval);
2446
2447	/*
2448	* Program cacheability overrides to not allocate cache
2449	* lines on a write miss
2450	*/
2451	a6xx_llc_rmw(a6xx_gpu,
2452	REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, mask: 0xF, or: 0x03);
2453	return;
2454	}
2455
2456	gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval);
2457	}
2458
2459	static void a7xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
2460	{
2461	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2462	struct msm_gpu *gpu = &adreno_gpu->base;
2463
2464	if (IS_ERR(ptr: a6xx_gpu->llc_mmio))
2465	return;
2466
2467	if (!llcc_slice_activate(desc: a6xx_gpu->llc_slice)) {
2468	u32 gpu_scid = llcc_get_slice_id(desc: a6xx_gpu->llc_slice);
2469
2470	gpu_scid &= GENMASK(4, 0);
2471
2472	gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL1,
2473	FIELD_PREP(GENMASK(29, 25), gpu_scid) |
2474	FIELD_PREP(GENMASK(24, 20), gpu_scid) |
2475	FIELD_PREP(GENMASK(19, 15), gpu_scid) |
2476	FIELD_PREP(GENMASK(14, 10), gpu_scid) |
2477	FIELD_PREP(GENMASK(9, 5), gpu_scid) |
2478	FIELD_PREP(GENMASK(4, 0), gpu_scid));
2479
2480	gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL0,
2481	FIELD_PREP(GENMASK(14, 10), gpu_scid) |
2482	BIT(8));
2483	}
2484
2485	llcc_slice_activate(desc: a6xx_gpu->htw_llc_slice);
2486	}
2487
2488	static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)
2489	{
2490	/* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */
2491	if (adreno_has_gmu_wrapper(gpu: &a6xx_gpu->base))
2492	return;
2493
2494	llcc_slice_putd(desc: a6xx_gpu->llc_slice);
2495	llcc_slice_putd(desc: a6xx_gpu->htw_llc_slice);
2496	}
2497
2498	static void a6xx_llc_slices_init(struct platform_device *pdev,
2499	struct a6xx_gpu *a6xx_gpu, bool is_a7xx)
2500	{
2501	struct device_node *phandle;
2502
2503	/* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */
2504	if (adreno_has_gmu_wrapper(gpu: &a6xx_gpu->base))
2505	return;
2506
2507	/*
2508	* There is a different programming path for A6xx targets with an
2509	* mmu500 attached, so detect if that is the case
2510	*/
2511	phandle = of_parse_phandle(np: pdev->dev.of_node, phandle_name: "iommus", index: 0);
2512	a6xx_gpu->have_mmu500 = (phandle &&
2513	of_device_is_compatible(device: phandle, "arm,mmu-500"));
2514	of_node_put(node: phandle);
2515
2516	if (is_a7xx || !a6xx_gpu->have_mmu500)
2517	a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem");
2518	else
2519	a6xx_gpu->llc_mmio = NULL;
2520
2521	a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU);
2522	a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW);
2523
2524	if (IS_ERR_OR_NULL(ptr: a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(ptr: a6xx_gpu->htw_llc_slice))
2525	a6xx_gpu->llc_mmio = ERR_PTR(error: -EINVAL);
2526	}
2527
2528	#define GBIF_CLIENT_HALT_MASK BIT(0)
2529	#define GBIF_ARB_HALT_MASK BIT(1)
2530	#define VBIF_XIN_HALT_CTRL0_MASK GENMASK(3, 0)
2531	#define VBIF_RESET_ACK_MASK 0xF0
2532	#define GPR0_GBIF_HALT_REQUEST 0x1E0
2533
2534	void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off)
2535	{
2536	struct msm_gpu *gpu = &adreno_gpu->base;
2537
2538	if (adreno_is_a619_holi(gpu: adreno_gpu)) {
2539	gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, GPR0_GBIF_HALT_REQUEST);
2540	spin_until((gpu_read(gpu, REG_A6XX_RBBM_VBIF_GX_RESET_STATUS) &
2541	(VBIF_RESET_ACK_MASK)) == VBIF_RESET_ACK_MASK);
2542	} else if (!a6xx_has_gbif(gpu: adreno_gpu)) {
2543	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, VBIF_XIN_HALT_CTRL0_MASK);
2544	spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) &
2545	(VBIF_XIN_HALT_CTRL0_MASK)) == VBIF_XIN_HALT_CTRL0_MASK);
2546	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
2547
2548	return;
2549	}
2550
2551	if (gx_off) {
2552	/* Halt the gx side of GBIF */
2553	gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1);
2554	spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1);
2555	}
2556
2557	/* Halt new client requests on GBIF */
2558	gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
2559	spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
2560	(GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
2561
2562	/* Halt all AXI requests on GBIF */
2563	gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
2564	spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
2565	(GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
2566
2567	/* The GBIF halt needs to be explicitly cleared */
2568	gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
2569	}
2570
2571	void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert)
2572	{
2573	/* 11nm chips (e.g. ones with A610) have hw issues with the reset line! */
2574	if (adreno_is_a610(to_adreno_gpu(gpu)))
2575	return;
2576
2577	gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, assert);
2578	/* Perform a bogus read and add a brief delay to ensure ordering. */
2579	gpu_read(gpu, REG_A6XX_RBBM_SW_RESET_CMD);
2580	udelay(1);
2581
2582	/* The reset line needs to be asserted for at least 100 us */
2583	if (assert)
2584	udelay(100);
2585	}
2586
2587	static int a6xx_gmu_pm_resume(struct msm_gpu *gpu)
2588	{
2589	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2590	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2591	int ret;
2592
2593	gpu->needs_hw_init = true;
2594
2595	trace_msm_gpu_resume(0);
2596
2597	mutex_lock(&a6xx_gpu->gmu.lock);
2598	ret = a6xx_gmu_resume(gpu: a6xx_gpu);
2599	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2600	if (ret)
2601	return ret;
2602
2603	msm_devfreq_resume(gpu);
2604
2605	adreno_is_a7xx(gpu: adreno_gpu) ? a7xx_llc_activate(a6xx_gpu) : a6xx_llc_activate(a6xx_gpu);
2606
2607	return ret;
2608	}
2609
2610	static int a6xx_pm_resume(struct msm_gpu *gpu)
2611	{
2612	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2613	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2614	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2615	unsigned long freq = gpu->fast_rate;
2616	struct dev_pm_opp *opp;
2617	int ret;
2618
2619	gpu->needs_hw_init = true;
2620
2621	trace_msm_gpu_resume(0);
2622
2623	mutex_lock(&a6xx_gpu->gmu.lock);
2624
2625	opp = dev_pm_opp_find_freq_ceil(dev: &gpu->pdev->dev, freq: &freq);
2626	if (IS_ERR(ptr: opp)) {
2627	ret = PTR_ERR(ptr: opp);
2628	goto err_set_opp;
2629	}
2630	dev_pm_opp_put(opp);
2631
2632	/* Set the core clock and bus bw, having VDD scaling in mind */
2633	dev_pm_opp_set_opp(dev: &gpu->pdev->dev, opp);
2634
2635	pm_runtime_resume_and_get(gmu->dev);
2636	pm_runtime_resume_and_get(gmu->gxpd);
2637
2638	ret = clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks);
2639	if (ret)
2640	goto err_bulk_clk;
2641
2642	if (adreno_is_a619_holi(gpu: adreno_gpu))
2643	a6xx_sptprac_enable(gmu);
2644
2645	/* If anything goes south, tear the GPU down piece by piece.. */
2646	if (ret) {
2647	err_bulk_clk:
2648	pm_runtime_put(gmu->gxpd);
2649	pm_runtime_put(gmu->dev);
2650	dev_pm_opp_set_opp(dev: &gpu->pdev->dev, NULL);
2651	}
2652	err_set_opp:
2653	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2654
2655	if (!ret)
2656	msm_devfreq_resume(gpu);
2657
2658	return ret;
2659	}
2660
2661	static int a6xx_gmu_pm_suspend(struct msm_gpu *gpu)
2662	{
2663	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2664	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2665	int i, ret;
2666
2667	trace_msm_gpu_suspend(0);
2668
2669	a6xx_llc_deactivate(a6xx_gpu);
2670
2671	msm_devfreq_suspend(gpu);
2672
2673	mutex_lock(&a6xx_gpu->gmu.lock);
2674	ret = a6xx_gmu_stop(gpu: a6xx_gpu);
2675	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2676	if (ret)
2677	return ret;
2678
2679	if (a6xx_gpu->shadow_bo)
2680	for (i = 0; i < gpu->nr_rings; i++)
2681	a6xx_gpu->shadow[i] = 0;
2682
2683	gpu->suspend_count++;
2684
2685	return 0;
2686	}
2687
2688	static int a6xx_pm_suspend(struct msm_gpu *gpu)
2689	{
2690	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2691	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2692	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2693	int i;
2694
2695	trace_msm_gpu_suspend(0);
2696
2697	msm_devfreq_suspend(gpu);
2698
2699	mutex_lock(&a6xx_gpu->gmu.lock);
2700
2701	/* Drain the outstanding traffic on memory buses */
2702	a6xx_bus_clear_pending_transactions(adreno_gpu, gx_off: true);
2703
2704	if (adreno_is_a619_holi(gpu: adreno_gpu))
2705	a6xx_sptprac_disable(gmu);
2706
2707	clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks);
2708
2709	pm_runtime_put_sync(gmu->gxpd);
2710	dev_pm_opp_set_opp(dev: &gpu->pdev->dev, NULL);
2711	pm_runtime_put_sync(gmu->dev);
2712
2713	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2714
2715	if (a6xx_gpu->shadow_bo)
2716	for (i = 0; i < gpu->nr_rings; i++)
2717	a6xx_gpu->shadow[i] = 0;
2718
2719	gpu->suspend_count++;
2720
2721	return 0;
2722	}
2723
2724	static int a6xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
2725	{
2726	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2727	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2728
2729	mutex_lock(&a6xx_gpu->gmu.lock);
2730
2731	/* Force the GPU power on so we can read this register */
2732	a6xx_gmu_set_oob(gmu: &a6xx_gpu->gmu, state: GMU_OOB_PERFCOUNTER_SET);
2733
2734	*value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
2735
2736	a6xx_gmu_clear_oob(gmu: &a6xx_gpu->gmu, state: GMU_OOB_PERFCOUNTER_SET);
2737
2738	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2739
2740	return 0;
2741	}
2742
2743	static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
2744	{
2745	*value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
2746	return 0;
2747	}
2748
2749	static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
2750	{
2751	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2752	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2753
2754	return a6xx_gpu->cur_ring;
2755	}
2756
2757	static void a6xx_destroy(struct msm_gpu *gpu)
2758	{
2759	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2760	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2761
2762	if (a6xx_gpu->sqe_bo) {
2763	msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
2764	drm_gem_object_put(a6xx_gpu->sqe_bo);
2765	}
2766
2767	if (a6xx_gpu->shadow_bo) {
2768	msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace);
2769	drm_gem_object_put(a6xx_gpu->shadow_bo);
2770	}
2771
2772	a6xx_llc_slices_destroy(a6xx_gpu);
2773
2774	a6xx_gmu_remove(a6xx_gpu);
2775
2776	adreno_gpu_cleanup(gpu: adreno_gpu);
2777
2778	kfree(objp: a6xx_gpu);
2779	}
2780
2781	static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
2782	{
2783	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2784	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2785	u64 busy_cycles;
2786
2787	/* 19.2MHz */
2788	*out_sample_rate = 19200000;
2789
2790	busy_cycles = gmu_read64(gmu: &a6xx_gpu->gmu,
2791	REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
2792	REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
2793
2794	return busy_cycles;
2795	}
2796
2797	static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
2798	bool suspended)
2799	{
2800	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2801	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2802
2803	mutex_lock(&a6xx_gpu->gmu.lock);
2804	a6xx_gmu_set_freq(gpu, opp, suspended);
2805	mutex_unlock(lock: &a6xx_gpu->gmu.lock);
2806	}
2807
2808	static struct msm_gem_address_space *
2809	a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
2810	{
2811	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2812	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2813	unsigned long quirks = 0;
2814
2815	/*
2816	* This allows GPU to set the bus attributes required to use system
2817	* cache on behalf of the iommu page table walker.
2818	*/
2819	if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice) &&
2820	!device_iommu_capable(&pdev->dev, IOMMU_CAP_CACHE_COHERENCY))
2821	quirks |= IO_PGTABLE_QUIRK_ARM_OUTER_WBWA;
2822
2823	return adreno_iommu_create_address_space(gpu, pdev, quirks);
2824	}
2825
2826	static struct msm_gem_address_space *
2827	a6xx_create_private_address_space(struct msm_gpu *gpu)
2828	{
2829	struct msm_mmu *mmu;
2830
2831	mmu = msm_iommu_pagetable_create(gpu->aspace->mmu);
2832
2833	if (IS_ERR(ptr: mmu))
2834	return ERR_CAST(ptr: mmu);
2835
2836	return msm_gem_address_space_create(mmu,
2837	"gpu", 0x100000000ULL,
2838	adreno_private_address_space_size(gpu));
2839	}
2840
2841	static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
2842	{
2843	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2844	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2845
2846	if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami)
2847	return a6xx_gpu->shadow[ring->id];
2848
2849	return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR);
2850	}
2851
2852	static bool a6xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
2853	{
2854	struct msm_cp_state cp_state = {
2855	.ib1_base = gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
2856	.ib2_base = gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
2857	.ib1_rem = gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
2858	.ib2_rem = gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE),
2859	};
2860	bool progress;
2861
2862	/*
2863	* Adjust the remaining data to account for what has already been
2864	* fetched from memory, but not yet consumed by the SQE.
2865	*
2866	* This is not *technically* correct, the amount buffered could
2867	* exceed the IB size due to hw prefetching ahead, but:
2868	*
2869	* (1) We aren't trying to find the exact position, just whether
2870	* progress has been made
2871	* (2) The CP_REG_TO_MEM at the end of a submit should be enough
2872	* to prevent prefetching into an unrelated submit. (And
2873	* either way, at some point the ROQ will be full.)
2874	*/
2875	cp_state.ib1_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB1) >> 16;
2876	cp_state.ib2_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB2) >> 16;
2877
2878	progress = !!memcmp(&cp_state, &ring->last_cp_state, sizeof(cp_state));
2879
2880	ring->last_cp_state = cp_state;
2881
2882	return progress;
2883	}
2884
2885	static u32 fuse_to_supp_hw(const struct adreno_info *info, u32 fuse)
2886	{
2887	if (!info->speedbins)
2888	return UINT_MAX;
2889
2890	for (int i = 0; info->speedbins[i].fuse != SHRT_MAX; i++)
2891	if (info->speedbins[i].fuse == fuse)
2892	return BIT(info->speedbins[i].speedbin);
2893
2894	return UINT_MAX;
2895	}
2896
2897	static int a6xx_set_supported_hw(struct device *dev, const struct adreno_info *info)
2898	{
2899	u32 supp_hw;
2900	u32 speedbin;
2901	int ret;
2902
2903	ret = adreno_read_speedbin(dev, speedbin: &speedbin);
2904	/*
2905	* -ENOENT means that the platform doesn't support speedbin which is
2906	* fine
2907	*/
2908	if (ret == -ENOENT) {
2909	return 0;
2910	} else if (ret) {
2911	dev_err_probe(dev, err: ret,
2912	fmt: "failed to read speed-bin. Some OPPs may not be supported by hardware\n");
2913	return ret;
2914	}
2915
2916	supp_hw = fuse_to_supp_hw(info, fuse: speedbin);
2917
2918	if (supp_hw == UINT_MAX) {
2919	DRM_DEV_ERROR(dev,
2920	"missing support for speed-bin: %u. Some OPPs may not be supported by hardware\n",
2921	speedbin);
2922	supp_hw = BIT(0); /* Default */
2923	}
2924
2925	ret = devm_pm_opp_set_supported_hw(dev, versions: &supp_hw, count: 1);
2926	if (ret)
2927	return ret;
2928
2929	return 0;
2930	}
2931
2932	static const struct adreno_gpu_funcs funcs = {
2933	.base = {
2934	.get_param = adreno_get_param,
2935	.set_param = adreno_set_param,
2936	.hw_init = a6xx_hw_init,
2937	.ucode_load = a6xx_ucode_load,
2938	.pm_suspend = a6xx_gmu_pm_suspend,
2939	.pm_resume = a6xx_gmu_pm_resume,
2940	.recover = a6xx_recover,
2941	.submit = a6xx_submit,
2942	.active_ring = a6xx_active_ring,
2943	.irq = a6xx_irq,
2944	.destroy = a6xx_destroy,
2945	#if defined(CONFIG_DRM_MSM_GPU_STATE)
2946	.show = a6xx_show,
2947	#endif
2948	.gpu_busy = a6xx_gpu_busy,
2949	.gpu_get_freq = a6xx_gmu_get_freq,
2950	.gpu_set_freq = a6xx_gpu_set_freq,
2951	#if defined(CONFIG_DRM_MSM_GPU_STATE)
2952	.gpu_state_get = a6xx_gpu_state_get,
2953	.gpu_state_put = a6xx_gpu_state_put,
2954	#endif
2955	.create_address_space = a6xx_create_address_space,
2956	.create_private_address_space = a6xx_create_private_address_space,
2957	.get_rptr = a6xx_get_rptr,
2958	.progress = a6xx_progress,
2959	},
2960	.get_timestamp = a6xx_gmu_get_timestamp,
2961	};
2962
2963	static const struct adreno_gpu_funcs funcs_gmuwrapper = {
2964	.base = {
2965	.get_param = adreno_get_param,
2966	.set_param = adreno_set_param,
2967	.hw_init = a6xx_hw_init,
2968	.ucode_load = a6xx_ucode_load,
2969	.pm_suspend = a6xx_pm_suspend,
2970	.pm_resume = a6xx_pm_resume,
2971	.recover = a6xx_recover,
2972	.submit = a6xx_submit,
2973	.active_ring = a6xx_active_ring,
2974	.irq = a6xx_irq,
2975	.destroy = a6xx_destroy,
2976	#if defined(CONFIG_DRM_MSM_GPU_STATE)
2977	.show = a6xx_show,
2978	#endif
2979	.gpu_busy = a6xx_gpu_busy,
2980	#if defined(CONFIG_DRM_MSM_GPU_STATE)
2981	.gpu_state_get = a6xx_gpu_state_get,
2982	.gpu_state_put = a6xx_gpu_state_put,
2983	#endif
2984	.create_address_space = a6xx_create_address_space,
2985	.create_private_address_space = a6xx_create_private_address_space,
2986	.get_rptr = a6xx_get_rptr,
2987	.progress = a6xx_progress,
2988	},
2989	.get_timestamp = a6xx_get_timestamp,
2990	};
2991
2992	static const struct adreno_gpu_funcs funcs_a7xx = {
2993	.base = {
2994	.get_param = adreno_get_param,
2995	.set_param = adreno_set_param,
2996	.hw_init = a6xx_hw_init,
2997	.ucode_load = a6xx_ucode_load,
2998	.pm_suspend = a6xx_gmu_pm_suspend,
2999	.pm_resume = a6xx_gmu_pm_resume,
3000	.recover = a6xx_recover,
3001	.submit = a7xx_submit,
3002	.active_ring = a6xx_active_ring,
3003	.irq = a6xx_irq,
3004	.destroy = a6xx_destroy,
3005	#if defined(CONFIG_DRM_MSM_GPU_STATE)
3006	.show = a6xx_show,
3007	#endif
3008	.gpu_busy = a6xx_gpu_busy,
3009	.gpu_get_freq = a6xx_gmu_get_freq,
3010	.gpu_set_freq = a6xx_gpu_set_freq,
3011	#if defined(CONFIG_DRM_MSM_GPU_STATE)
3012	.gpu_state_get = a6xx_gpu_state_get,
3013	.gpu_state_put = a6xx_gpu_state_put,
3014	#endif
3015	.create_address_space = a6xx_create_address_space,
3016	.create_private_address_space = a6xx_create_private_address_space,
3017	.get_rptr = a6xx_get_rptr,
3018	.progress = a6xx_progress,
3019	},
3020	.get_timestamp = a6xx_gmu_get_timestamp,
3021	};
3022
3023	struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
3024	{
3025	struct msm_drm_private *priv = dev->dev_private;
3026	struct platform_device *pdev = priv->gpu_pdev;
3027	struct adreno_platform_config *config = pdev->dev.platform_data;
3028	struct device_node *node;
3029	struct a6xx_gpu *a6xx_gpu;
3030	struct adreno_gpu *adreno_gpu;
3031	struct msm_gpu *gpu;
3032	bool is_a7xx;
3033	int ret;
3034
3035	a6xx_gpu = kzalloc(size: sizeof(*a6xx_gpu), GFP_KERNEL);
3036	if (!a6xx_gpu)
3037	return ERR_PTR(error: -ENOMEM);
3038
3039	adreno_gpu = &a6xx_gpu->base;
3040	gpu = &adreno_gpu->base;
3041
3042	mutex_init(&a6xx_gpu->gmu.lock);
3043
3044	adreno_gpu->registers = NULL;
3045
3046	/* Check if there is a GMU phandle and set it up */
3047	node = of_parse_phandle(np: pdev->dev.of_node, phandle_name: "qcom,gmu", index: 0);
3048	/* FIXME: How do we gracefully handle this? */
3049	BUG_ON(!node);
3050
3051	adreno_gpu->gmu_is_wrapper = of_device_is_compatible(device: node, "qcom,adreno-gmu-wrapper");
3052
3053	adreno_gpu->base.hw_apriv =
3054	!!(config->info->quirks & ADRENO_QUIRK_HAS_HW_APRIV);
3055
3056	/* gpu->info only gets assigned in adreno_gpu_init() */
3057	is_a7xx = config->info->family == ADRENO_7XX_GEN1 ||
3058	config->info->family == ADRENO_7XX_GEN2 ||
3059	config->info->family == ADRENO_7XX_GEN3;
3060
3061	a6xx_llc_slices_init(pdev, a6xx_gpu, is_a7xx);
3062
3063	ret = a6xx_set_supported_hw(dev: &pdev->dev, info: config->info);
3064	if (ret) {
3065	a6xx_destroy(gpu: &(a6xx_gpu->base.base));
3066	return ERR_PTR(error: ret);
3067	}
3068
3069	if (is_a7xx)
3070	ret = adreno_gpu_init(drm: dev, pdev, gpu: adreno_gpu, funcs: &funcs_a7xx, nr_rings: 1);
3071	else if (adreno_has_gmu_wrapper(gpu: adreno_gpu))
3072	ret = adreno_gpu_init(drm: dev, pdev, gpu: adreno_gpu, funcs: &funcs_gmuwrapper, nr_rings: 1);
3073	else
3074	ret = adreno_gpu_init(drm: dev, pdev, gpu: adreno_gpu, funcs: &funcs, nr_rings: 1);
3075	if (ret) {
3076	a6xx_destroy(gpu: &(a6xx_gpu->base.base));
3077	return ERR_PTR(error: ret);
3078	}
3079
3080	/*
3081	* For now only clamp to idle freq for devices where this is known not
3082	* to cause power supply issues:
3083	*/
3084	if (adreno_is_a618(gpu: adreno_gpu) || adreno_is_7c3(gpu: adreno_gpu))
3085	priv->gpu_clamp_to_idle = true;
3086
3087	if (adreno_has_gmu_wrapper(gpu: adreno_gpu))
3088	ret = a6xx_gmu_wrapper_init(a6xx_gpu, node);
3089	else
3090	ret = a6xx_gmu_init(a6xx_gpu, node);
3091	of_node_put(node);
3092	if (ret) {
3093	a6xx_destroy(gpu: &(a6xx_gpu->base.base));
3094	return ERR_PTR(error: ret);
3095	}
3096
3097	if (gpu->aspace)
3098	msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
3099	a6xx_fault_handler);
3100
3101	a6xx_calc_ubwc_config(gpu: adreno_gpu);
3102
3103	return gpu;
3104	}
3105