1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved.
4	*/
5
6	#include <linux/bitfield.h>
7	#include <linux/clk.h>
8	#include <linux/clk/tegra.h>
9	#include <linux/debugfs.h>
10	#include <linux/delay.h>
11	#include <linux/kernel.h>
12	#include <linux/mod_devicetable.h>
13	#include <linux/module.h>
14	#include <linux/of_reserved_mem.h>
15	#include <linux/platform_device.h>
16	#include <linux/slab.h>
17	#include <linux/thermal.h>
18	#include <soc/tegra/fuse.h>
19	#include <soc/tegra/mc.h>
20
21	#include "tegra210-emc.h"
22	#include "tegra210-mc.h"
23
24	/* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
25	#define EMC_CLK_EMC_2X_CLK_SRC_SHIFT 29
26	#define EMC_CLK_EMC_2X_CLK_SRC_MASK \
27	(0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
28	#define EMC_CLK_SOURCE_PLLM_LJ 0x4
29	#define EMC_CLK_SOURCE_PLLMB_LJ 0x5
30	#define EMC_CLK_FORCE_CC_TRIGGER BIT(27)
31	#define EMC_CLK_MC_EMC_SAME_FREQ BIT(16)
32	#define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT 0
33	#define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK \
34	(0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
35
36	/* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
37	#define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT 29
38	#define DLL_CLK_EMC_DLL_CLK_SRC_MASK \
39	(0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
40	#define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT 10
41	#define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK \
42	(0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
43	#define PLLM_VCOA 0
44	#define PLLM_VCOB 1
45	#define EMC_DLL_SWITCH_OUT 2
46	#define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT 0
47	#define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK \
48	(0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
49
50	/* MC_EMEM_ARB_MISC0 */
51	#define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ BIT(27)
52
53	/* EMC_DATA_BRLSHFT_X */
54	#define EMC0_EMC_DATA_BRLSHFT_0_INDEX 2
55	#define EMC1_EMC_DATA_BRLSHFT_0_INDEX 3
56	#define EMC0_EMC_DATA_BRLSHFT_1_INDEX 4
57	#define EMC1_EMC_DATA_BRLSHFT_1_INDEX 5
58
59	#define TRIM_REG(chan, rank, reg, byte) \
60	(((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
61	_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \
62	next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \
63	rank ## _ ## reg ## _INDEX]) >> \
64	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
65	_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \
66	+ \
67	(((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
68	byte ## _DATA_BRLSHFT_MASK & \
69	next->trim_perch_regs[EMC ## chan ## \
70	_EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
71	EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
72	byte ## _DATA_BRLSHFT_SHIFT) * 64))
73
74	#define CALC_TEMP(rank, reg, byte1, byte2, n) \
75	(((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
76	reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
77	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
78	_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \
79	| \
80	((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
81	reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
82	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
83	_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
84
85	#define REFRESH_SPEEDUP(value, speedup) \
86	(((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
87
88	#define LPDDR2_MR4_SRR GENMASK(2, 0)
89
90	static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
91	&tegra210_emc_r21021,
92	};
93
94	static const struct tegra210_emc_table_register_offsets
95	tegra210_emc_table_register_offsets = {
96	.burst = {
97	EMC_RC,
98	EMC_RFC,
99	EMC_RFCPB,
100	EMC_REFCTRL2,
101	EMC_RFC_SLR,
102	EMC_RAS,
103	EMC_RP,
104	EMC_R2W,
105	EMC_W2R,
106	EMC_R2P,
107	EMC_W2P,
108	EMC_R2R,
109	EMC_TPPD,
110	EMC_CCDMW,
111	EMC_RD_RCD,
112	EMC_WR_RCD,
113	EMC_RRD,
114	EMC_REXT,
115	EMC_WEXT,
116	EMC_WDV_CHK,
117	EMC_WDV,
118	EMC_WSV,
119	EMC_WEV,
120	EMC_WDV_MASK,
121	EMC_WS_DURATION,
122	EMC_WE_DURATION,
123	EMC_QUSE,
124	EMC_QUSE_WIDTH,
125	EMC_IBDLY,
126	EMC_OBDLY,
127	EMC_EINPUT,
128	EMC_MRW6,
129	EMC_EINPUT_DURATION,
130	EMC_PUTERM_EXTRA,
131	EMC_PUTERM_WIDTH,
132	EMC_QRST,
133	EMC_QSAFE,
134	EMC_RDV,
135	EMC_RDV_MASK,
136	EMC_RDV_EARLY,
137	EMC_RDV_EARLY_MASK,
138	EMC_REFRESH,
139	EMC_BURST_REFRESH_NUM,
140	EMC_PRE_REFRESH_REQ_CNT,
141	EMC_PDEX2WR,
142	EMC_PDEX2RD,
143	EMC_PCHG2PDEN,
144	EMC_ACT2PDEN,
145	EMC_AR2PDEN,
146	EMC_RW2PDEN,
147	EMC_CKE2PDEN,
148	EMC_PDEX2CKE,
149	EMC_PDEX2MRR,
150	EMC_TXSR,
151	EMC_TXSRDLL,
152	EMC_TCKE,
153	EMC_TCKESR,
154	EMC_TPD,
155	EMC_TFAW,
156	EMC_TRPAB,
157	EMC_TCLKSTABLE,
158	EMC_TCLKSTOP,
159	EMC_MRW7,
160	EMC_TREFBW,
161	EMC_ODT_WRITE,
162	EMC_FBIO_CFG5,
163	EMC_FBIO_CFG7,
164	EMC_CFG_DIG_DLL,
165	EMC_CFG_DIG_DLL_PERIOD,
166	EMC_PMACRO_IB_RXRT,
167	EMC_CFG_PIPE_1,
168	EMC_CFG_PIPE_2,
169	EMC_PMACRO_QUSE_DDLL_RANK0_4,
170	EMC_PMACRO_QUSE_DDLL_RANK0_5,
171	EMC_PMACRO_QUSE_DDLL_RANK1_4,
172	EMC_PMACRO_QUSE_DDLL_RANK1_5,
173	EMC_MRW8,
174	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
175	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
176	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
177	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
178	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
179	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
180	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
181	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
182	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
183	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
184	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
185	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
186	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
187	EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
188	EMC_PMACRO_DDLL_LONG_CMD_0,
189	EMC_PMACRO_DDLL_LONG_CMD_1,
190	EMC_PMACRO_DDLL_LONG_CMD_2,
191	EMC_PMACRO_DDLL_LONG_CMD_3,
192	EMC_PMACRO_DDLL_LONG_CMD_4,
193	EMC_PMACRO_DDLL_SHORT_CMD_0,
194	EMC_PMACRO_DDLL_SHORT_CMD_1,
195	EMC_PMACRO_DDLL_SHORT_CMD_2,
196	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
197	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
198	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
199	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
200	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
201	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
202	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
203	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
204	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
205	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
206	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
207	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
208	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
209	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
210	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
211	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
212	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
213	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
214	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
215	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
216	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
217	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
218	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
219	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
220	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
221	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
222	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
223	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
224	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
225	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
226	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
227	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
228	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
229	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
230	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
231	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
232	EMC_TXDSRVTTGEN,
233	EMC_FDPD_CTRL_DQ,
234	EMC_FDPD_CTRL_CMD,
235	EMC_FBIO_SPARE,
236	EMC_ZCAL_INTERVAL,
237	EMC_ZCAL_WAIT_CNT,
238	EMC_MRS_WAIT_CNT,
239	EMC_MRS_WAIT_CNT2,
240	EMC_AUTO_CAL_CHANNEL,
241	EMC_DLL_CFG_0,
242	EMC_DLL_CFG_1,
243	EMC_PMACRO_AUTOCAL_CFG_COMMON,
244	EMC_PMACRO_ZCTRL,
245	EMC_CFG,
246	EMC_CFG_PIPE,
247	EMC_DYN_SELF_REF_CONTROL,
248	EMC_QPOP,
249	EMC_DQS_BRLSHFT_0,
250	EMC_DQS_BRLSHFT_1,
251	EMC_CMD_BRLSHFT_2,
252	EMC_CMD_BRLSHFT_3,
253	EMC_PMACRO_PAD_CFG_CTRL,
254	EMC_PMACRO_DATA_PAD_RX_CTRL,
255	EMC_PMACRO_CMD_PAD_RX_CTRL,
256	EMC_PMACRO_DATA_RX_TERM_MODE,
257	EMC_PMACRO_CMD_RX_TERM_MODE,
258	EMC_PMACRO_CMD_PAD_TX_CTRL,
259	EMC_PMACRO_DATA_PAD_TX_CTRL,
260	EMC_PMACRO_COMMON_PAD_TX_CTRL,
261	EMC_PMACRO_VTTGEN_CTRL_0,
262	EMC_PMACRO_VTTGEN_CTRL_1,
263	EMC_PMACRO_VTTGEN_CTRL_2,
264	EMC_PMACRO_BRICK_CTRL_RFU1,
265	EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
266	EMC_PMACRO_BRICK_CTRL_RFU2,
267	EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
268	EMC_PMACRO_BG_BIAS_CTRL_0,
269	EMC_CFG_3,
270	EMC_PMACRO_TX_PWRD_0,
271	EMC_PMACRO_TX_PWRD_1,
272	EMC_PMACRO_TX_PWRD_2,
273	EMC_PMACRO_TX_PWRD_3,
274	EMC_PMACRO_TX_PWRD_4,
275	EMC_PMACRO_TX_PWRD_5,
276	EMC_CONFIG_SAMPLE_DELAY,
277	EMC_PMACRO_TX_SEL_CLK_SRC_0,
278	EMC_PMACRO_TX_SEL_CLK_SRC_1,
279	EMC_PMACRO_TX_SEL_CLK_SRC_2,
280	EMC_PMACRO_TX_SEL_CLK_SRC_3,
281	EMC_PMACRO_TX_SEL_CLK_SRC_4,
282	EMC_PMACRO_TX_SEL_CLK_SRC_5,
283	EMC_PMACRO_DDLL_BYPASS,
284	EMC_PMACRO_DDLL_PWRD_0,
285	EMC_PMACRO_DDLL_PWRD_1,
286	EMC_PMACRO_DDLL_PWRD_2,
287	EMC_PMACRO_CMD_CTRL_0,
288	EMC_PMACRO_CMD_CTRL_1,
289	EMC_PMACRO_CMD_CTRL_2,
290	EMC_TR_TIMING_0,
291	EMC_TR_DVFS,
292	EMC_TR_CTRL_1,
293	EMC_TR_RDV,
294	EMC_TR_QPOP,
295	EMC_TR_RDV_MASK,
296	EMC_MRW14,
297	EMC_TR_QSAFE,
298	EMC_TR_QRST,
299	EMC_TRAINING_CTRL,
300	EMC_TRAINING_SETTLE,
301	EMC_TRAINING_VREF_SETTLE,
302	EMC_TRAINING_CA_FINE_CTRL,
303	EMC_TRAINING_CA_CTRL_MISC,
304	EMC_TRAINING_CA_CTRL_MISC1,
305	EMC_TRAINING_CA_VREF_CTRL,
306	EMC_TRAINING_QUSE_CORS_CTRL,
307	EMC_TRAINING_QUSE_FINE_CTRL,
308	EMC_TRAINING_QUSE_CTRL_MISC,
309	EMC_TRAINING_QUSE_VREF_CTRL,
310	EMC_TRAINING_READ_FINE_CTRL,
311	EMC_TRAINING_READ_CTRL_MISC,
312	EMC_TRAINING_READ_VREF_CTRL,
313	EMC_TRAINING_WRITE_FINE_CTRL,
314	EMC_TRAINING_WRITE_CTRL_MISC,
315	EMC_TRAINING_WRITE_VREF_CTRL,
316	EMC_TRAINING_MPC,
317	EMC_MRW15,
318	},
319	.trim = {
320	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
321	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
322	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
323	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
324	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
325	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
326	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
327	EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
328	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
329	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
330	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
331	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
332	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
333	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
334	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
335	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
336	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
337	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
338	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
339	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
340	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
341	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
342	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
343	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
344	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
345	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
346	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
347	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
348	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
349	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
350	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
351	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
352	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
353	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
354	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
355	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
356	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
357	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
358	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
359	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
360	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
361	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
362	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
363	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
364	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
365	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
366	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
367	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
368	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
369	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
370	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
371	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
372	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
373	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
374	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
375	EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
376	EMC_PMACRO_IB_VREF_DQS_0,
377	EMC_PMACRO_IB_VREF_DQS_1,
378	EMC_PMACRO_IB_VREF_DQ_0,
379	EMC_PMACRO_IB_VREF_DQ_1,
380	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
381	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
382	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
383	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
384	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
385	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
386	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
387	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
388	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
389	EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
390	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
391	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
392	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
393	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
394	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
395	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
396	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
397	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
398	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
399	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
400	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
401	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
402	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
403	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
404	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
405	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
406	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
407	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
408	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
409	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
410	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
411	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
412	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
413	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
414	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
415	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
416	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
417	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
418	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
419	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
420	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
421	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
422	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
423	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
424	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
425	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
426	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
427	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
428	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
429	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
430	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
431	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
432	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
433	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
434	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
435	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
436	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
437	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
438	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
439	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
440	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
441	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
442	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
443	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
444	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
445	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
446	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
447	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
448	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
449	EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
450	EMC_PMACRO_QUSE_DDLL_RANK0_0,
451	EMC_PMACRO_QUSE_DDLL_RANK0_1,
452	EMC_PMACRO_QUSE_DDLL_RANK0_2,
453	EMC_PMACRO_QUSE_DDLL_RANK0_3,
454	EMC_PMACRO_QUSE_DDLL_RANK1_0,
455	EMC_PMACRO_QUSE_DDLL_RANK1_1,
456	EMC_PMACRO_QUSE_DDLL_RANK1_2,
457	EMC_PMACRO_QUSE_DDLL_RANK1_3
458	},
459	.burst_mc = {
460	MC_EMEM_ARB_CFG,
461	MC_EMEM_ARB_OUTSTANDING_REQ,
462	MC_EMEM_ARB_REFPB_HP_CTRL,
463	MC_EMEM_ARB_REFPB_BANK_CTRL,
464	MC_EMEM_ARB_TIMING_RCD,
465	MC_EMEM_ARB_TIMING_RP,
466	MC_EMEM_ARB_TIMING_RC,
467	MC_EMEM_ARB_TIMING_RAS,
468	MC_EMEM_ARB_TIMING_FAW,
469	MC_EMEM_ARB_TIMING_RRD,
470	MC_EMEM_ARB_TIMING_RAP2PRE,
471	MC_EMEM_ARB_TIMING_WAP2PRE,
472	MC_EMEM_ARB_TIMING_R2R,
473	MC_EMEM_ARB_TIMING_W2W,
474	MC_EMEM_ARB_TIMING_R2W,
475	MC_EMEM_ARB_TIMING_CCDMW,
476	MC_EMEM_ARB_TIMING_W2R,
477	MC_EMEM_ARB_TIMING_RFCPB,
478	MC_EMEM_ARB_DA_TURNS,
479	MC_EMEM_ARB_DA_COVERS,
480	MC_EMEM_ARB_MISC0,
481	MC_EMEM_ARB_MISC1,
482	MC_EMEM_ARB_MISC2,
483	MC_EMEM_ARB_RING1_THROTTLE,
484	MC_EMEM_ARB_DHYST_CTRL,
485	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
486	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
487	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
488	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
489	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
490	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
491	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
492	MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
493	},
494	.la_scale = {
495	MC_MLL_MPCORER_PTSA_RATE,
496	MC_FTOP_PTSA_RATE,
497	MC_PTSA_GRANT_DECREMENT,
498	MC_LATENCY_ALLOWANCE_XUSB_0,
499	MC_LATENCY_ALLOWANCE_XUSB_1,
500	MC_LATENCY_ALLOWANCE_TSEC_0,
501	MC_LATENCY_ALLOWANCE_SDMMCA_0,
502	MC_LATENCY_ALLOWANCE_SDMMCAA_0,
503	MC_LATENCY_ALLOWANCE_SDMMC_0,
504	MC_LATENCY_ALLOWANCE_SDMMCAB_0,
505	MC_LATENCY_ALLOWANCE_PPCS_0,
506	MC_LATENCY_ALLOWANCE_PPCS_1,
507	MC_LATENCY_ALLOWANCE_MPCORE_0,
508	MC_LATENCY_ALLOWANCE_HC_0,
509	MC_LATENCY_ALLOWANCE_HC_1,
510	MC_LATENCY_ALLOWANCE_AVPC_0,
511	MC_LATENCY_ALLOWANCE_GPU_0,
512	MC_LATENCY_ALLOWANCE_GPU2_0,
513	MC_LATENCY_ALLOWANCE_NVENC_0,
514	MC_LATENCY_ALLOWANCE_NVDEC_0,
515	MC_LATENCY_ALLOWANCE_VIC_0,
516	MC_LATENCY_ALLOWANCE_VI2_0,
517	MC_LATENCY_ALLOWANCE_ISP2_0,
518	MC_LATENCY_ALLOWANCE_ISP2_1,
519	},
520	.burst_per_channel = {
521	{ .bank = 0, .offset = EMC_MRW10, },
522	{ .bank = 1, .offset = EMC_MRW10, },
523	{ .bank = 0, .offset = EMC_MRW11, },
524	{ .bank = 1, .offset = EMC_MRW11, },
525	{ .bank = 0, .offset = EMC_MRW12, },
526	{ .bank = 1, .offset = EMC_MRW12, },
527	{ .bank = 0, .offset = EMC_MRW13, },
528	{ .bank = 1, .offset = EMC_MRW13, },
529	},
530	.trim_per_channel = {
531	{ .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
532	{ .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
533	{ .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
534	{ .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
535	{ .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
536	{ .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
537	{ .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
538	{ .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
539	{ .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
540	{ .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
541	},
542	.vref_per_channel = {
543	{
544	.bank = 0,
545	.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
546	}, {
547	.bank = 1,
548	.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
549	}, {
550	.bank = 0,
551	.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
552	}, {
553	.bank = 1,
554	.offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
555	},
556	},
557	};
558
559	static void tegra210_emc_train(struct timer_list *timer)
560	{
561	struct tegra210_emc *emc = from_timer(emc, timer, training);
562	unsigned long flags;
563
564	if (!emc->last)
565	return;
566
567	spin_lock_irqsave(&emc->lock, flags);
568
569	if (emc->sequence->periodic_compensation)
570	emc->sequence->periodic_compensation(emc);
571
572	spin_unlock_irqrestore(lock: &emc->lock, flags);
573
574	mod_timer(timer: &emc->training,
575	expires: jiffies + msecs_to_jiffies(m: emc->training_interval));
576	}
577
578	static void tegra210_emc_training_start(struct tegra210_emc *emc)
579	{
580	mod_timer(timer: &emc->training,
581	expires: jiffies + msecs_to_jiffies(m: emc->training_interval));
582	}
583
584	static void tegra210_emc_training_stop(struct tegra210_emc *emc)
585	{
586	del_timer(timer: &emc->training);
587	}
588
589	static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
590	{
591	unsigned long flags;
592	u32 value, max = 0;
593	unsigned int i;
594
595	spin_lock_irqsave(&emc->lock, flags);
596
597	for (i = 0; i < emc->num_devices; i++) {
598	value = tegra210_emc_mrr_read(emc, chip: i, address: 4);
599
600	if (value & BIT(7))
601	dev_dbg(emc->dev,
602	"sensor reading changed for device %u: %08x\n",
603	i, value);
604
605	value = FIELD_GET(LPDDR2_MR4_SRR, value);
606	if (value > max)
607	max = value;
608	}
609
610	spin_unlock_irqrestore(lock: &emc->lock, flags);
611
612	return max;
613	}
614
615	static void tegra210_emc_poll_refresh(struct timer_list *timer)
616	{
617	struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
618	unsigned int temperature;
619
620	if (!emc->debugfs.temperature)
621	temperature = tegra210_emc_get_temperature(emc);
622	else
623	temperature = emc->debugfs.temperature;
624
625	if (temperature == emc->temperature)
626	goto reset;
627
628	switch (temperature) {
629	case 0 ... 3:
630	/* temperature is fine, using regular refresh */
631	dev_dbg(emc->dev, "switching to nominal refresh...\n");
632	tegra210_emc_set_refresh(emc, refresh: TEGRA210_EMC_REFRESH_NOMINAL);
633	break;
634
635	case 4:
636	dev_dbg(emc->dev, "switching to 2x refresh...\n");
637	tegra210_emc_set_refresh(emc, refresh: TEGRA210_EMC_REFRESH_2X);
638	break;
639
640	case 5:
641	dev_dbg(emc->dev, "switching to 4x refresh...\n");
642	tegra210_emc_set_refresh(emc, refresh: TEGRA210_EMC_REFRESH_4X);
643	break;
644
645	case 6 ... 7:
646	dev_dbg(emc->dev, "switching to throttle refresh...\n");
647	tegra210_emc_set_refresh(emc, refresh: TEGRA210_EMC_REFRESH_THROTTLE);
648	break;
649
650	default:
651	WARN(1, "invalid DRAM temperature state %u\n", temperature);
652	return;
653	}
654
655	emc->temperature = temperature;
656
657	reset:
658	if (atomic_read(v: &emc->refresh_poll) > 0) {
659	unsigned int interval = emc->refresh_poll_interval;
660	unsigned int timeout = msecs_to_jiffies(m: interval);
661
662	mod_timer(timer: &emc->refresh_timer, expires: jiffies + timeout);
663	}
664	}
665
666	static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
667	{
668	atomic_set(v: &emc->refresh_poll, i: 0);
669	del_timer_sync(timer: &emc->refresh_timer);
670	}
671
672	static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
673	{
674	atomic_set(v: &emc->refresh_poll, i: 1);
675
676	mod_timer(timer: &emc->refresh_timer,
677	expires: jiffies + msecs_to_jiffies(m: emc->refresh_poll_interval));
678	}
679
680	static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
681	unsigned long *state)
682	{
683	*state = 1;
684
685	return 0;
686	}
687
688	static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
689	unsigned long *state)
690	{
691	struct tegra210_emc *emc = cd->devdata;
692
693	*state = atomic_read(v: &emc->refresh_poll);
694
695	return 0;
696	}
697
698	static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
699	unsigned long state)
700	{
701	struct tegra210_emc *emc = cd->devdata;
702
703	if (state == atomic_read(v: &emc->refresh_poll))
704	return 0;
705
706	if (state)
707	tegra210_emc_poll_refresh_start(emc);
708	else
709	tegra210_emc_poll_refresh_stop(emc);
710
711	return 0;
712	}
713
714	static const struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
715	.get_max_state = tegra210_emc_cd_max_state,
716	.get_cur_state = tegra210_emc_cd_get_state,
717	.set_cur_state = tegra210_emc_cd_set_state,
718	};
719
720	static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
721	{
722	emc->sequence->set_clock(emc, clksrc);
723
724	if (emc->next->periodic_training)
725	tegra210_emc_training_start(emc);
726	else
727	tegra210_emc_training_stop(emc);
728	}
729
730	static void tegra210_change_dll_src(struct tegra210_emc *emc,
731	u32 clksrc)
732	{
733	u32 dll_setting = emc->next->dll_clk_src;
734	u32 emc_clk_src;
735	u32 emc_clk_div;
736
737	emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
738	EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
739	emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
740	EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
741
742	dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
743	DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
744	dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
745	dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
746
747	dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
748	if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
749	dll_setting |= (PLLM_VCOB <<
750	DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
751	else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
752	dll_setting |= (PLLM_VCOA <<
753	DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
754	else
755	dll_setting |= (EMC_DLL_SWITCH_OUT <<
756	DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
757
758	tegra210_clk_emc_dll_update_setting(emc_dll_src_value: dll_setting);
759
760	if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
761	tegra210_clk_emc_dll_enable(flag: true);
762	else
763	tegra210_clk_emc_dll_enable(flag: false);
764	}
765
766	int tegra210_emc_set_refresh(struct tegra210_emc *emc,
767	enum tegra210_emc_refresh refresh)
768	{
769	struct tegra210_emc_timing *timings;
770	unsigned long flags;
771
772	if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
773	emc->dram_type != DRAM_TYPE_LPDDR4) ||
774	!emc->last)
775	return -ENODEV;
776
777	if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
778	return -EINVAL;
779
780	if (refresh == emc->refresh)
781	return 0;
782
783	spin_lock_irqsave(&emc->lock, flags);
784
785	if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
786	timings = emc->derated;
787	else
788	timings = emc->nominal;
789
790	if (timings != emc->timings) {
791	unsigned int index = emc->last - emc->timings;
792	u32 clksrc;
793
794	clksrc = emc->provider.configs[index].value |
795	EMC_CLK_FORCE_CC_TRIGGER;
796
797	emc->next = &timings[index];
798	emc->timings = timings;
799
800	tegra210_emc_set_clock(emc, clksrc);
801	} else {
802	tegra210_emc_adjust_timing(emc, timing: emc->last);
803	tegra210_emc_timing_update(emc);
804
805	if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
806	emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
807	}
808
809	spin_unlock_irqrestore(lock: &emc->lock, flags);
810
811	return 0;
812	}
813
814	u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
815	unsigned int address)
816	{
817	u32 value, ret = 0;
818	unsigned int i;
819
820	value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
821	(address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
822	emc_writel(emc, value, EMC_MRR);
823
824	for (i = 0; i < emc->num_channels; i++)
825	WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
826	EMC_EMC_STATUS_MRR_DIVLD, 1),
827	"Timed out waiting for MRR %u (ch=%u)\n", address, i);
828
829	for (i = 0; i < emc->num_channels; i++) {
830	value = emc_channel_readl(emc, channel: i, EMC_MRR);
831	value &= EMC_MRR_DATA_MASK;
832
833	ret = (ret << 16) | value;
834	}
835
836	return ret;
837	}
838
839	void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
840	{
841	int err;
842
843	mc_readl(mc: emc->mc, MC_EMEM_ADR_CFG);
844	emc_readl(emc, EMC_INTSTATUS);
845
846	tegra210_clk_emc_update_setting(emc_src_value: clksrc);
847
848	err = tegra210_emc_wait_for_update(emc, channel: 0, EMC_INTSTATUS,
849	EMC_INTSTATUS_CLKCHANGE_COMPLETE,
850	state: true);
851	if (err)
852	dev_warn(emc->dev, "clock change completion error: %d\n", err);
853	}
854
855	struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
856	unsigned long rate)
857	{
858	unsigned int i;
859
860	for (i = 0; i < emc->num_timings; i++)
861	if (emc->timings[i].rate * 1000UL == rate)
862	return &emc->timings[i];
863
864	return NULL;
865	}
866
867	int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
868	unsigned int offset, u32 bit_mask, bool state)
869	{
870	unsigned int i;
871	u32 value;
872
873	for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
874	value = emc_channel_readl(emc, channel, offset);
875	if (!!(value & bit_mask) == state)
876	return 0;
877
878	udelay(1);
879	}
880
881	return -ETIMEDOUT;
882	}
883
884	void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
885	{
886	u32 emc_dbg = emc_readl(emc, EMC_DBG);
887
888	if (set)
889	emc_writel(emc, value: emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
890	else
891	emc_writel(emc, value: emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
892	}
893
894	u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
895	{
896	if (next->emc_emrs & 0x1)
897	return 0;
898
899	return 1;
900	}
901
902	void tegra210_emc_timing_update(struct tegra210_emc *emc)
903	{
904	unsigned int i;
905	int err = 0;
906
907	emc_writel(emc, value: 0x1, EMC_TIMING_CONTROL);
908
909	for (i = 0; i < emc->num_channels; i++) {
910	err |= tegra210_emc_wait_for_update(emc, channel: i, EMC_EMC_STATUS,
911	EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
912	state: false);
913	}
914
915	if (err)
916	dev_warn(emc->dev, "timing update error: %d\n", err);
917	}
918
919	unsigned long tegra210_emc_actual_osc_clocks(u32 in)
920	{
921	if (in < 0x40)
922	return in * 16;
923	else if (in < 0x80)
924	return 2048;
925	else if (in < 0xc0)
926	return 4096;
927	else
928	return 8192;
929	}
930
931	void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
932	{
933	u32 mpc_req = 0x4b;
934
935	emc_writel(emc, value: mpc_req, EMC_MPC);
936	mpc_req = emc_readl(emc, EMC_MPC);
937	}
938
939	u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
940	{
941	u32 temp = 0, rate = next->rate / 1000;
942	s32 delta[4], delta_taps[4];
943	s32 new[] = {
944	TRIM_REG(0, 0, 0, 0),
945	TRIM_REG(0, 0, 0, 1),
946	TRIM_REG(0, 0, 1, 2),
947	TRIM_REG(0, 0, 1, 3),
948
949	TRIM_REG(1, 0, 2, 4),
950	TRIM_REG(1, 0, 2, 5),
951	TRIM_REG(1, 0, 3, 6),
952	TRIM_REG(1, 0, 3, 7),
953
954	TRIM_REG(0, 1, 0, 0),
955	TRIM_REG(0, 1, 0, 1),
956	TRIM_REG(0, 1, 1, 2),
957	TRIM_REG(0, 1, 1, 3),
958
959	TRIM_REG(1, 1, 2, 4),
960	TRIM_REG(1, 1, 2, 5),
961	TRIM_REG(1, 1, 3, 6),
962	TRIM_REG(1, 1, 3, 7)
963	};
964	unsigned i;
965
966	switch (offset) {
967	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
968	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
969	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
970	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
971	case EMC_DATA_BRLSHFT_0:
972	delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
973	next->trained_dram_clktree[C0D0U0]);
974	delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
975	next->trained_dram_clktree[C0D0U1]);
976	delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
977	next->trained_dram_clktree[C1D0U0]);
978	delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
979	next->trained_dram_clktree[C1D0U1]);
980
981	delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
982	delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
983	delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
984	delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
985
986	for (i = 0; i < 4; i++) {
987	if ((delta_taps[i] > next->tree_margin) ||
988	(delta_taps[i] < (-1 * next->tree_margin))) {
989	new[i * 2] = new[i * 2] + delta_taps[i];
990	new[i * 2 + 1] = new[i * 2 + 1] +
991	delta_taps[i];
992	}
993	}
994
995	if (offset == EMC_DATA_BRLSHFT_0) {
996	for (i = 0; i < 8; i++)
997	new[i] = new[i] / 64;
998	} else {
999	for (i = 0; i < 8; i++)
1000	new[i] = new[i] % 64;
1001	}
1002
1003	break;
1004
1005	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1006	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1007	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1008	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1009	case EMC_DATA_BRLSHFT_1:
1010	delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1011	next->trained_dram_clktree[C0D1U0]);
1012	delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1013	next->trained_dram_clktree[C0D1U1]);
1014	delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1015	next->trained_dram_clktree[C1D1U0]);
1016	delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1017	next->trained_dram_clktree[C1D1U1]);
1018
1019	delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1020	delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1021	delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1022	delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1023
1024	for (i = 0; i < 4; i++) {
1025	if ((delta_taps[i] > next->tree_margin) ||
1026	(delta_taps[i] < (-1 * next->tree_margin))) {
1027	new[8 + i * 2] = new[8 + i * 2] +
1028	delta_taps[i];
1029	new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1030	delta_taps[i];
1031	}
1032	}
1033
1034	if (offset == EMC_DATA_BRLSHFT_1) {
1035	for (i = 0; i < 8; i++)
1036	new[i + 8] = new[i + 8] / 64;
1037	} else {
1038	for (i = 0; i < 8; i++)
1039	new[i + 8] = new[i + 8] % 64;
1040	}
1041
1042	break;
1043	}
1044
1045	switch (offset) {
1046	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1047	temp = CALC_TEMP(0, 0, 0, 1, 0);
1048	break;
1049
1050	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1051	temp = CALC_TEMP(0, 1, 2, 3, 2);
1052	break;
1053
1054	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1055	temp = CALC_TEMP(0, 2, 4, 5, 4);
1056	break;
1057
1058	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1059	temp = CALC_TEMP(0, 3, 6, 7, 6);
1060	break;
1061
1062	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1063	temp = CALC_TEMP(1, 0, 0, 1, 8);
1064	break;
1065
1066	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1067	temp = CALC_TEMP(1, 1, 2, 3, 10);
1068	break;
1069
1070	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1071	temp = CALC_TEMP(1, 2, 4, 5, 12);
1072	break;
1073
1074	case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1075	temp = CALC_TEMP(1, 3, 6, 7, 14);
1076	break;
1077
1078	case EMC_DATA_BRLSHFT_0:
1079	temp = ((new[0] <<
1080	EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1081	EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1082	((new[1] <<
1083	EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1084	EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1085	((new[2] <<
1086	EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1087	EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1088	((new[3] <<
1089	EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1090	EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1091	((new[4] <<
1092	EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1093	EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1094	((new[5] <<
1095	EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1096	EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1097	((new[6] <<
1098	EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1099	EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1100	((new[7] <<
1101	EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1102	EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1103	break;
1104
1105	case EMC_DATA_BRLSHFT_1:
1106	temp = ((new[8] <<
1107	EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1108	EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1109	((new[9] <<
1110	EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1111	EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1112	((new[10] <<
1113	EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1114	EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1115	((new[11] <<
1116	EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1117	EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1118	((new[12] <<
1119	EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1120	EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1121	((new[13] <<
1122	EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1123	EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1124	((new[14] <<
1125	EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1126	EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1127	((new[15] <<
1128	EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1129	EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1130	break;
1131
1132	default:
1133	break;
1134	}
1135
1136	return temp;
1137	}
1138
1139	u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1140	{
1141	unsigned int i;
1142	u32 value;
1143
1144	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1145	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1146	value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1147	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1148	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1149	value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1150	value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1151	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1152	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1153	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1154	emc_writel(emc, value: 1, EMC_TIMING_CONTROL);
1155
1156	for (i = 0; i < emc->num_channels; i++)
1157	tegra210_emc_wait_for_update(emc, channel: i, EMC_EMC_STATUS,
1158	EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1159	state: 0);
1160
1161	for (i = 0; i < emc->num_channels; i++) {
1162	while (true) {
1163	value = emc_channel_readl(emc, channel: i, EMC_CFG_DIG_DLL);
1164	if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1165	break;
1166	}
1167	}
1168
1169	value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1170	emc_writel(emc, value, EMC_DLL_CFG_0);
1171
1172	value = emc_readl(emc, EMC_DLL_CFG_1);
1173	value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1174
1175	if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1176	value |= 150;
1177	else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1178	value |= 100;
1179	else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1180	value |= 70;
1181	else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1182	value |= 30;
1183	else
1184	value |= 20;
1185
1186	emc_writel(emc, value, EMC_DLL_CFG_1);
1187
1188	tegra210_change_dll_src(emc, clksrc);
1189
1190	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1191	value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1192	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1193
1194	tegra210_emc_timing_update(emc);
1195
1196	for (i = 0; i < emc->num_channels; i++) {
1197	while (true) {
1198	value = emc_channel_readl(emc, channel: 0, EMC_CFG_DIG_DLL);
1199	if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1200	break;
1201	}
1202	}
1203
1204	while (true) {
1205	value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1206
1207	if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1208	continue;
1209
1210	if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1211	continue;
1212
1213	break;
1214	}
1215
1216	value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1217
1218	return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1219	}
1220
1221	u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1222	bool flip_backward)
1223	{
1224	u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1225	const struct tegra210_emc_timing *timing;
1226
1227	if (flip_backward)
1228	timing = emc->last;
1229	else
1230	timing = emc->next;
1231
1232	cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1233	dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1234	rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1235	cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1236	common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1237
1238	cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1239
1240	if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1241	ccfifo_writel(emc, value: common_tx & 0xa,
1242	EMC_PMACRO_COMMON_PAD_TX_CTRL, delay: 0);
1243	ccfifo_writel(emc, value: common_tx & 0xf,
1244	EMC_PMACRO_COMMON_PAD_TX_CTRL,
1245	delay: (100000 / clk) + 1);
1246	ramp_up_wait += 100000;
1247	} else {
1248	ccfifo_writel(emc, value: common_tx | 0x8,
1249	EMC_PMACRO_COMMON_PAD_TX_CTRL, delay: 0);
1250	}
1251
1252	if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1253	if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1254	cmd_pad |=
1255	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1256	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1257	cmd_pad &=
1258	~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1259	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1260	ccfifo_writel(emc, value: cmd_pad,
1261	EMC_PMACRO_CMD_PAD_TX_CTRL,
1262	delay: (100000 / clk) + 1);
1263	ramp_up_wait += 100000;
1264
1265	dq_pad |=
1266	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1267	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1268	dq_pad &=
1269	~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1270	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1271	ccfifo_writel(emc, value: dq_pad,
1272	EMC_PMACRO_DATA_PAD_TX_CTRL, delay: 0);
1273	ccfifo_writel(emc, value: rfu1 & 0xfe40fe40,
1274	EMC_PMACRO_BRICK_CTRL_RFU1, delay: 0);
1275	} else {
1276	ccfifo_writel(emc, value: rfu1 & 0xfe40fe40,
1277	EMC_PMACRO_BRICK_CTRL_RFU1,
1278	delay: (100000 / clk) + 1);
1279	ramp_up_wait += 100000;
1280	}
1281
1282	ccfifo_writel(emc, value: rfu1 & 0xfeedfeed,
1283	EMC_PMACRO_BRICK_CTRL_RFU1, delay: (100000 / clk) + 1);
1284	ramp_up_wait += 100000;
1285
1286	if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1287	cmd_pad |=
1288	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1289	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1290	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1291	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1292	ccfifo_writel(emc, value: cmd_pad,
1293	EMC_PMACRO_CMD_PAD_TX_CTRL,
1294	delay: (100000 / clk) + 1);
1295	ramp_up_wait += 100000;
1296
1297	dq_pad |=
1298	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1299	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1300	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1301	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1302	ccfifo_writel(emc, value: dq_pad,
1303	EMC_PMACRO_DATA_PAD_TX_CTRL, delay: 0);
1304	ccfifo_writel(emc, value: rfu1,
1305	EMC_PMACRO_BRICK_CTRL_RFU1, delay: 0);
1306	} else {
1307	ccfifo_writel(emc, value: rfu1,
1308	EMC_PMACRO_BRICK_CTRL_RFU1,
1309	delay: (100000 / clk) + 1);
1310	ramp_up_wait += 100000;
1311	}
1312
1313	ccfifo_writel(emc, value: cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1314	EMC_FBIO_CFG5, delay: (100000 / clk) + 10);
1315	ramp_up_wait += 100000 + (10 * clk);
1316	} else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1317	ccfifo_writel(emc, value: rfu1 | 0x06000600,
1318	EMC_PMACRO_BRICK_CTRL_RFU1, delay: (100000 / clk) + 1);
1319	ccfifo_writel(emc, value: cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1320	EMC_FBIO_CFG5, delay: (100000 / clk) + 10);
1321	ramp_up_wait += 100000 + 10 * clk;
1322	} else {
1323	ccfifo_writel(emc, value: rfu1 | 0x00000600,
1324	EMC_PMACRO_BRICK_CTRL_RFU1, delay: 0);
1325	ccfifo_writel(emc, value: cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1326	EMC_FBIO_CFG5, delay: 12);
1327	ramp_up_wait += 12 * clk;
1328	}
1329
1330	cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1331	ccfifo_writel(emc, value: cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, delay: 5);
1332
1333	return ramp_up_wait;
1334	}
1335
1336	u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1337	bool flip_backward)
1338	{
1339	u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1340	const struct tegra210_emc_timing *entry;
1341	u32 seq_wait;
1342
1343	if (flip_backward)
1344	entry = emc->next;
1345	else
1346	entry = emc->last;
1347
1348	cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1349	dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1350	rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1351	cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1352	common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1353
1354	cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1355
1356	ccfifo_writel(emc, value: cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, delay: 0);
1357	ccfifo_writel(emc, value: cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1358	EMC_FBIO_CFG5, delay: 12);
1359	ramp_down_wait = 12 * clk;
1360
1361	seq_wait = (100000 / clk) + 1;
1362
1363	if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1364	if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1365	cmd_pad &=
1366	~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1367	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1368	cmd_pad |=
1369	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1370	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1371	ccfifo_writel(emc, value: cmd_pad,
1372	EMC_PMACRO_CMD_PAD_TX_CTRL, delay: seq_wait);
1373	ramp_down_wait += 100000;
1374
1375	dq_pad &=
1376	~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1377	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1378	dq_pad |=
1379	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1380	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1381	ccfifo_writel(emc, value: dq_pad,
1382	EMC_PMACRO_DATA_PAD_TX_CTRL, delay: 0);
1383	ccfifo_writel(emc, value: rfu1 & ~0x01120112,
1384	EMC_PMACRO_BRICK_CTRL_RFU1, delay: 0);
1385	} else {
1386	ccfifo_writel(emc, value: rfu1 & ~0x01120112,
1387	EMC_PMACRO_BRICK_CTRL_RFU1, delay: seq_wait);
1388	ramp_down_wait += 100000;
1389	}
1390
1391	ccfifo_writel(emc, value: rfu1 & ~0x01bf01bf,
1392	EMC_PMACRO_BRICK_CTRL_RFU1, delay: seq_wait);
1393	ramp_down_wait += 100000;
1394
1395	if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1396	cmd_pad &=
1397	~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1398	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1399	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1400	EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1401	ccfifo_writel(emc, value: cmd_pad,
1402	EMC_PMACRO_CMD_PAD_TX_CTRL, delay: seq_wait);
1403	ramp_down_wait += 100000;
1404
1405	dq_pad &=
1406	~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1407	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1408	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1409	EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1410	ccfifo_writel(emc, value: dq_pad,
1411	EMC_PMACRO_DATA_PAD_TX_CTRL, delay: 0);
1412	ccfifo_writel(emc, value: rfu1 & ~0x07ff07ff,
1413	EMC_PMACRO_BRICK_CTRL_RFU1, delay: 0);
1414	} else {
1415	ccfifo_writel(emc, value: rfu1 & ~0x07ff07ff,
1416	EMC_PMACRO_BRICK_CTRL_RFU1, delay: seq_wait);
1417	ramp_down_wait += 100000;
1418	}
1419	} else {
1420	ccfifo_writel(emc, value: rfu1 & ~0xffff07ff,
1421	EMC_PMACRO_BRICK_CTRL_RFU1, delay: seq_wait + 19);
1422	ramp_down_wait += 100000 + (20 * clk);
1423	}
1424
1425	if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1426	ramp_down_wait += 100000;
1427	ccfifo_writel(emc, value: common_tx & ~0x5,
1428	EMC_PMACRO_COMMON_PAD_TX_CTRL, delay: seq_wait);
1429	ramp_down_wait += 100000;
1430	ccfifo_writel(emc, value: common_tx & ~0xf,
1431	EMC_PMACRO_COMMON_PAD_TX_CTRL, delay: seq_wait);
1432	ramp_down_wait += 100000;
1433	ccfifo_writel(emc, value: 0, offset: 0, delay: seq_wait);
1434	ramp_down_wait += 100000;
1435	} else {
1436	ccfifo_writel(emc, value: common_tx & ~0xf,
1437	EMC_PMACRO_COMMON_PAD_TX_CTRL, delay: seq_wait);
1438	}
1439
1440	return ramp_down_wait;
1441	}
1442
1443	void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1444	{
1445	timing->current_dram_clktree[C0D0U0] =
1446	timing->trained_dram_clktree[C0D0U0];
1447	timing->current_dram_clktree[C0D0U1] =
1448	timing->trained_dram_clktree[C0D0U1];
1449	timing->current_dram_clktree[C1D0U0] =
1450	timing->trained_dram_clktree[C1D0U0];
1451	timing->current_dram_clktree[C1D0U1] =
1452	timing->trained_dram_clktree[C1D0U1];
1453	timing->current_dram_clktree[C1D1U0] =
1454	timing->trained_dram_clktree[C1D1U0];
1455	timing->current_dram_clktree[C1D1U1] =
1456	timing->trained_dram_clktree[C1D1U1];
1457	}
1458
1459	static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1460	{
1461	unsigned int i;
1462
1463	emc_writel(emc, value, EMC_CFG_DIG_DLL);
1464	tegra210_emc_timing_update(emc);
1465
1466	for (i = 0; i < emc->num_channels; i++)
1467	tegra210_emc_wait_for_update(emc, channel: i, EMC_CFG_DIG_DLL,
1468	EMC_CFG_DIG_DLL_CFG_DLL_EN,
1469	state);
1470	}
1471
1472	void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1473	{
1474	u32 value;
1475
1476	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1477	value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1478
1479	update_dll_control(emc, value, state: false);
1480	}
1481
1482	void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1483	{
1484	u32 value;
1485
1486	value = emc_readl(emc, EMC_CFG_DIG_DLL);
1487	value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1488
1489	update_dll_control(emc, value, state: true);
1490	}
1491
1492	void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1493	struct tegra210_emc_timing *timing)
1494	{
1495	u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1496	u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1497	u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1498
1499	switch (emc->refresh) {
1500	case TEGRA210_EMC_REFRESH_NOMINAL:
1501	case TEGRA210_EMC_REFRESH_THROTTLE:
1502	break;
1503
1504	case TEGRA210_EMC_REFRESH_2X:
1505	ref = REFRESH_SPEEDUP(ref, 2);
1506	pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1507	dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1508	break;
1509
1510	case TEGRA210_EMC_REFRESH_4X:
1511	ref = REFRESH_SPEEDUP(ref, 4);
1512	pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1513	dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1514	break;
1515
1516	default:
1517	dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1518	return;
1519	}
1520
1521	emc_writel(emc, value: ref, offset: emc->offsets->burst[EMC_REFRESH_INDEX]);
1522	emc_writel(emc, value: pre_ref,
1523	offset: emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1524	emc_writel(emc, value: dsr_cntrl,
1525	offset: emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1526	}
1527
1528	static int tegra210_emc_set_rate(struct device *dev,
1529	const struct tegra210_clk_emc_config *config)
1530	{
1531	struct tegra210_emc *emc = dev_get_drvdata(dev);
1532	struct tegra210_emc_timing *timing = NULL;
1533	unsigned long rate = config->rate;
1534	s64 last_change_delay;
1535	unsigned long flags;
1536	unsigned int i;
1537
1538	if (rate == emc->last->rate * 1000UL)
1539	return 0;
1540
1541	for (i = 0; i < emc->num_timings; i++) {
1542	if (emc->timings[i].rate * 1000UL == rate) {
1543	timing = &emc->timings[i];
1544	break;
1545	}
1546	}
1547
1548	if (!timing)
1549	return -EINVAL;
1550
1551	if (rate > 204000000 && !timing->trained)
1552	return -EINVAL;
1553
1554	emc->next = timing;
1555	last_change_delay = ktime_us_delta(later: ktime_get(), earlier: emc->clkchange_time);
1556
1557	/* XXX use non-busy-looping sleep? */
1558	if ((last_change_delay >= 0) &&
1559	(last_change_delay < emc->clkchange_delay))
1560	udelay(emc->clkchange_delay - (int)last_change_delay);
1561
1562	spin_lock_irqsave(&emc->lock, flags);
1563	tegra210_emc_set_clock(emc, clksrc: config->value);
1564	emc->clkchange_time = ktime_get();
1565	emc->last = timing;
1566	spin_unlock_irqrestore(lock: &emc->lock, flags);
1567
1568	return 0;
1569	}
1570
1571	/*
1572	* debugfs interface
1573	*
1574	* The memory controller driver exposes some files in debugfs that can be used
1575	* to control the EMC frequency. The top-level directory can be found here:
1576	*
1577	* /sys/kernel/debug/emc
1578	*
1579	* It contains the following files:
1580	*
1581	* - available_rates: This file contains a list of valid, space-separated
1582	* EMC frequencies.
1583	*
1584	* - min_rate: Writing a value to this file sets the given frequency as the
1585	* floor of the permitted range. If this is higher than the currently
1586	* configured EMC frequency, this will cause the frequency to be
1587	* increased so that it stays within the valid range.
1588	*
1589	* - max_rate: Similarily to the min_rate file, writing a value to this file
1590	* sets the given frequency as the ceiling of the permitted range. If
1591	* the value is lower than the currently configured EMC frequency, this
1592	* will cause the frequency to be decreased so that it stays within the
1593	* valid range.
1594	*/
1595
1596	static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1597	unsigned long rate)
1598	{
1599	unsigned int i;
1600
1601	for (i = 0; i < emc->num_timings; i++)
1602	if (rate == emc->timings[i].rate * 1000UL)
1603	return true;
1604
1605	return false;
1606	}
1607
1608	static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1609	void *data)
1610	{
1611	struct tegra210_emc *emc = s->private;
1612	const char *prefix = "";
1613	unsigned int i;
1614
1615	for (i = 0; i < emc->num_timings; i++) {
1616	seq_printf(m: s, fmt: "%s%u", prefix, emc->timings[i].rate * 1000);
1617	prefix = " ";
1618	}
1619
1620	seq_puts(m: s, s: "\n");
1621
1622	return 0;
1623	}
1624	DEFINE_SHOW_ATTRIBUTE(tegra210_emc_debug_available_rates);
1625
1626	static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1627	{
1628	struct tegra210_emc *emc = data;
1629
1630	*rate = emc->debugfs.min_rate;
1631
1632	return 0;
1633	}
1634
1635	static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1636	{
1637	struct tegra210_emc *emc = data;
1638	int err;
1639
1640	if (!tegra210_emc_validate_rate(emc, rate))
1641	return -EINVAL;
1642
1643	err = clk_set_min_rate(clk: emc->clk, rate);
1644	if (err < 0)
1645	return err;
1646
1647	emc->debugfs.min_rate = rate;
1648
1649	return 0;
1650	}
1651
1652	DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1653	tegra210_emc_debug_min_rate_get,
1654	tegra210_emc_debug_min_rate_set, "%llu\n");
1655
1656	static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1657	{
1658	struct tegra210_emc *emc = data;
1659
1660	*rate = emc->debugfs.max_rate;
1661
1662	return 0;
1663	}
1664
1665	static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1666	{
1667	struct tegra210_emc *emc = data;
1668	int err;
1669
1670	if (!tegra210_emc_validate_rate(emc, rate))
1671	return -EINVAL;
1672
1673	err = clk_set_max_rate(clk: emc->clk, rate);
1674	if (err < 0)
1675	return err;
1676
1677	emc->debugfs.max_rate = rate;
1678
1679	return 0;
1680	}
1681
1682	DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1683	tegra210_emc_debug_max_rate_get,
1684	tegra210_emc_debug_max_rate_set, "%llu\n");
1685
1686	static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1687	{
1688	struct tegra210_emc *emc = data;
1689	unsigned int value;
1690
1691	if (!emc->debugfs.temperature)
1692	value = tegra210_emc_get_temperature(emc);
1693	else
1694	value = emc->debugfs.temperature;
1695
1696	*temperature = value;
1697
1698	return 0;
1699	}
1700
1701	static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1702	{
1703	struct tegra210_emc *emc = data;
1704
1705	if (temperature > 7)
1706	return -EINVAL;
1707
1708	emc->debugfs.temperature = temperature;
1709
1710	return 0;
1711	}
1712
1713	DEFINE_DEBUGFS_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1714	tegra210_emc_debug_temperature_get,
1715	tegra210_emc_debug_temperature_set, "%llu\n");
1716
1717	static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1718	{
1719	struct device *dev = emc->dev;
1720	unsigned int i;
1721	int err;
1722
1723	emc->debugfs.min_rate = ULONG_MAX;
1724	emc->debugfs.max_rate = 0;
1725
1726	for (i = 0; i < emc->num_timings; i++) {
1727	if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1728	emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1729
1730	if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1731	emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1732	}
1733
1734	if (!emc->num_timings) {
1735	emc->debugfs.min_rate = clk_get_rate(clk: emc->clk);
1736	emc->debugfs.max_rate = emc->debugfs.min_rate;
1737	}
1738
1739	err = clk_set_rate_range(clk: emc->clk, min: emc->debugfs.min_rate,
1740	max: emc->debugfs.max_rate);
1741	if (err < 0) {
1742	dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1743	emc->debugfs.min_rate, emc->debugfs.max_rate,
1744	emc->clk);
1745	return;
1746	}
1747
1748	emc->debugfs.root = debugfs_create_dir(name: "emc", NULL);
1749
1750	debugfs_create_file(name: "available_rates", mode: 0444, parent: emc->debugfs.root, data: emc,
1751	fops: &tegra210_emc_debug_available_rates_fops);
1752	debugfs_create_file(name: "min_rate", mode: 0644, parent: emc->debugfs.root, data: emc,
1753	fops: &tegra210_emc_debug_min_rate_fops);
1754	debugfs_create_file(name: "max_rate", mode: 0644, parent: emc->debugfs.root, data: emc,
1755	fops: &tegra210_emc_debug_max_rate_fops);
1756	debugfs_create_file(name: "temperature", mode: 0644, parent: emc->debugfs.root, data: emc,
1757	fops: &tegra210_emc_debug_temperature_fops);
1758	}
1759
1760	static void tegra210_emc_detect(struct tegra210_emc *emc)
1761	{
1762	u32 value;
1763
1764	/* probe the number of connected DRAM devices */
1765	value = mc_readl(mc: emc->mc, MC_EMEM_ADR_CFG);
1766
1767	if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1768	emc->num_devices = 2;
1769	else
1770	emc->num_devices = 1;
1771
1772	/* probe the type of DRAM */
1773	value = emc_readl(emc, EMC_FBIO_CFG5);
1774	emc->dram_type = value & 0x3;
1775
1776	/* probe the number of channels */
1777	value = emc_readl(emc, EMC_FBIO_CFG7);
1778
1779	if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1780	(value & EMC_FBIO_CFG7_CH0_ENABLE))
1781	emc->num_channels = 2;
1782	else
1783	emc->num_channels = 1;
1784	}
1785
1786	static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1787	struct tegra210_emc_timing *timings,
1788	unsigned int num_timings)
1789	{
1790	unsigned int i;
1791
1792	for (i = 0; i < num_timings; i++) {
1793	u32 min_volt = timings[i].min_volt;
1794	u32 rate = timings[i].rate;
1795
1796	if (!rate)
1797	return -EINVAL;
1798
1799	if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1800	(min_volt < timings[i - 1].min_volt)))
1801	return -EINVAL;
1802
1803	if (timings[i].revision != timings[0].revision)
1804	continue;
1805	}
1806
1807	return 0;
1808	}
1809
1810	static int tegra210_emc_probe(struct platform_device *pdev)
1811	{
1812	struct thermal_cooling_device *cd;
1813	unsigned long current_rate;
1814	struct tegra210_emc *emc;
1815	struct device_node *np;
1816	unsigned int i;
1817	int err;
1818
1819	emc = devm_kzalloc(dev: &pdev->dev, size: sizeof(*emc), GFP_KERNEL);
1820	if (!emc)
1821	return -ENOMEM;
1822
1823	emc->clk = devm_clk_get(dev: &pdev->dev, id: "emc");
1824	if (IS_ERR(ptr: emc->clk))
1825	return PTR_ERR(ptr: emc->clk);
1826
1827	platform_set_drvdata(pdev, data: emc);
1828	spin_lock_init(&emc->lock);
1829	emc->dev = &pdev->dev;
1830
1831	emc->mc = devm_tegra_memory_controller_get(dev: &pdev->dev);
1832	if (IS_ERR(ptr: emc->mc))
1833	return PTR_ERR(ptr: emc->mc);
1834
1835	emc->regs = devm_platform_ioremap_resource(pdev, index: 0);
1836	if (IS_ERR(ptr: emc->regs))
1837	return PTR_ERR(ptr: emc->regs);
1838
1839	for (i = 0; i < 2; i++) {
1840	emc->channel[i] = devm_platform_ioremap_resource(pdev, index: 1 + i);
1841	if (IS_ERR(ptr: emc->channel[i]))
1842	return PTR_ERR(ptr: emc->channel[i]);
1843
1844	}
1845
1846	tegra210_emc_detect(emc);
1847	np = pdev->dev.of_node;
1848
1849	/* attach to the nominal and (optional) derated tables */
1850	err = of_reserved_mem_device_init_by_name(dev: emc->dev, np, name: "nominal");
1851	if (err < 0) {
1852	dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1853	return err;
1854	}
1855
1856	err = of_reserved_mem_device_init_by_name(dev: emc->dev, np, name: "derated");
1857	if (err < 0 && err != -ENODEV) {
1858	dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1859	goto release;
1860	}
1861
1862	/* validate the tables */
1863	if (emc->nominal) {
1864	err = tegra210_emc_validate_timings(emc, timings: emc->nominal,
1865	num_timings: emc->num_timings);
1866	if (err < 0)
1867	goto release;
1868	}
1869
1870	if (emc->derated) {
1871	err = tegra210_emc_validate_timings(emc, timings: emc->derated,
1872	num_timings: emc->num_timings);
1873	if (err < 0)
1874	goto release;
1875	}
1876
1877	/* default to the nominal table */
1878	emc->timings = emc->nominal;
1879
1880	/* pick the current timing based on the current EMC clock rate */
1881	current_rate = clk_get_rate(clk: emc->clk) / 1000;
1882
1883	for (i = 0; i < emc->num_timings; i++) {
1884	if (emc->timings[i].rate == current_rate) {
1885	emc->last = &emc->timings[i];
1886	break;
1887	}
1888	}
1889
1890	if (i == emc->num_timings) {
1891	dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1892	current_rate);
1893	err = -ENOENT;
1894	goto release;
1895	}
1896
1897	/* pick a compatible clock change sequence for the EMC table */
1898	for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1899	const struct tegra210_emc_sequence *sequence =
1900	tegra210_emc_sequences[i];
1901
1902	if (emc->timings[0].revision == sequence->revision) {
1903	emc->sequence = sequence;
1904	break;
1905	}
1906	}
1907
1908	if (!emc->sequence) {
1909	dev_err(&pdev->dev, "sequence %u not supported\n",
1910	emc->timings[0].revision);
1911	err = -ENOTSUPP;
1912	goto release;
1913	}
1914
1915	emc->offsets = &tegra210_emc_table_register_offsets;
1916	emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1917
1918	emc->provider.owner = THIS_MODULE;
1919	emc->provider.dev = &pdev->dev;
1920	emc->provider.set_rate = tegra210_emc_set_rate;
1921
1922	emc->provider.configs = devm_kcalloc(dev: &pdev->dev, n: emc->num_timings,
1923	size: sizeof(*emc->provider.configs),
1924	GFP_KERNEL);
1925	if (!emc->provider.configs) {
1926	err = -ENOMEM;
1927	goto release;
1928	}
1929
1930	emc->provider.num_configs = emc->num_timings;
1931
1932	for (i = 0; i < emc->provider.num_configs; i++) {
1933	struct tegra210_emc_timing *timing = &emc->timings[i];
1934	struct tegra210_clk_emc_config *config =
1935	&emc->provider.configs[i];
1936	u32 value;
1937
1938	config->rate = timing->rate * 1000UL;
1939	config->value = timing->clk_src_emc;
1940
1941	value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1942
1943	if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1944	config->same_freq = false;
1945	else
1946	config->same_freq = true;
1947	}
1948
1949	err = tegra210_clk_emc_attach(clk: emc->clk, provider: &emc->provider);
1950	if (err < 0) {
1951	dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1952	goto release;
1953	}
1954
1955	emc->clkchange_delay = 100;
1956	emc->training_interval = 100;
1957	dev_set_drvdata(dev: emc->dev, data: emc);
1958
1959	timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1960	TIMER_DEFERRABLE);
1961	atomic_set(v: &emc->refresh_poll, i: 0);
1962	emc->refresh_poll_interval = 1000;
1963
1964	timer_setup(&emc->training, tegra210_emc_train, 0);
1965
1966	tegra210_emc_debugfs_init(emc);
1967
1968	cd = devm_thermal_of_cooling_device_register(dev: emc->dev, np, type: "emc", devdata: emc,
1969	ops: &tegra210_emc_cd_ops);
1970	if (IS_ERR(ptr: cd)) {
1971	err = PTR_ERR(ptr: cd);
1972	dev_err(emc->dev, "failed to register cooling device: %d\n",
1973	err);
1974	goto detach;
1975	}
1976
1977	return 0;
1978
1979	detach:
1980	debugfs_remove_recursive(dentry: emc->debugfs.root);
1981	tegra210_clk_emc_detach(clk: emc->clk);
1982	release:
1983	of_reserved_mem_device_release(dev: emc->dev);
1984
1985	return err;
1986	}
1987
1988	static void tegra210_emc_remove(struct platform_device *pdev)
1989	{
1990	struct tegra210_emc *emc = platform_get_drvdata(pdev);
1991
1992	debugfs_remove_recursive(dentry: emc->debugfs.root);
1993	tegra210_clk_emc_detach(clk: emc->clk);
1994	of_reserved_mem_device_release(dev: emc->dev);
1995	}
1996
1997	static int __maybe_unused tegra210_emc_suspend(struct device *dev)
1998	{
1999	struct tegra210_emc *emc = dev_get_drvdata(dev);
2000	int err;
2001
2002	err = clk_rate_exclusive_get(clk: emc->clk);
2003	if (err < 0) {
2004	dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2005	return err;
2006	}
2007
2008	emc->resume_rate = clk_get_rate(clk: emc->clk);
2009
2010	clk_set_rate(clk: emc->clk, rate: 204000000);
2011	tegra210_clk_emc_detach(clk: emc->clk);
2012
2013	dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2014
2015	return 0;
2016	}
2017
2018	static int __maybe_unused tegra210_emc_resume(struct device *dev)
2019	{
2020	struct tegra210_emc *emc = dev_get_drvdata(dev);
2021	int err;
2022
2023	err = tegra210_clk_emc_attach(clk: emc->clk, provider: &emc->provider);
2024	if (err < 0) {
2025	dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2026	return err;
2027	}
2028
2029	clk_set_rate(clk: emc->clk, rate: emc->resume_rate);
2030	clk_rate_exclusive_put(clk: emc->clk);
2031
2032	dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2033
2034	return 0;
2035	}
2036
2037	static const struct dev_pm_ops tegra210_emc_pm_ops = {
2038	SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2039	};
2040
2041	static const struct of_device_id tegra210_emc_of_match[] = {
2042	{ .compatible = "nvidia,tegra210-emc", },
2043	{ },
2044	};
2045	MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2046
2047	static struct platform_driver tegra210_emc_driver = {
2048	.driver = {
2049	.name = "tegra210-emc",
2050	.of_match_table = tegra210_emc_of_match,
2051	.pm = &tegra210_emc_pm_ops,
2052	},
2053	.probe = tegra210_emc_probe,
2054	.remove_new = tegra210_emc_remove,
2055	};
2056
2057	module_platform_driver(tegra210_emc_driver);
2058
2059	MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2060	MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2061	MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2062	MODULE_LICENSE("GPL v2");
2063