1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/kernel.h> |
7 | #include <linux/io.h> |
8 | #include <linux/clk.h> |
9 | #include <linux/delay.h> |
10 | #include <linux/of.h> |
11 | |
12 | #include <soc/tegra/mc.h> |
13 | |
14 | #include "tegra210-emc.h" |
15 | #include "tegra210-mc.h" |
16 | |
17 | /* |
18 | * Enable flags for specifying verbosity. |
19 | */ |
20 | #define INFO (1 << 0) |
21 | #define STEPS (1 << 1) |
22 | #define SUB_STEPS (1 << 2) |
23 | #define PRELOCK (1 << 3) |
24 | #define PRELOCK_STEPS (1 << 4) |
25 | #define ACTIVE_EN (1 << 5) |
26 | #define PRAMP_UP (1 << 6) |
27 | #define PRAMP_DN (1 << 7) |
28 | #define EMA_WRITES (1 << 10) |
29 | #define EMA_UPDATES (1 << 11) |
30 | #define PER_TRAIN (1 << 16) |
31 | #define CC_PRINT (1 << 17) |
32 | #define CCFIFO (1 << 29) |
33 | #define REGS (1 << 30) |
34 | #define REG_LISTS (1 << 31) |
35 | |
36 | #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) |
37 | |
38 | #define DVFS_CLOCK_CHANGE_VERSION 21021 |
39 | #define EMC_PRELOCK_VERSION 2101 |
40 | |
41 | enum { |
42 | DVFS_SEQUENCE = 1, |
43 | WRITE_TRAINING_SEQUENCE = 2, |
44 | PERIODIC_TRAINING_SEQUENCE = 3, |
45 | DVFS_PT1 = 10, |
46 | DVFS_UPDATE = 11, |
47 | TRAINING_PT1 = 12, |
48 | TRAINING_UPDATE = 13, |
49 | PERIODIC_TRAINING_UPDATE = 14 |
50 | }; |
51 | |
52 | /* |
53 | * PTFV defines - basically just indexes into the per table PTFV array. |
54 | */ |
55 | #define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0 |
56 | #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1 |
57 | #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2 |
58 | #define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3 |
59 | #define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4 |
60 | #define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5 |
61 | #define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6 |
62 | #define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7 |
63 | #define PTFV_DVFS_SAMPLES_INDEX 9 |
64 | #define PTFV_MOVAVG_WEIGHT_INDEX 10 |
65 | #define PTFV_CONFIG_CTRL_INDEX 11 |
66 | |
67 | #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0) |
68 | |
69 | /* |
70 | * Do arithmetic in fixed point. |
71 | */ |
72 | #define MOVAVG_PRECISION_FACTOR 100 |
73 | |
74 | /* |
75 | * The division portion of the average operation. |
76 | */ |
77 | #define __AVERAGE_PTFV(dev) \ |
78 | ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \ |
79 | next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ |
80 | next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) |
81 | |
82 | /* |
83 | * Convert val to fixed point and add it to the temporary average. |
84 | */ |
85 | #define __INCREMENT_PTFV(dev, val) \ |
86 | ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \ |
87 | ((val) * MOVAVG_PRECISION_FACTOR); }) |
88 | |
89 | /* |
90 | * Convert a moving average back to integral form and return the value. |
91 | */ |
92 | #define __MOVAVG_AC(timing, dev) \ |
93 | ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ |
94 | MOVAVG_PRECISION_FACTOR) |
95 | |
96 | /* Weighted update. */ |
97 | #define __WEIGHTED_UPDATE_PTFV(dev, nval) \ |
98 | do { \ |
99 | int w = PTFV_MOVAVG_WEIGHT_INDEX; \ |
100 | int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \ |
101 | \ |
102 | next->ptfv_list[dqs] = \ |
103 | ((nval * MOVAVG_PRECISION_FACTOR) + \ |
104 | (next->ptfv_list[dqs] * \ |
105 | next->ptfv_list[w])) / \ |
106 | (next->ptfv_list[w] + 1); \ |
107 | \ |
108 | emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \ |
109 | __stringify(dev), nval, next->ptfv_list[dqs]); \ |
110 | } while (0) |
111 | |
112 | /* Access a particular average. */ |
113 | #define __MOVAVG(timing, dev) \ |
114 | ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX]) |
115 | |
116 | static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type) |
117 | { |
118 | bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE; |
119 | struct tegra210_emc_timing *last = emc->last; |
120 | struct tegra210_emc_timing *next = emc->next; |
121 | u32 last_timing_rate_mhz = last->rate / 1000; |
122 | u32 next_timing_rate_mhz = next->rate / 1000; |
123 | bool dvfs_update = type == DVFS_UPDATE; |
124 | s32 tdel = 0, tmdel = 0, adel = 0; |
125 | bool dvfs_pt1 = type == DVFS_PT1; |
126 | unsigned long cval = 0; |
127 | u32 temp[2][2], value; |
128 | unsigned int i; |
129 | |
130 | /* |
131 | * Dev0 MSB. |
132 | */ |
133 | if (dvfs_pt1 || periodic_training_update) { |
134 | value = tegra210_emc_mrr_read(emc, chip: 2, address: 19); |
135 | |
136 | for (i = 0; i < emc->num_channels; i++) { |
137 | temp[i][0] = (value & 0x00ff) << 8; |
138 | temp[i][1] = (value & 0xff00) << 0; |
139 | value >>= 16; |
140 | } |
141 | |
142 | /* |
143 | * Dev0 LSB. |
144 | */ |
145 | value = tegra210_emc_mrr_read(emc, chip: 2, address: 18); |
146 | |
147 | for (i = 0; i < emc->num_channels; i++) { |
148 | temp[i][0] |= (value & 0x00ff) >> 0; |
149 | temp[i][1] |= (value & 0xff00) >> 8; |
150 | value >>= 16; |
151 | } |
152 | } |
153 | |
154 | if (dvfs_pt1 || periodic_training_update) { |
155 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
156 | cval *= 1000000; |
157 | cval /= last_timing_rate_mhz * 2 * temp[0][0]; |
158 | } |
159 | |
160 | if (dvfs_pt1) |
161 | __INCREMENT_PTFV(C0D0U0, cval); |
162 | else if (dvfs_update) |
163 | __AVERAGE_PTFV(C0D0U0); |
164 | else if (periodic_training_update) |
165 | __WEIGHTED_UPDATE_PTFV(C0D0U0, cval); |
166 | |
167 | if (dvfs_update || periodic_training_update) { |
168 | tdel = next->current_dram_clktree[C0D0U0] - |
169 | __MOVAVG_AC(next, C0D0U0); |
170 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
171 | adel = tmdel; |
172 | |
173 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
174 | next->tree_margin) |
175 | next->current_dram_clktree[C0D0U0] = |
176 | __MOVAVG_AC(next, C0D0U0); |
177 | } |
178 | |
179 | if (dvfs_pt1 || periodic_training_update) { |
180 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
181 | cval *= 1000000; |
182 | cval /= last_timing_rate_mhz * 2 * temp[0][1]; |
183 | } |
184 | |
185 | if (dvfs_pt1) |
186 | __INCREMENT_PTFV(C0D0U1, cval); |
187 | else if (dvfs_update) |
188 | __AVERAGE_PTFV(C0D0U1); |
189 | else if (periodic_training_update) |
190 | __WEIGHTED_UPDATE_PTFV(C0D0U1, cval); |
191 | |
192 | if (dvfs_update || periodic_training_update) { |
193 | tdel = next->current_dram_clktree[C0D0U1] - |
194 | __MOVAVG_AC(next, C0D0U1); |
195 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
196 | |
197 | if (tmdel > adel) |
198 | adel = tmdel; |
199 | |
200 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
201 | next->tree_margin) |
202 | next->current_dram_clktree[C0D0U1] = |
203 | __MOVAVG_AC(next, C0D0U1); |
204 | } |
205 | |
206 | if (emc->num_channels > 1) { |
207 | if (dvfs_pt1 || periodic_training_update) { |
208 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
209 | cval *= 1000000; |
210 | cval /= last_timing_rate_mhz * 2 * temp[1][0]; |
211 | } |
212 | |
213 | if (dvfs_pt1) |
214 | __INCREMENT_PTFV(C1D0U0, cval); |
215 | else if (dvfs_update) |
216 | __AVERAGE_PTFV(C1D0U0); |
217 | else if (periodic_training_update) |
218 | __WEIGHTED_UPDATE_PTFV(C1D0U0, cval); |
219 | |
220 | if (dvfs_update || periodic_training_update) { |
221 | tdel = next->current_dram_clktree[C1D0U0] - |
222 | __MOVAVG_AC(next, C1D0U0); |
223 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
224 | |
225 | if (tmdel > adel) |
226 | adel = tmdel; |
227 | |
228 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
229 | next->tree_margin) |
230 | next->current_dram_clktree[C1D0U0] = |
231 | __MOVAVG_AC(next, C1D0U0); |
232 | } |
233 | |
234 | if (dvfs_pt1 || periodic_training_update) { |
235 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
236 | cval *= 1000000; |
237 | cval /= last_timing_rate_mhz * 2 * temp[1][1]; |
238 | } |
239 | |
240 | if (dvfs_pt1) |
241 | __INCREMENT_PTFV(C1D0U1, cval); |
242 | else if (dvfs_update) |
243 | __AVERAGE_PTFV(C1D0U1); |
244 | else if (periodic_training_update) |
245 | __WEIGHTED_UPDATE_PTFV(C1D0U1, cval); |
246 | |
247 | if (dvfs_update || periodic_training_update) { |
248 | tdel = next->current_dram_clktree[C1D0U1] - |
249 | __MOVAVG_AC(next, C1D0U1); |
250 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
251 | |
252 | if (tmdel > adel) |
253 | adel = tmdel; |
254 | |
255 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
256 | next->tree_margin) |
257 | next->current_dram_clktree[C1D0U1] = |
258 | __MOVAVG_AC(next, C1D0U1); |
259 | } |
260 | } |
261 | |
262 | if (emc->num_devices < 2) |
263 | goto done; |
264 | |
265 | /* |
266 | * Dev1 MSB. |
267 | */ |
268 | if (dvfs_pt1 || periodic_training_update) { |
269 | value = tegra210_emc_mrr_read(emc, chip: 1, address: 19); |
270 | |
271 | for (i = 0; i < emc->num_channels; i++) { |
272 | temp[i][0] = (value & 0x00ff) << 8; |
273 | temp[i][1] = (value & 0xff00) << 0; |
274 | value >>= 16; |
275 | } |
276 | |
277 | /* |
278 | * Dev1 LSB. |
279 | */ |
280 | value = tegra210_emc_mrr_read(emc, chip: 1, address: 18); |
281 | |
282 | for (i = 0; i < emc->num_channels; i++) { |
283 | temp[i][0] |= (value & 0x00ff) >> 0; |
284 | temp[i][1] |= (value & 0xff00) >> 8; |
285 | value >>= 16; |
286 | } |
287 | } |
288 | |
289 | if (dvfs_pt1 || periodic_training_update) { |
290 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
291 | cval *= 1000000; |
292 | cval /= last_timing_rate_mhz * 2 * temp[0][0]; |
293 | } |
294 | |
295 | if (dvfs_pt1) |
296 | __INCREMENT_PTFV(C0D1U0, cval); |
297 | else if (dvfs_update) |
298 | __AVERAGE_PTFV(C0D1U0); |
299 | else if (periodic_training_update) |
300 | __WEIGHTED_UPDATE_PTFV(C0D1U0, cval); |
301 | |
302 | if (dvfs_update || periodic_training_update) { |
303 | tdel = next->current_dram_clktree[C0D1U0] - |
304 | __MOVAVG_AC(next, C0D1U0); |
305 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
306 | |
307 | if (tmdel > adel) |
308 | adel = tmdel; |
309 | |
310 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
311 | next->tree_margin) |
312 | next->current_dram_clktree[C0D1U0] = |
313 | __MOVAVG_AC(next, C0D1U0); |
314 | } |
315 | |
316 | if (dvfs_pt1 || periodic_training_update) { |
317 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
318 | cval *= 1000000; |
319 | cval /= last_timing_rate_mhz * 2 * temp[0][1]; |
320 | } |
321 | |
322 | if (dvfs_pt1) |
323 | __INCREMENT_PTFV(C0D1U1, cval); |
324 | else if (dvfs_update) |
325 | __AVERAGE_PTFV(C0D1U1); |
326 | else if (periodic_training_update) |
327 | __WEIGHTED_UPDATE_PTFV(C0D1U1, cval); |
328 | |
329 | if (dvfs_update || periodic_training_update) { |
330 | tdel = next->current_dram_clktree[C0D1U1] - |
331 | __MOVAVG_AC(next, C0D1U1); |
332 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
333 | |
334 | if (tmdel > adel) |
335 | adel = tmdel; |
336 | |
337 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
338 | next->tree_margin) |
339 | next->current_dram_clktree[C0D1U1] = |
340 | __MOVAVG_AC(next, C0D1U1); |
341 | } |
342 | |
343 | if (emc->num_channels > 1) { |
344 | if (dvfs_pt1 || periodic_training_update) { |
345 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
346 | cval *= 1000000; |
347 | cval /= last_timing_rate_mhz * 2 * temp[1][0]; |
348 | } |
349 | |
350 | if (dvfs_pt1) |
351 | __INCREMENT_PTFV(C1D1U0, cval); |
352 | else if (dvfs_update) |
353 | __AVERAGE_PTFV(C1D1U0); |
354 | else if (periodic_training_update) |
355 | __WEIGHTED_UPDATE_PTFV(C1D1U0, cval); |
356 | |
357 | if (dvfs_update || periodic_training_update) { |
358 | tdel = next->current_dram_clktree[C1D1U0] - |
359 | __MOVAVG_AC(next, C1D1U0); |
360 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
361 | |
362 | if (tmdel > adel) |
363 | adel = tmdel; |
364 | |
365 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
366 | next->tree_margin) |
367 | next->current_dram_clktree[C1D1U0] = |
368 | __MOVAVG_AC(next, C1D1U0); |
369 | } |
370 | |
371 | if (dvfs_pt1 || periodic_training_update) { |
372 | cval = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
373 | cval *= 1000000; |
374 | cval /= last_timing_rate_mhz * 2 * temp[1][1]; |
375 | } |
376 | |
377 | if (dvfs_pt1) |
378 | __INCREMENT_PTFV(C1D1U1, cval); |
379 | else if (dvfs_update) |
380 | __AVERAGE_PTFV(C1D1U1); |
381 | else if (periodic_training_update) |
382 | __WEIGHTED_UPDATE_PTFV(C1D1U1, cval); |
383 | |
384 | if (dvfs_update || periodic_training_update) { |
385 | tdel = next->current_dram_clktree[C1D1U1] - |
386 | __MOVAVG_AC(next, C1D1U1); |
387 | tmdel = (tdel < 0) ? -1 * tdel : tdel; |
388 | |
389 | if (tmdel > adel) |
390 | adel = tmdel; |
391 | |
392 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > |
393 | next->tree_margin) |
394 | next->current_dram_clktree[C1D1U1] = |
395 | __MOVAVG_AC(next, C1D1U1); |
396 | } |
397 | } |
398 | |
399 | done: |
400 | return adel; |
401 | } |
402 | |
403 | static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type, |
404 | struct tegra210_emc_timing *last, |
405 | struct tegra210_emc_timing *next) |
406 | { |
407 | #define __COPY_EMA(nt, lt, dev) \ |
408 | ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \ |
409 | (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) |
410 | |
411 | u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; |
412 | u32 delay; |
413 | |
414 | delay = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
415 | delay *= 1000; |
416 | delay = 2 + (delay / last->rate); |
417 | |
418 | if (!next->periodic_training) |
419 | return 0; |
420 | |
421 | if (type == DVFS_SEQUENCE) { |
422 | if (last->periodic_training && |
423 | (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & |
424 | PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) { |
425 | /* |
426 | * If the previous frequency was using periodic |
427 | * calibration then we can reuse the previous |
428 | * frequencies EMA data. |
429 | */ |
430 | __COPY_EMA(next, last, C0D0U0); |
431 | __COPY_EMA(next, last, C0D0U1); |
432 | __COPY_EMA(next, last, C1D0U0); |
433 | __COPY_EMA(next, last, C1D0U1); |
434 | __COPY_EMA(next, last, C0D1U0); |
435 | __COPY_EMA(next, last, C0D1U1); |
436 | __COPY_EMA(next, last, C1D1U0); |
437 | __COPY_EMA(next, last, C1D1U1); |
438 | } else { |
439 | /* Reset the EMA.*/ |
440 | __MOVAVG(next, C0D0U0) = 0; |
441 | __MOVAVG(next, C0D0U1) = 0; |
442 | __MOVAVG(next, C1D0U0) = 0; |
443 | __MOVAVG(next, C1D0U1) = 0; |
444 | __MOVAVG(next, C0D1U0) = 0; |
445 | __MOVAVG(next, C0D1U1) = 0; |
446 | __MOVAVG(next, C1D1U0) = 0; |
447 | __MOVAVG(next, C1D1U1) = 0; |
448 | |
449 | for (i = 0; i < samples; i++) { |
450 | tegra210_emc_start_periodic_compensation(emc); |
451 | udelay(delay); |
452 | |
453 | /* |
454 | * Generate next sample of data. |
455 | */ |
456 | adel = update_clock_tree_delay(emc, type: DVFS_PT1); |
457 | } |
458 | } |
459 | |
460 | /* |
461 | * Seems like it should be part of the |
462 | * 'if (last_timing->periodic_training)' conditional |
463 | * since is already done for the else clause. |
464 | */ |
465 | adel = update_clock_tree_delay(emc, type: DVFS_UPDATE); |
466 | } |
467 | |
468 | if (type == PERIODIC_TRAINING_SEQUENCE) { |
469 | tegra210_emc_start_periodic_compensation(emc); |
470 | udelay(delay); |
471 | |
472 | adel = update_clock_tree_delay(emc, type: PERIODIC_TRAINING_UPDATE); |
473 | } |
474 | |
475 | return adel; |
476 | } |
477 | |
478 | static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) |
479 | { |
480 | u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value; |
481 | static const u32 list[] = { |
482 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, |
483 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, |
484 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, |
485 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, |
486 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, |
487 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, |
488 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, |
489 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, |
490 | EMC_DATA_BRLSHFT_0, |
491 | EMC_DATA_BRLSHFT_1 |
492 | }; |
493 | struct tegra210_emc_timing *last = emc->last; |
494 | unsigned int items = ARRAY_SIZE(list), i; |
495 | unsigned long delay; |
496 | |
497 | if (last->periodic_training) { |
498 | emc_dbg(emc, PER_TRAIN, "Periodic training starting\n" ); |
499 | |
500 | value = emc_readl(emc, EMC_DBG); |
501 | emc_cfg_o = emc_readl(emc, EMC_CFG); |
502 | emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | |
503 | EMC_CFG_DRAM_ACPD | |
504 | EMC_CFG_DRAM_CLKSTOP_PD); |
505 | |
506 | |
507 | /* |
508 | * 1. Power optimizations should be off. |
509 | */ |
510 | emc_writel(emc, value: emc_cfg, EMC_CFG); |
511 | |
512 | /* Does emc_timing_update() for above changes. */ |
513 | tegra210_emc_dll_disable(emc); |
514 | |
515 | for (i = 0; i < emc->num_channels; i++) |
516 | tegra210_emc_wait_for_update(emc, channel: i, EMC_EMC_STATUS, |
517 | EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, |
518 | state: 0); |
519 | |
520 | for (i = 0; i < emc->num_channels; i++) |
521 | tegra210_emc_wait_for_update(emc, channel: i, EMC_EMC_STATUS, |
522 | EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, |
523 | state: 0); |
524 | |
525 | emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); |
526 | value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK; |
527 | value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT); |
528 | emc_writel(emc, value, EMC_CFG_UPDATE); |
529 | |
530 | /* |
531 | * 2. osc kick off - this assumes training and dvfs have set |
532 | * correct MR23. |
533 | */ |
534 | tegra210_emc_start_periodic_compensation(emc); |
535 | |
536 | /* |
537 | * 3. Let dram capture its clock tree delays. |
538 | */ |
539 | delay = tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
540 | delay *= 1000; |
541 | delay /= last->rate + 1; |
542 | udelay(delay); |
543 | |
544 | /* |
545 | * 4. Check delta wrt previous values (save value if margin |
546 | * exceeds what is set in table). |
547 | */ |
548 | del = periodic_compensation_handler(emc, |
549 | type: PERIODIC_TRAINING_SEQUENCE, |
550 | last, next: last); |
551 | |
552 | /* |
553 | * 5. Apply compensation w.r.t. trained values (if clock tree |
554 | * has drifted more than the set margin). |
555 | */ |
556 | if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) { |
557 | for (i = 0; i < items; i++) { |
558 | value = tegra210_emc_compensate(next: last, offset: list[i]); |
559 | emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n" , |
560 | list[i], value); |
561 | emc_writel(emc, value, offset: list[i]); |
562 | } |
563 | } |
564 | |
565 | emc_writel(emc, value: emc_cfg_o, EMC_CFG); |
566 | |
567 | /* |
568 | * 6. Timing update actally applies the new trimmers. |
569 | */ |
570 | tegra210_emc_timing_update(emc); |
571 | |
572 | /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */ |
573 | emc_writel(emc, value: emc_cfg_update, EMC_CFG_UPDATE); |
574 | |
575 | /* 6.2. Restore the DLL. */ |
576 | tegra210_emc_dll_enable(emc); |
577 | } |
578 | |
579 | return 0; |
580 | } |
581 | |
582 | /* |
583 | * Do the clock change sequence. |
584 | */ |
585 | static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) |
586 | { |
587 | /* state variables */ |
588 | static bool fsp_for_next_freq; |
589 | /* constant configuration parameters */ |
590 | const bool save_restore_clkstop_pd = true; |
591 | const u32 zqcal_before_cc_cutoff = 2400; |
592 | const bool cya_allow_ref_cc = false; |
593 | const bool cya_issue_pc_ref = false; |
594 | const bool opt_cc_short_zcal = true; |
595 | const bool ref_b4_sref_en = false; |
596 | const u32 tZQCAL_lpddr4 = 1000000; |
597 | const bool opt_short_zcal = true; |
598 | const bool opt_do_sw_qrst = true; |
599 | const u32 opt_dvfs_mode = MAN_SR; |
600 | /* |
601 | * This is the timing table for the source frequency. It does _not_ |
602 | * necessarily correspond to the actual timing values in the EMC at the |
603 | * moment. If the boot BCT differs from the table then this can happen. |
604 | * However, we need it for accessing the dram_timings (which are not |
605 | * really registers) array for the current frequency. |
606 | */ |
607 | struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; |
608 | u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST; |
609 | u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait; |
610 | u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj; |
611 | u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl; |
612 | u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; |
613 | u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg; |
614 | bool opt_zcal_en_cc = false, is_lpddr3 = false; |
615 | bool compensate_trimmer_applicable = false; |
616 | u32 emc_dbg, emc_cfg_pipe_clk, emc_pin; |
617 | u32 src_clk_period, dst_clk_period; /* in picoseconds */ |
618 | bool shared_zq_resistor = false; |
619 | u32 value, dram_type; |
620 | u32 opt_dll_mode = 0; |
621 | unsigned long delay; |
622 | unsigned int i; |
623 | |
624 | emc_dbg(emc, INFO, "Running clock change.\n" ); |
625 | |
626 | /* XXX fake == last */ |
627 | fake = tegra210_emc_find_timing(emc, rate: last->rate * 1000UL); |
628 | fsp_for_next_freq = !fsp_for_next_freq; |
629 | |
630 | value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; |
631 | dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT; |
632 | |
633 | if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) |
634 | shared_zq_resistor = true; |
635 | |
636 | if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && |
637 | last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || |
638 | dram_type == DRAM_TYPE_LPDDR4) |
639 | opt_zcal_en_cc = true; |
640 | |
641 | if (dram_type == DRAM_TYPE_DDR3) |
642 | opt_dll_mode = tegra210_emc_get_dll_state(next); |
643 | |
644 | if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && |
645 | (dram_type == DRAM_TYPE_LPDDR2)) |
646 | is_lpddr3 = true; |
647 | |
648 | emc_readl(emc, EMC_CFG); |
649 | emc_readl(emc, EMC_AUTO_CAL_CONFIG); |
650 | |
651 | src_clk_period = 1000000000 / last->rate; |
652 | dst_clk_period = 1000000000 / next->rate; |
653 | |
654 | if (dst_clk_period <= zqcal_before_cc_cutoff) |
655 | tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; |
656 | else |
657 | tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4; |
658 | |
659 | tZQCAL_lpddr4_fc_adj /= dst_clk_period; |
660 | |
661 | emc_dbg = emc_readl(emc, EMC_DBG); |
662 | emc_pin = emc_readl(emc, EMC_PIN); |
663 | emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); |
664 | |
665 | emc_cfg = next->burst_regs[EMC_CFG_INDEX]; |
666 | emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | |
667 | EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD); |
668 | emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; |
669 | emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | |
670 | EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | |
671 | EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | |
672 | EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | |
673 | EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN); |
674 | |
675 | emc_dbg(emc, INFO, "Clock change version: %d\n" , |
676 | DVFS_CLOCK_CHANGE_VERSION); |
677 | emc_dbg(emc, INFO, "DRAM type = %d\n" , dram_type); |
678 | emc_dbg(emc, INFO, "DRAM dev #: %u\n" , emc->num_devices); |
679 | emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n" , clksrc); |
680 | emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n" , next->dll_clk_src); |
681 | emc_dbg(emc, INFO, "last rate: %u, next rate %u\n" , last->rate, |
682 | next->rate); |
683 | emc_dbg(emc, INFO, "last period: %u, next period: %u\n" , |
684 | src_clk_period, dst_clk_period); |
685 | emc_dbg(emc, INFO, " shared_zq_resistor: %d\n" , !!shared_zq_resistor); |
686 | emc_dbg(emc, INFO, " num_channels: %u\n" , emc->num_channels); |
687 | emc_dbg(emc, INFO, " opt_dll_mode: %d\n" , opt_dll_mode); |
688 | |
689 | /* |
690 | * Step 1: |
691 | * Pre DVFS SW sequence. |
692 | */ |
693 | emc_dbg(emc, STEPS, "Step 1\n" ); |
694 | emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n" ); |
695 | |
696 | value = emc_readl(emc, EMC_CFG_DIG_DLL); |
697 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; |
698 | emc_writel(emc, value, EMC_CFG_DIG_DLL); |
699 | |
700 | tegra210_emc_timing_update(emc); |
701 | |
702 | for (i = 0; i < emc->num_channels; i++) |
703 | tegra210_emc_wait_for_update(emc, channel: i, EMC_CFG_DIG_DLL, |
704 | EMC_CFG_DIG_DLL_CFG_DLL_EN, state: 0); |
705 | |
706 | emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n" ); |
707 | |
708 | emc_auto_cal_config = next->emc_auto_cal_config; |
709 | auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE; |
710 | emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START; |
711 | emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL; |
712 | emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL; |
713 | emc_auto_cal_config |= auto_cal_en; |
714 | emc_writel(emc, value: emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); |
715 | emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ |
716 | |
717 | emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n" ); |
718 | |
719 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
720 | emc_writel(emc, value: emc_cfg, EMC_CFG); |
721 | emc_writel(emc, value: emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); |
722 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
723 | |
724 | if (next->periodic_training) { |
725 | tegra210_emc_reset_dram_clktree_values(timing: next); |
726 | |
727 | for (i = 0; i < emc->num_channels; i++) |
728 | tegra210_emc_wait_for_update(emc, channel: i, EMC_EMC_STATUS, |
729 | EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, |
730 | state: 0); |
731 | |
732 | for (i = 0; i < emc->num_channels; i++) |
733 | tegra210_emc_wait_for_update(emc, channel: i, EMC_EMC_STATUS, |
734 | EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, |
735 | state: 0); |
736 | |
737 | tegra210_emc_start_periodic_compensation(emc); |
738 | |
739 | delay = 1000 * tegra210_emc_actual_osc_clocks(in: last->run_clocks); |
740 | udelay((delay / last->rate) + 2); |
741 | |
742 | value = periodic_compensation_handler(emc, type: DVFS_SEQUENCE, last: fake, |
743 | next); |
744 | value = (value * 128 * next->rate / 1000) / 1000000; |
745 | |
746 | if (next->periodic_training && value > next->tree_margin) |
747 | compensate_trimmer_applicable = true; |
748 | } |
749 | |
750 | emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); |
751 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
752 | emc_writel(emc, value: emc_cfg, EMC_CFG); |
753 | emc_writel(emc, value: emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); |
754 | emc_writel(emc, value: emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, |
755 | EMC_CFG_PIPE_CLK); |
756 | emc_writel(emc, value: next->emc_fdpd_ctrl_cmd_no_ramp & |
757 | ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, |
758 | EMC_FDPD_CTRL_CMD_NO_RAMP); |
759 | |
760 | bg_reg_mode_change = |
761 | ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
762 | EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ |
763 | (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
764 | EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || |
765 | ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
766 | EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ |
767 | (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
768 | EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD)); |
769 | enable_bglp_reg = |
770 | (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
771 | EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0; |
772 | enable_bg_reg = |
773 | (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
774 | EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0; |
775 | |
776 | if (bg_reg_mode_change) { |
777 | if (enable_bg_reg) |
778 | emc_writel(emc, value: last->burst_regs |
779 | [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
780 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, |
781 | EMC_PMACRO_BG_BIAS_CTRL_0); |
782 | |
783 | if (enable_bglp_reg) |
784 | emc_writel(emc, value: last->burst_regs |
785 | [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
786 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, |
787 | EMC_PMACRO_BG_BIAS_CTRL_0); |
788 | } |
789 | |
790 | /* Check if we need to turn on VREF generator. */ |
791 | if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & |
792 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) && |
793 | ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & |
794 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) || |
795 | (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & |
796 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) && |
797 | ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & |
798 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) { |
799 | u32 pad_tx_ctrl = |
800 | next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; |
801 | u32 last_pad_tx_ctrl = |
802 | last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; |
803 | u32 next_dq_e_ivref, next_dqs_e_ivref; |
804 | |
805 | next_dqs_e_ivref = pad_tx_ctrl & |
806 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF; |
807 | next_dq_e_ivref = pad_tx_ctrl & |
808 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF; |
809 | value = (last_pad_tx_ctrl & |
810 | ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & |
811 | ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | |
812 | next_dq_e_ivref | next_dqs_e_ivref; |
813 | emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); |
814 | udelay(1); |
815 | } else if (bg_reg_mode_change) { |
816 | udelay(1); |
817 | } |
818 | |
819 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
820 | |
821 | /* |
822 | * Step 2: |
823 | * Prelock the DLL. |
824 | */ |
825 | emc_dbg(emc, STEPS, "Step 2\n" ); |
826 | |
827 | if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & |
828 | EMC_CFG_DIG_DLL_CFG_DLL_EN) { |
829 | emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n" ); |
830 | value = tegra210_emc_dll_prelock(emc, clksrc); |
831 | emc_dbg(emc, INFO, "DLL out: 0x%03x\n" , value); |
832 | } else { |
833 | emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n" ); |
834 | tegra210_emc_dll_disable(emc); |
835 | } |
836 | |
837 | /* |
838 | * Step 3: |
839 | * Prepare autocal for the clock change. |
840 | */ |
841 | emc_dbg(emc, STEPS, "Step 3\n" ); |
842 | |
843 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
844 | emc_writel(emc, value: next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); |
845 | emc_writel(emc, value: next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); |
846 | emc_writel(emc, value: next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); |
847 | emc_writel(emc, value: next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); |
848 | emc_writel(emc, value: next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); |
849 | emc_writel(emc, value: next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); |
850 | emc_writel(emc, value: next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); |
851 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
852 | |
853 | emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | |
854 | auto_cal_en); |
855 | emc_writel(emc, value: emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); |
856 | |
857 | /* |
858 | * Step 4: |
859 | * Update EMC_CFG. (??) |
860 | */ |
861 | emc_dbg(emc, STEPS, "Step 4\n" ); |
862 | |
863 | if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4) |
864 | ccfifo_writel(emc, value: 1, EMC_SELF_REF, delay: 0); |
865 | else |
866 | emc_writel(emc, value: next->emc_cfg_2, EMC_CFG_2); |
867 | |
868 | /* |
869 | * Step 5: |
870 | * Prepare reference variables for ZQCAL regs. |
871 | */ |
872 | emc_dbg(emc, STEPS, "Step 5\n" ); |
873 | |
874 | if (dram_type == DRAM_TYPE_LPDDR4) |
875 | zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period)); |
876 | else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3) |
877 | zq_wait_long = max(next->min_mrs_wait, |
878 | div_o3(360000, dst_clk_period)) + 4; |
879 | else if (dram_type == DRAM_TYPE_DDR3) |
880 | zq_wait_long = max((u32)256, |
881 | div_o3(320000, dst_clk_period) + 2); |
882 | else |
883 | zq_wait_long = 0; |
884 | |
885 | /* |
886 | * Step 6: |
887 | * Training code - removed. |
888 | */ |
889 | emc_dbg(emc, STEPS, "Step 6\n" ); |
890 | |
891 | /* |
892 | * Step 7: |
893 | * Program FSP reference registers and send MRWs to new FSPWR. |
894 | */ |
895 | emc_dbg(emc, STEPS, "Step 7\n" ); |
896 | emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P" ); |
897 | |
898 | /* WAR 200024907 */ |
899 | if (dram_type == DRAM_TYPE_LPDDR4) { |
900 | u32 nRTP = 16; |
901 | |
902 | if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */ |
903 | nRTP = 14; |
904 | |
905 | if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */ |
906 | nRTP = 12; |
907 | |
908 | if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */ |
909 | nRTP = 10; |
910 | |
911 | if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */ |
912 | nRTP = 8; |
913 | |
914 | deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8); |
915 | |
916 | /* |
917 | * Originally there was a + .5 in the tRPST calculation. |
918 | * However since we can't do FP in the kernel and the tRTM |
919 | * computation was in a floating point ceiling function, adding |
920 | * one to tRTP should be ok. There is no other source of non |
921 | * integer values, so the result was always going to be |
922 | * something for the form: f_ceil(N + .5) = N + 1; |
923 | */ |
924 | tRPST = (last->emc_mrw & 0x80) >> 7; |
925 | tRTM = fake->dram_timings[RL] + div_o3(a: 3600, b: src_clk_period) + |
926 | max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST + |
927 | 1 + nRTP; |
928 | |
929 | emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n" , tRTM, |
930 | next->burst_regs[EMC_RP_INDEX]); |
931 | |
932 | if (last->burst_regs[EMC_RP_INDEX] < tRTM) { |
933 | if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + |
934 | last->burst_regs[EMC_RP_INDEX])) { |
935 | R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; |
936 | RP_war = last->burst_regs[EMC_RP_INDEX]; |
937 | TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; |
938 | |
939 | if (R2P_war > 63) { |
940 | RP_war = R2P_war + |
941 | last->burst_regs[EMC_RP_INDEX] - 63; |
942 | |
943 | if (TRPab_war < RP_war) |
944 | TRPab_war = RP_war; |
945 | |
946 | R2P_war = 63; |
947 | } |
948 | } else { |
949 | R2P_war = last->burst_regs[EMC_R2P_INDEX]; |
950 | RP_war = last->burst_regs[EMC_RP_INDEX]; |
951 | TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; |
952 | } |
953 | |
954 | if (RP_war < deltaTWATM) { |
955 | W2P_war = last->burst_regs[EMC_W2P_INDEX] |
956 | + deltaTWATM - RP_war; |
957 | if (W2P_war > 63) { |
958 | RP_war = RP_war + W2P_war - 63; |
959 | if (TRPab_war < RP_war) |
960 | TRPab_war = RP_war; |
961 | W2P_war = 63; |
962 | } |
963 | } else { |
964 | W2P_war = last->burst_regs[ |
965 | EMC_W2P_INDEX]; |
966 | } |
967 | |
968 | if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || |
969 | (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || |
970 | (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || |
971 | (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { |
972 | emc_writel(emc, value: RP_war, EMC_RP); |
973 | emc_writel(emc, value: R2P_war, EMC_R2P); |
974 | emc_writel(emc, value: W2P_war, EMC_W2P); |
975 | emc_writel(emc, value: TRPab_war, EMC_TRPAB); |
976 | } |
977 | |
978 | tegra210_emc_timing_update(emc); |
979 | } else { |
980 | emc_dbg(emc, INFO, "Skipped WAR\n" ); |
981 | } |
982 | } |
983 | |
984 | if (!fsp_for_next_freq) { |
985 | mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; |
986 | mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; |
987 | } else { |
988 | mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; |
989 | mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; |
990 | } |
991 | |
992 | if (dram_type == DRAM_TYPE_LPDDR4) { |
993 | emc_writel(emc, value: mr13_flip_fspwr, EMC_MRW3); |
994 | emc_writel(emc, value: next->emc_mrw, EMC_MRW); |
995 | emc_writel(emc, value: next->emc_mrw2, EMC_MRW2); |
996 | } |
997 | |
998 | /* |
999 | * Step 8: |
1000 | * Program the shadow registers. |
1001 | */ |
1002 | emc_dbg(emc, STEPS, "Step 8\n" ); |
1003 | emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n" ); |
1004 | |
1005 | for (i = 0; i < next->num_burst; i++) { |
1006 | const u16 *offsets = emc->offsets->burst; |
1007 | u16 offset; |
1008 | |
1009 | if (!offsets[i]) |
1010 | continue; |
1011 | |
1012 | value = next->burst_regs[i]; |
1013 | offset = offsets[i]; |
1014 | |
1015 | if (dram_type != DRAM_TYPE_LPDDR4 && |
1016 | (offset == EMC_MRW6 || offset == EMC_MRW7 || |
1017 | offset == EMC_MRW8 || offset == EMC_MRW9 || |
1018 | offset == EMC_MRW10 || offset == EMC_MRW11 || |
1019 | offset == EMC_MRW12 || offset == EMC_MRW13 || |
1020 | offset == EMC_MRW14 || offset == EMC_MRW15 || |
1021 | offset == EMC_TRAINING_CTRL)) |
1022 | continue; |
1023 | |
1024 | /* Pain... And suffering. */ |
1025 | if (offset == EMC_CFG) { |
1026 | value &= ~EMC_CFG_DRAM_ACPD; |
1027 | value &= ~EMC_CFG_DYN_SELF_REF; |
1028 | |
1029 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1030 | value &= ~EMC_CFG_DRAM_CLKSTOP_SR; |
1031 | value &= ~EMC_CFG_DRAM_CLKSTOP_PD; |
1032 | } |
1033 | } else if (offset == EMC_MRS_WAIT_CNT && |
1034 | dram_type == DRAM_TYPE_LPDDR2 && |
1035 | opt_zcal_en_cc && !opt_cc_short_zcal && |
1036 | opt_short_zcal) { |
1037 | value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK << |
1038 | EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) | |
1039 | ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) << |
1040 | EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); |
1041 | } else if (offset == EMC_ZCAL_WAIT_CNT && |
1042 | dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc && |
1043 | !opt_cc_short_zcal && opt_short_zcal) { |
1044 | value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK << |
1045 | EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) | |
1046 | ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << |
1047 | EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); |
1048 | } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) { |
1049 | value = 0; /* EMC_ZCAL_INTERVAL reset value. */ |
1050 | } else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) { |
1051 | value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS; |
1052 | } else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) { |
1053 | value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | |
1054 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | |
1055 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | |
1056 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); |
1057 | } else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) { |
1058 | value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; |
1059 | value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | |
1060 | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | |
1061 | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | |
1062 | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); |
1063 | } else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) { |
1064 | value &= 0xf800f800; |
1065 | } else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) { |
1066 | value &= 0xfffffff0; |
1067 | } |
1068 | |
1069 | emc_writel(emc, value, offset); |
1070 | } |
1071 | |
1072 | /* SW addition: do EMC refresh adjustment here. */ |
1073 | tegra210_emc_adjust_timing(emc, timing: next); |
1074 | |
1075 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1076 | value = (23 << EMC_MRW_MRW_MA_SHIFT) | |
1077 | (next->run_clocks & EMC_MRW_MRW_OP_MASK); |
1078 | emc_writel(emc, value, EMC_MRW); |
1079 | } |
1080 | |
1081 | /* Per channel burst registers. */ |
1082 | emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n" ); |
1083 | |
1084 | for (i = 0; i < next->num_burst_per_ch; i++) { |
1085 | const struct tegra210_emc_per_channel_regs *burst = |
1086 | emc->offsets->burst_per_channel; |
1087 | |
1088 | if (!burst[i].offset) |
1089 | continue; |
1090 | |
1091 | if (dram_type != DRAM_TYPE_LPDDR4 && |
1092 | (burst[i].offset == EMC_MRW6 || |
1093 | burst[i].offset == EMC_MRW7 || |
1094 | burst[i].offset == EMC_MRW8 || |
1095 | burst[i].offset == EMC_MRW9 || |
1096 | burst[i].offset == EMC_MRW10 || |
1097 | burst[i].offset == EMC_MRW11 || |
1098 | burst[i].offset == EMC_MRW12 || |
1099 | burst[i].offset == EMC_MRW13 || |
1100 | burst[i].offset == EMC_MRW14 || |
1101 | burst[i].offset == EMC_MRW15)) |
1102 | continue; |
1103 | |
1104 | /* Filter out second channel if not in DUAL_CHANNEL mode. */ |
1105 | if (emc->num_channels < 2 && burst[i].bank >= 1) |
1106 | continue; |
1107 | |
1108 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1109 | next->burst_reg_per_ch[i], burst[i].offset); |
1110 | emc_channel_writel(emc, channel: burst[i].bank, |
1111 | value: next->burst_reg_per_ch[i], |
1112 | offset: burst[i].offset); |
1113 | } |
1114 | |
1115 | /* Vref regs. */ |
1116 | emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n" ); |
1117 | |
1118 | for (i = 0; i < next->vref_num; i++) { |
1119 | const struct tegra210_emc_per_channel_regs *vref = |
1120 | emc->offsets->vref_per_channel; |
1121 | |
1122 | if (!vref[i].offset) |
1123 | continue; |
1124 | |
1125 | if (emc->num_channels < 2 && vref[i].bank >= 1) |
1126 | continue; |
1127 | |
1128 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1129 | next->vref_perch_regs[i], vref[i].offset); |
1130 | emc_channel_writel(emc, channel: vref[i].bank, value: next->vref_perch_regs[i], |
1131 | offset: vref[i].offset); |
1132 | } |
1133 | |
1134 | /* Trimmers. */ |
1135 | emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n" ); |
1136 | |
1137 | for (i = 0; i < next->num_trim; i++) { |
1138 | const u16 *offsets = emc->offsets->trim; |
1139 | |
1140 | if (!offsets[i]) |
1141 | continue; |
1142 | |
1143 | if (compensate_trimmer_applicable && |
1144 | (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || |
1145 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || |
1146 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || |
1147 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || |
1148 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || |
1149 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || |
1150 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || |
1151 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || |
1152 | offsets[i] == EMC_DATA_BRLSHFT_0 || |
1153 | offsets[i] == EMC_DATA_BRLSHFT_1)) { |
1154 | value = tegra210_emc_compensate(next, offset: offsets[i]); |
1155 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1156 | value, offsets[i]); |
1157 | emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n" , |
1158 | (u32)(u64)offsets[i], value); |
1159 | emc_writel(emc, value, offset: offsets[i]); |
1160 | } else { |
1161 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1162 | next->trim_regs[i], offsets[i]); |
1163 | emc_writel(emc, value: next->trim_regs[i], offset: offsets[i]); |
1164 | } |
1165 | } |
1166 | |
1167 | /* Per channel trimmers. */ |
1168 | emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n" ); |
1169 | |
1170 | for (i = 0; i < next->num_trim_per_ch; i++) { |
1171 | const struct tegra210_emc_per_channel_regs *trim = |
1172 | &emc->offsets->trim_per_channel[0]; |
1173 | unsigned int offset; |
1174 | |
1175 | if (!trim[i].offset) |
1176 | continue; |
1177 | |
1178 | if (emc->num_channels < 2 && trim[i].bank >= 1) |
1179 | continue; |
1180 | |
1181 | offset = trim[i].offset; |
1182 | |
1183 | if (compensate_trimmer_applicable && |
1184 | (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || |
1185 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || |
1186 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || |
1187 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || |
1188 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || |
1189 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || |
1190 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || |
1191 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || |
1192 | offset == EMC_DATA_BRLSHFT_0 || |
1193 | offset == EMC_DATA_BRLSHFT_1)) { |
1194 | value = tegra210_emc_compensate(next, offset); |
1195 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1196 | value, offset); |
1197 | emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n" , offset, |
1198 | value); |
1199 | emc_channel_writel(emc, channel: trim[i].bank, value, offset); |
1200 | } else { |
1201 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1202 | next->trim_perch_regs[i], offset); |
1203 | emc_channel_writel(emc, channel: trim[i].bank, |
1204 | value: next->trim_perch_regs[i], offset); |
1205 | } |
1206 | } |
1207 | |
1208 | emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n" ); |
1209 | |
1210 | for (i = 0; i < next->num_mc_regs; i++) { |
1211 | const u16 *offsets = emc->offsets->burst_mc; |
1212 | u32 *values = next->burst_mc_regs; |
1213 | |
1214 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1215 | values[i], offsets[i]); |
1216 | mc_writel(mc: emc->mc, value: values[i], offset: offsets[i]); |
1217 | } |
1218 | |
1219 | /* Registers to be programmed on the faster clock. */ |
1220 | if (next->rate < last->rate) { |
1221 | const u16 *la = emc->offsets->la_scale; |
1222 | |
1223 | emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n" ); |
1224 | |
1225 | for (i = 0; i < next->num_up_down; i++) { |
1226 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n" , i, |
1227 | next->la_scale_regs[i], la[i]); |
1228 | mc_writel(mc: emc->mc, value: next->la_scale_regs[i], offset: la[i]); |
1229 | } |
1230 | } |
1231 | |
1232 | /* Flush all the burst register writes. */ |
1233 | mc_readl(mc: emc->mc, MC_EMEM_ADR_CFG); |
1234 | |
1235 | /* |
1236 | * Step 9: |
1237 | * LPDDR4 section A. |
1238 | */ |
1239 | emc_dbg(emc, STEPS, "Step 9\n" ); |
1240 | |
1241 | value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; |
1242 | value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK; |
1243 | |
1244 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1245 | emc_writel(emc, value: 0, EMC_ZCAL_INTERVAL); |
1246 | emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); |
1247 | |
1248 | value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | |
1249 | EMC_DBG_WRITE_ACTIVE_ONLY); |
1250 | |
1251 | emc_writel(emc, value, EMC_DBG); |
1252 | emc_writel(emc, value: 0, EMC_ZCAL_INTERVAL); |
1253 | emc_writel(emc, value: emc_dbg, EMC_DBG); |
1254 | } |
1255 | |
1256 | /* |
1257 | * Step 10: |
1258 | * LPDDR4 and DDR3 common section. |
1259 | */ |
1260 | emc_dbg(emc, STEPS, "Step 10\n" ); |
1261 | |
1262 | if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) { |
1263 | if (dram_type == DRAM_TYPE_LPDDR4) |
1264 | ccfifo_writel(emc, value: 0x101, EMC_SELF_REF, delay: 0); |
1265 | else |
1266 | ccfifo_writel(emc, value: 0x1, EMC_SELF_REF, delay: 0); |
1267 | |
1268 | if (dram_type == DRAM_TYPE_LPDDR4 && |
1269 | dst_clk_period <= zqcal_before_cc_cutoff) { |
1270 | ccfifo_writel(emc, value: mr13_flip_fspwr ^ 0x40, EMC_MRW3, delay: 0); |
1271 | ccfifo_writel(emc, value: (next->burst_regs[EMC_MRW6_INDEX] & |
1272 | 0xFFFF3F3F) | |
1273 | (last->burst_regs[EMC_MRW6_INDEX] & |
1274 | 0x0000C0C0), EMC_MRW6, delay: 0); |
1275 | ccfifo_writel(emc, value: (next->burst_regs[EMC_MRW14_INDEX] & |
1276 | 0xFFFF0707) | |
1277 | (last->burst_regs[EMC_MRW14_INDEX] & |
1278 | 0x00003838), EMC_MRW14, delay: 0); |
1279 | |
1280 | if (emc->num_devices > 1) { |
1281 | ccfifo_writel(emc, |
1282 | value: (next->burst_regs[EMC_MRW7_INDEX] & |
1283 | 0xFFFF3F3F) | |
1284 | (last->burst_regs[EMC_MRW7_INDEX] & |
1285 | 0x0000C0C0), EMC_MRW7, delay: 0); |
1286 | ccfifo_writel(emc, |
1287 | value: (next->burst_regs[EMC_MRW15_INDEX] & |
1288 | 0xFFFF0707) | |
1289 | (last->burst_regs[EMC_MRW15_INDEX] & |
1290 | 0x00003838), EMC_MRW15, delay: 0); |
1291 | } |
1292 | |
1293 | if (opt_zcal_en_cc) { |
1294 | if (emc->num_devices < 2) |
1295 | ccfifo_writel(emc, |
1296 | value: 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1297 | | EMC_ZQ_CAL_ZQ_CAL_CMD, |
1298 | EMC_ZQ_CAL, delay: 0); |
1299 | else if (shared_zq_resistor) |
1300 | ccfifo_writel(emc, |
1301 | value: 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
1302 | | EMC_ZQ_CAL_ZQ_CAL_CMD, |
1303 | EMC_ZQ_CAL, delay: 0); |
1304 | else |
1305 | ccfifo_writel(emc, |
1306 | EMC_ZQ_CAL_ZQ_CAL_CMD, |
1307 | EMC_ZQ_CAL, delay: 0); |
1308 | } |
1309 | } |
1310 | } |
1311 | |
1312 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1313 | value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; |
1314 | ccfifo_writel(emc, value: mr13_flip_fspop | 0x8, EMC_MRW3, delay: value); |
1315 | ccfifo_writel(emc, value: 0, offset: 0, delay: tFC_lpddr4 / src_clk_period); |
1316 | } |
1317 | |
1318 | if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) { |
1319 | delay = 30; |
1320 | |
1321 | if (cya_allow_ref_cc) { |
1322 | delay += (1000 * fake->dram_timings[T_RP]) / |
1323 | src_clk_period; |
1324 | delay += 4000 * fake->dram_timings[T_RFC]; |
1325 | } |
1326 | |
1327 | ccfifo_writel(emc, value: emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | |
1328 | EMC_PIN_PIN_CKEB | |
1329 | EMC_PIN_PIN_CKE), |
1330 | EMC_PIN, delay); |
1331 | } |
1332 | |
1333 | /* calculate reference delay multiplier */ |
1334 | value = 1; |
1335 | |
1336 | if (ref_b4_sref_en) |
1337 | value++; |
1338 | |
1339 | if (cya_allow_ref_cc) |
1340 | value++; |
1341 | |
1342 | if (cya_issue_pc_ref) |
1343 | value++; |
1344 | |
1345 | if (dram_type != DRAM_TYPE_LPDDR4) { |
1346 | delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + |
1347 | (1000 * fake->dram_timings[T_RFC] / src_clk_period)); |
1348 | delay = value * delay + 20; |
1349 | } else { |
1350 | delay = 0; |
1351 | } |
1352 | |
1353 | /* |
1354 | * Step 11: |
1355 | * Ramp down. |
1356 | */ |
1357 | emc_dbg(emc, STEPS, "Step 11\n" ); |
1358 | |
1359 | ccfifo_writel(emc, value: 0x0, EMC_CFG_SYNC, delay); |
1360 | |
1361 | value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY; |
1362 | ccfifo_writel(emc, value, EMC_DBG, delay: 0); |
1363 | |
1364 | ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, clk: src_clk_period, |
1365 | flip_backward: 0); |
1366 | |
1367 | /* |
1368 | * Step 12: |
1369 | * And finally - trigger the clock change. |
1370 | */ |
1371 | emc_dbg(emc, STEPS, "Step 12\n" ); |
1372 | |
1373 | ccfifo_writel(emc, value: 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, delay: 0); |
1374 | value &= ~EMC_DBG_WRITE_ACTIVE_ONLY; |
1375 | ccfifo_writel(emc, value, EMC_DBG, delay: 0); |
1376 | |
1377 | /* |
1378 | * Step 13: |
1379 | * Ramp up. |
1380 | */ |
1381 | emc_dbg(emc, STEPS, "Step 13\n" ); |
1382 | |
1383 | ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, clk: dst_clk_period, flip_backward: 0); |
1384 | ccfifo_writel(emc, value: emc_dbg, EMC_DBG, delay: 0); |
1385 | |
1386 | /* |
1387 | * Step 14: |
1388 | * Bringup CKE pins. |
1389 | */ |
1390 | emc_dbg(emc, STEPS, "Step 14\n" ); |
1391 | |
1392 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1393 | value = emc_pin | EMC_PIN_PIN_CKE; |
1394 | |
1395 | if (emc->num_devices <= 1) |
1396 | value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV); |
1397 | else |
1398 | value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV; |
1399 | |
1400 | ccfifo_writel(emc, value, EMC_PIN, delay: 0); |
1401 | } |
1402 | |
1403 | /* |
1404 | * Step 15: (two step 15s ??) |
1405 | * Calculate zqlatch wait time; has dependency on ramping times. |
1406 | */ |
1407 | emc_dbg(emc, STEPS, "Step 15\n" ); |
1408 | |
1409 | if (dst_clk_period <= zqcal_before_cc_cutoff) { |
1410 | s32 t = (s32)(ramp_up_wait + ramp_down_wait) / |
1411 | (s32)dst_clk_period; |
1412 | zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; |
1413 | } else { |
1414 | zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - |
1415 | div_o3(a: 1000 * next->dram_timings[T_PDEX], |
1416 | b: dst_clk_period); |
1417 | } |
1418 | |
1419 | emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n" , tZQCAL_lpddr4_fc_adj); |
1420 | emc_dbg(emc, INFO, "dst_clk_period = %u\n" , |
1421 | dst_clk_period); |
1422 | emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n" , |
1423 | next->dram_timings[T_PDEX]); |
1424 | emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n" , |
1425 | max_t(s32, 0, zq_latch_dvfs_wait_time)); |
1426 | |
1427 | if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) { |
1428 | delay = div_o3(a: 1000 * next->dram_timings[T_PDEX], |
1429 | b: dst_clk_period); |
1430 | |
1431 | if (emc->num_devices < 2) { |
1432 | if (dst_clk_period > zqcal_before_cc_cutoff) |
1433 | ccfifo_writel(emc, |
1434 | value: 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1435 | EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, |
1436 | delay); |
1437 | |
1438 | value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; |
1439 | ccfifo_writel(emc, value, EMC_MRW3, delay); |
1440 | ccfifo_writel(emc, value: 0, EMC_SELF_REF, delay: 0); |
1441 | ccfifo_writel(emc, value: 0, EMC_REF, delay: 0); |
1442 | ccfifo_writel(emc, value: 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1443 | EMC_ZQ_CAL_ZQ_LATCH_CMD, |
1444 | EMC_ZQ_CAL, |
1445 | max_t(s32, 0, zq_latch_dvfs_wait_time)); |
1446 | } else if (shared_zq_resistor) { |
1447 | if (dst_clk_period > zqcal_before_cc_cutoff) |
1448 | ccfifo_writel(emc, |
1449 | value: 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1450 | EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, |
1451 | delay); |
1452 | |
1453 | ccfifo_writel(emc, value: 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1454 | EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, |
1455 | max_t(s32, 0, zq_latch_dvfs_wait_time) + |
1456 | delay); |
1457 | ccfifo_writel(emc, value: 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1458 | EMC_ZQ_CAL_ZQ_LATCH_CMD, |
1459 | EMC_ZQ_CAL, delay: 0); |
1460 | |
1461 | value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; |
1462 | ccfifo_writel(emc, value, EMC_MRW3, delay: 0); |
1463 | ccfifo_writel(emc, value: 0, EMC_SELF_REF, delay: 0); |
1464 | ccfifo_writel(emc, value: 0, EMC_REF, delay: 0); |
1465 | |
1466 | ccfifo_writel(emc, value: 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1467 | EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, |
1468 | delay: tZQCAL_lpddr4 / dst_clk_period); |
1469 | } else { |
1470 | if (dst_clk_period > zqcal_before_cc_cutoff) |
1471 | ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, |
1472 | EMC_ZQ_CAL, delay); |
1473 | |
1474 | value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; |
1475 | ccfifo_writel(emc, value, EMC_MRW3, delay); |
1476 | ccfifo_writel(emc, value: 0, EMC_SELF_REF, delay: 0); |
1477 | ccfifo_writel(emc, value: 0, EMC_REF, delay: 0); |
1478 | |
1479 | ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, |
1480 | max_t(s32, 0, zq_latch_dvfs_wait_time)); |
1481 | } |
1482 | } |
1483 | |
1484 | /* WAR: delay for zqlatch */ |
1485 | ccfifo_writel(emc, value: 0, offset: 0, delay: 10); |
1486 | |
1487 | /* |
1488 | * Step 16: |
1489 | * LPDDR4 Conditional Training Kickoff. Removed. |
1490 | */ |
1491 | |
1492 | /* |
1493 | * Step 17: |
1494 | * MANSR exit self refresh. |
1495 | */ |
1496 | emc_dbg(emc, STEPS, "Step 17\n" ); |
1497 | |
1498 | if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4) |
1499 | ccfifo_writel(emc, value: 0, EMC_SELF_REF, delay: 0); |
1500 | |
1501 | /* |
1502 | * Step 18: |
1503 | * Send MRWs to LPDDR3/DDR3. |
1504 | */ |
1505 | emc_dbg(emc, STEPS, "Step 18\n" ); |
1506 | |
1507 | if (dram_type == DRAM_TYPE_LPDDR2) { |
1508 | ccfifo_writel(emc, value: next->emc_mrw2, EMC_MRW2, delay: 0); |
1509 | ccfifo_writel(emc, value: next->emc_mrw, EMC_MRW, delay: 0); |
1510 | if (is_lpddr3) |
1511 | ccfifo_writel(emc, value: next->emc_mrw4, EMC_MRW4, delay: 0); |
1512 | } else if (dram_type == DRAM_TYPE_DDR3) { |
1513 | if (opt_dll_mode) |
1514 | ccfifo_writel(emc, value: next->emc_emrs & |
1515 | ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, delay: 0); |
1516 | ccfifo_writel(emc, value: next->emc_emrs2 & |
1517 | ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, delay: 0); |
1518 | ccfifo_writel(emc, value: next->emc_mrs | |
1519 | EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, delay: 0); |
1520 | } |
1521 | |
1522 | /* |
1523 | * Step 19: |
1524 | * ZQCAL for LPDDR3/DDR3 |
1525 | */ |
1526 | emc_dbg(emc, STEPS, "Step 19\n" ); |
1527 | |
1528 | if (opt_zcal_en_cc) { |
1529 | if (dram_type == DRAM_TYPE_LPDDR2) { |
1530 | value = opt_cc_short_zcal ? 90000 : 360000; |
1531 | value = div_o3(a: value, b: dst_clk_period); |
1532 | value = value << |
1533 | EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | |
1534 | value << |
1535 | EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT; |
1536 | ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, delay: 0); |
1537 | |
1538 | value = opt_cc_short_zcal ? 0x56 : 0xab; |
1539 | ccfifo_writel(emc, value: 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | |
1540 | EMC_MRW_USE_MRW_EXT_CNT | |
1541 | 10 << EMC_MRW_MRW_MA_SHIFT | |
1542 | value << EMC_MRW_MRW_OP_SHIFT, |
1543 | EMC_MRW, delay: 0); |
1544 | |
1545 | if (emc->num_devices > 1) { |
1546 | value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | |
1547 | EMC_MRW_USE_MRW_EXT_CNT | |
1548 | 10 << EMC_MRW_MRW_MA_SHIFT | |
1549 | value << EMC_MRW_MRW_OP_SHIFT; |
1550 | ccfifo_writel(emc, value, EMC_MRW, delay: 0); |
1551 | } |
1552 | } else if (dram_type == DRAM_TYPE_DDR3) { |
1553 | value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG; |
1554 | |
1555 | ccfifo_writel(emc, value: value | |
1556 | 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1557 | EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, |
1558 | delay: 0); |
1559 | |
1560 | if (emc->num_devices > 1) { |
1561 | value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1562 | EMC_ZQ_CAL_ZQ_CAL_CMD; |
1563 | ccfifo_writel(emc, value, EMC_ZQ_CAL, delay: 0); |
1564 | } |
1565 | } |
1566 | } |
1567 | |
1568 | if (bg_reg_mode_change) { |
1569 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
1570 | |
1571 | if (ramp_up_wait <= 1250000) |
1572 | delay = (1250000 - ramp_up_wait) / dst_clk_period; |
1573 | else |
1574 | delay = 0; |
1575 | |
1576 | ccfifo_writel(emc, |
1577 | value: next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], |
1578 | EMC_PMACRO_BG_BIAS_CTRL_0, delay); |
1579 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
1580 | } |
1581 | |
1582 | /* |
1583 | * Step 20: |
1584 | * Issue ref and optional QRST. |
1585 | */ |
1586 | emc_dbg(emc, STEPS, "Step 20\n" ); |
1587 | |
1588 | if (dram_type != DRAM_TYPE_LPDDR4) |
1589 | ccfifo_writel(emc, value: 0, EMC_REF, delay: 0); |
1590 | |
1591 | if (opt_do_sw_qrst) { |
1592 | ccfifo_writel(emc, value: 1, EMC_ISSUE_QRST, delay: 0); |
1593 | ccfifo_writel(emc, value: 0, EMC_ISSUE_QRST, delay: 2); |
1594 | } |
1595 | |
1596 | /* |
1597 | * Step 21: |
1598 | * Restore ZCAL and ZCAL interval. |
1599 | */ |
1600 | emc_dbg(emc, STEPS, "Step 21\n" ); |
1601 | |
1602 | if (save_restore_clkstop_pd || opt_zcal_en_cc) { |
1603 | ccfifo_writel(emc, value: emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, |
1604 | EMC_DBG, delay: 0); |
1605 | if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4) |
1606 | ccfifo_writel(emc, value: next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], |
1607 | EMC_ZCAL_INTERVAL, delay: 0); |
1608 | |
1609 | if (save_restore_clkstop_pd) |
1610 | ccfifo_writel(emc, value: next->burst_regs[EMC_CFG_INDEX] & |
1611 | ~EMC_CFG_DYN_SELF_REF, |
1612 | EMC_CFG, delay: 0); |
1613 | ccfifo_writel(emc, value: emc_dbg, EMC_DBG, delay: 0); |
1614 | } |
1615 | |
1616 | /* |
1617 | * Step 22: |
1618 | * Restore EMC_CFG_PIPE_CLK. |
1619 | */ |
1620 | emc_dbg(emc, STEPS, "Step 22\n" ); |
1621 | |
1622 | ccfifo_writel(emc, value: emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, delay: 0); |
1623 | |
1624 | if (bg_reg_mode_change) { |
1625 | if (enable_bg_reg) |
1626 | emc_writel(emc, |
1627 | value: next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
1628 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, |
1629 | EMC_PMACRO_BG_BIAS_CTRL_0); |
1630 | else |
1631 | emc_writel(emc, |
1632 | value: next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & |
1633 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, |
1634 | EMC_PMACRO_BG_BIAS_CTRL_0); |
1635 | } |
1636 | |
1637 | /* |
1638 | * Step 23: |
1639 | */ |
1640 | emc_dbg(emc, STEPS, "Step 23\n" ); |
1641 | |
1642 | value = emc_readl(emc, EMC_CFG_DIG_DLL); |
1643 | value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; |
1644 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; |
1645 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; |
1646 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; |
1647 | value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | |
1648 | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); |
1649 | emc_writel(emc, value, EMC_CFG_DIG_DLL); |
1650 | |
1651 | tegra210_emc_do_clock_change(emc, clksrc); |
1652 | |
1653 | /* |
1654 | * Step 24: |
1655 | * Save training results. Removed. |
1656 | */ |
1657 | |
1658 | /* |
1659 | * Step 25: |
1660 | * Program MC updown registers. |
1661 | */ |
1662 | emc_dbg(emc, STEPS, "Step 25\n" ); |
1663 | |
1664 | if (next->rate > last->rate) { |
1665 | for (i = 0; i < next->num_up_down; i++) |
1666 | mc_writel(mc: emc->mc, value: next->la_scale_regs[i], |
1667 | offset: emc->offsets->la_scale[i]); |
1668 | |
1669 | tegra210_emc_timing_update(emc); |
1670 | } |
1671 | |
1672 | /* |
1673 | * Step 26: |
1674 | * Restore ZCAL registers. |
1675 | */ |
1676 | emc_dbg(emc, STEPS, "Step 26\n" ); |
1677 | |
1678 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1679 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
1680 | emc_writel(emc, value: next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], |
1681 | EMC_ZCAL_WAIT_CNT); |
1682 | emc_writel(emc, value: next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], |
1683 | EMC_ZCAL_INTERVAL); |
1684 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
1685 | } |
1686 | |
1687 | if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc && |
1688 | !opt_short_zcal && opt_cc_short_zcal) { |
1689 | udelay(2); |
1690 | |
1691 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
1692 | if (dram_type == DRAM_TYPE_LPDDR2) |
1693 | emc_writel(emc, value: next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], |
1694 | EMC_MRS_WAIT_CNT); |
1695 | else if (dram_type == DRAM_TYPE_DDR3) |
1696 | emc_writel(emc, value: next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], |
1697 | EMC_ZCAL_WAIT_CNT); |
1698 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
1699 | } |
1700 | |
1701 | /* |
1702 | * Step 27: |
1703 | * Restore EMC_CFG, FDPD registers. |
1704 | */ |
1705 | emc_dbg(emc, STEPS, "Step 27\n" ); |
1706 | |
1707 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
1708 | emc_writel(emc, value: next->burst_regs[EMC_CFG_INDEX], EMC_CFG); |
1709 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
1710 | emc_writel(emc, value: next->emc_fdpd_ctrl_cmd_no_ramp, |
1711 | EMC_FDPD_CTRL_CMD_NO_RAMP); |
1712 | emc_writel(emc, value: next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); |
1713 | |
1714 | /* |
1715 | * Step 28: |
1716 | * Training recover. Removed. |
1717 | */ |
1718 | emc_dbg(emc, STEPS, "Step 28\n" ); |
1719 | |
1720 | tegra210_emc_set_shadow_bypass(emc, set: ACTIVE); |
1721 | emc_writel(emc, |
1722 | value: next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], |
1723 | EMC_PMACRO_AUTOCAL_CFG_COMMON); |
1724 | tegra210_emc_set_shadow_bypass(emc, set: ASSEMBLY); |
1725 | |
1726 | /* |
1727 | * Step 29: |
1728 | * Power fix WAR. |
1729 | */ |
1730 | emc_dbg(emc, STEPS, "Step 29\n" ); |
1731 | |
1732 | emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | |
1733 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 | |
1734 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 | |
1735 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 | |
1736 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 | |
1737 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 | |
1738 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 | |
1739 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7, |
1740 | EMC_PMACRO_CFG_PM_GLOBAL_0); |
1741 | emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, |
1742 | EMC_PMACRO_TRAINING_CTRL_0); |
1743 | emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, |
1744 | EMC_PMACRO_TRAINING_CTRL_1); |
1745 | emc_writel(emc, value: 0, EMC_PMACRO_CFG_PM_GLOBAL_0); |
1746 | |
1747 | /* |
1748 | * Step 30: |
1749 | * Re-enable autocal. |
1750 | */ |
1751 | emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n" ); |
1752 | |
1753 | if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { |
1754 | value = emc_readl(emc, EMC_CFG_DIG_DLL); |
1755 | value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; |
1756 | value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; |
1757 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; |
1758 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; |
1759 | value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | |
1760 | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); |
1761 | emc_writel(emc, value, EMC_CFG_DIG_DLL); |
1762 | tegra210_emc_timing_update(emc); |
1763 | } |
1764 | |
1765 | emc_writel(emc, value: next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); |
1766 | |
1767 | /* Done! Yay. */ |
1768 | } |
1769 | |
1770 | const struct tegra210_emc_sequence tegra210_emc_r21021 = { |
1771 | .revision = 0x7, |
1772 | .set_clock = tegra210_emc_r21021_set_clock, |
1773 | .periodic_compensation = tegra210_emc_r21021_periodic_compensation, |
1774 | }; |
1775 | |