1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/clk.h> |
7 | #include <linux/delay.h> |
8 | #include <linux/dma-mapping.h> |
9 | #include <linux/export.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/kernel.h> |
12 | #include <linux/module.h> |
13 | #include <linux/of.h> |
14 | #include <linux/of_platform.h> |
15 | #include <linux/platform_device.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/sort.h> |
18 | #include <linux/tegra-icc.h> |
19 | |
20 | #include <soc/tegra/fuse.h> |
21 | |
22 | #include "mc.h" |
23 | |
24 | static const struct of_device_id tegra_mc_of_match[] = { |
25 | #ifdef CONFIG_ARCH_TEGRA_2x_SOC |
26 | { .compatible = "nvidia,tegra20-mc-gart" , .data = &tegra20_mc_soc }, |
27 | #endif |
28 | #ifdef CONFIG_ARCH_TEGRA_3x_SOC |
29 | { .compatible = "nvidia,tegra30-mc" , .data = &tegra30_mc_soc }, |
30 | #endif |
31 | #ifdef CONFIG_ARCH_TEGRA_114_SOC |
32 | { .compatible = "nvidia,tegra114-mc" , .data = &tegra114_mc_soc }, |
33 | #endif |
34 | #ifdef CONFIG_ARCH_TEGRA_124_SOC |
35 | { .compatible = "nvidia,tegra124-mc" , .data = &tegra124_mc_soc }, |
36 | #endif |
37 | #ifdef CONFIG_ARCH_TEGRA_132_SOC |
38 | { .compatible = "nvidia,tegra132-mc" , .data = &tegra132_mc_soc }, |
39 | #endif |
40 | #ifdef CONFIG_ARCH_TEGRA_210_SOC |
41 | { .compatible = "nvidia,tegra210-mc" , .data = &tegra210_mc_soc }, |
42 | #endif |
43 | #ifdef CONFIG_ARCH_TEGRA_186_SOC |
44 | { .compatible = "nvidia,tegra186-mc" , .data = &tegra186_mc_soc }, |
45 | #endif |
46 | #ifdef CONFIG_ARCH_TEGRA_194_SOC |
47 | { .compatible = "nvidia,tegra194-mc" , .data = &tegra194_mc_soc }, |
48 | #endif |
49 | #ifdef CONFIG_ARCH_TEGRA_234_SOC |
50 | { .compatible = "nvidia,tegra234-mc" , .data = &tegra234_mc_soc }, |
51 | #endif |
52 | { /* sentinel */ } |
53 | }; |
54 | MODULE_DEVICE_TABLE(of, tegra_mc_of_match); |
55 | |
56 | static void tegra_mc_devm_action_put_device(void *data) |
57 | { |
58 | struct tegra_mc *mc = data; |
59 | |
60 | put_device(dev: mc->dev); |
61 | } |
62 | |
63 | /** |
64 | * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle |
65 | * @dev: device pointer for the consumer device |
66 | * |
67 | * This function will search for the Memory Controller node in a device-tree |
68 | * and retrieve the Memory Controller handle. |
69 | * |
70 | * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc. |
71 | */ |
72 | struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev) |
73 | { |
74 | struct platform_device *pdev; |
75 | struct device_node *np; |
76 | struct tegra_mc *mc; |
77 | int err; |
78 | |
79 | np = of_parse_phandle(np: dev->of_node, phandle_name: "nvidia,memory-controller" , index: 0); |
80 | if (!np) |
81 | return ERR_PTR(error: -ENOENT); |
82 | |
83 | pdev = of_find_device_by_node(np); |
84 | of_node_put(node: np); |
85 | if (!pdev) |
86 | return ERR_PTR(error: -ENODEV); |
87 | |
88 | mc = platform_get_drvdata(pdev); |
89 | if (!mc) { |
90 | put_device(dev: &pdev->dev); |
91 | return ERR_PTR(error: -EPROBE_DEFER); |
92 | } |
93 | |
94 | err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc); |
95 | if (err) |
96 | return ERR_PTR(error: err); |
97 | |
98 | return mc; |
99 | } |
100 | EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get); |
101 | |
102 | int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev) |
103 | { |
104 | if (mc->soc->ops && mc->soc->ops->probe_device) |
105 | return mc->soc->ops->probe_device(mc, dev); |
106 | |
107 | return 0; |
108 | } |
109 | EXPORT_SYMBOL_GPL(tegra_mc_probe_device); |
110 | |
111 | int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id, |
112 | phys_addr_t *base, u64 *size) |
113 | { |
114 | u32 offset; |
115 | |
116 | if (id < 1 || id >= mc->soc->num_carveouts) |
117 | return -EINVAL; |
118 | |
119 | if (id < 6) |
120 | offset = 0xc0c + 0x50 * (id - 1); |
121 | else |
122 | offset = 0x2004 + 0x50 * (id - 6); |
123 | |
124 | *base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset: offset + 0x0); |
125 | #ifdef CONFIG_PHYS_ADDR_T_64BIT |
126 | *base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset: offset + 0x4) << 32; |
127 | #endif |
128 | |
129 | if (size) |
130 | *size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset: offset + 0x8) << 17; |
131 | |
132 | return 0; |
133 | } |
134 | EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info); |
135 | |
136 | static int tegra_mc_block_dma_common(struct tegra_mc *mc, |
137 | const struct tegra_mc_reset *rst) |
138 | { |
139 | unsigned long flags; |
140 | u32 value; |
141 | |
142 | spin_lock_irqsave(&mc->lock, flags); |
143 | |
144 | value = mc_readl(mc, offset: rst->control) | BIT(rst->bit); |
145 | mc_writel(mc, value, offset: rst->control); |
146 | |
147 | spin_unlock_irqrestore(lock: &mc->lock, flags); |
148 | |
149 | return 0; |
150 | } |
151 | |
152 | static bool tegra_mc_dma_idling_common(struct tegra_mc *mc, |
153 | const struct tegra_mc_reset *rst) |
154 | { |
155 | return (mc_readl(mc, offset: rst->status) & BIT(rst->bit)) != 0; |
156 | } |
157 | |
158 | static int tegra_mc_unblock_dma_common(struct tegra_mc *mc, |
159 | const struct tegra_mc_reset *rst) |
160 | { |
161 | unsigned long flags; |
162 | u32 value; |
163 | |
164 | spin_lock_irqsave(&mc->lock, flags); |
165 | |
166 | value = mc_readl(mc, offset: rst->control) & ~BIT(rst->bit); |
167 | mc_writel(mc, value, offset: rst->control); |
168 | |
169 | spin_unlock_irqrestore(lock: &mc->lock, flags); |
170 | |
171 | return 0; |
172 | } |
173 | |
174 | static int tegra_mc_reset_status_common(struct tegra_mc *mc, |
175 | const struct tegra_mc_reset *rst) |
176 | { |
177 | return (mc_readl(mc, offset: rst->control) & BIT(rst->bit)) != 0; |
178 | } |
179 | |
180 | const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = { |
181 | .block_dma = tegra_mc_block_dma_common, |
182 | .dma_idling = tegra_mc_dma_idling_common, |
183 | .unblock_dma = tegra_mc_unblock_dma_common, |
184 | .reset_status = tegra_mc_reset_status_common, |
185 | }; |
186 | |
187 | static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev) |
188 | { |
189 | return container_of(rcdev, struct tegra_mc, reset); |
190 | } |
191 | |
192 | static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc, |
193 | unsigned long id) |
194 | { |
195 | unsigned int i; |
196 | |
197 | for (i = 0; i < mc->soc->num_resets; i++) |
198 | if (mc->soc->resets[i].id == id) |
199 | return &mc->soc->resets[i]; |
200 | |
201 | return NULL; |
202 | } |
203 | |
204 | static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, |
205 | unsigned long id) |
206 | { |
207 | struct tegra_mc *mc = reset_to_mc(rcdev); |
208 | const struct tegra_mc_reset_ops *rst_ops; |
209 | const struct tegra_mc_reset *rst; |
210 | int retries = 500; |
211 | int err; |
212 | |
213 | rst = tegra_mc_reset_find(mc, id); |
214 | if (!rst) |
215 | return -ENODEV; |
216 | |
217 | rst_ops = mc->soc->reset_ops; |
218 | if (!rst_ops) |
219 | return -ENODEV; |
220 | |
221 | /* DMA flushing will fail if reset is already asserted */ |
222 | if (rst_ops->reset_status) { |
223 | /* check whether reset is asserted */ |
224 | if (rst_ops->reset_status(mc, rst)) |
225 | return 0; |
226 | } |
227 | |
228 | if (rst_ops->block_dma) { |
229 | /* block clients DMA requests */ |
230 | err = rst_ops->block_dma(mc, rst); |
231 | if (err) { |
232 | dev_err(mc->dev, "failed to block %s DMA: %d\n" , |
233 | rst->name, err); |
234 | return err; |
235 | } |
236 | } |
237 | |
238 | if (rst_ops->dma_idling) { |
239 | /* wait for completion of the outstanding DMA requests */ |
240 | while (!rst_ops->dma_idling(mc, rst)) { |
241 | if (!retries--) { |
242 | dev_err(mc->dev, "failed to flush %s DMA\n" , |
243 | rst->name); |
244 | return -EBUSY; |
245 | } |
246 | |
247 | usleep_range(min: 10, max: 100); |
248 | } |
249 | } |
250 | |
251 | if (rst_ops->hotreset_assert) { |
252 | /* clear clients DMA requests sitting before arbitration */ |
253 | err = rst_ops->hotreset_assert(mc, rst); |
254 | if (err) { |
255 | dev_err(mc->dev, "failed to hot reset %s: %d\n" , |
256 | rst->name, err); |
257 | return err; |
258 | } |
259 | } |
260 | |
261 | return 0; |
262 | } |
263 | |
264 | static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev, |
265 | unsigned long id) |
266 | { |
267 | struct tegra_mc *mc = reset_to_mc(rcdev); |
268 | const struct tegra_mc_reset_ops *rst_ops; |
269 | const struct tegra_mc_reset *rst; |
270 | int err; |
271 | |
272 | rst = tegra_mc_reset_find(mc, id); |
273 | if (!rst) |
274 | return -ENODEV; |
275 | |
276 | rst_ops = mc->soc->reset_ops; |
277 | if (!rst_ops) |
278 | return -ENODEV; |
279 | |
280 | if (rst_ops->hotreset_deassert) { |
281 | /* take out client from hot reset */ |
282 | err = rst_ops->hotreset_deassert(mc, rst); |
283 | if (err) { |
284 | dev_err(mc->dev, "failed to deassert hot reset %s: %d\n" , |
285 | rst->name, err); |
286 | return err; |
287 | } |
288 | } |
289 | |
290 | if (rst_ops->unblock_dma) { |
291 | /* allow new DMA requests to proceed to arbitration */ |
292 | err = rst_ops->unblock_dma(mc, rst); |
293 | if (err) { |
294 | dev_err(mc->dev, "failed to unblock %s DMA : %d\n" , |
295 | rst->name, err); |
296 | return err; |
297 | } |
298 | } |
299 | |
300 | return 0; |
301 | } |
302 | |
303 | static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev, |
304 | unsigned long id) |
305 | { |
306 | struct tegra_mc *mc = reset_to_mc(rcdev); |
307 | const struct tegra_mc_reset_ops *rst_ops; |
308 | const struct tegra_mc_reset *rst; |
309 | |
310 | rst = tegra_mc_reset_find(mc, id); |
311 | if (!rst) |
312 | return -ENODEV; |
313 | |
314 | rst_ops = mc->soc->reset_ops; |
315 | if (!rst_ops) |
316 | return -ENODEV; |
317 | |
318 | return rst_ops->reset_status(mc, rst); |
319 | } |
320 | |
321 | static const struct reset_control_ops tegra_mc_reset_ops = { |
322 | .assert = tegra_mc_hotreset_assert, |
323 | .deassert = tegra_mc_hotreset_deassert, |
324 | .status = tegra_mc_hotreset_status, |
325 | }; |
326 | |
327 | static int tegra_mc_reset_setup(struct tegra_mc *mc) |
328 | { |
329 | int err; |
330 | |
331 | mc->reset.ops = &tegra_mc_reset_ops; |
332 | mc->reset.owner = THIS_MODULE; |
333 | mc->reset.of_node = mc->dev->of_node; |
334 | mc->reset.of_reset_n_cells = 1; |
335 | mc->reset.nr_resets = mc->soc->num_resets; |
336 | |
337 | err = reset_controller_register(rcdev: &mc->reset); |
338 | if (err < 0) |
339 | return err; |
340 | |
341 | return 0; |
342 | } |
343 | |
344 | int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) |
345 | { |
346 | unsigned int i; |
347 | struct tegra_mc_timing *timing = NULL; |
348 | |
349 | for (i = 0; i < mc->num_timings; i++) { |
350 | if (mc->timings[i].rate == rate) { |
351 | timing = &mc->timings[i]; |
352 | break; |
353 | } |
354 | } |
355 | |
356 | if (!timing) { |
357 | dev_err(mc->dev, "no memory timing registered for rate %lu\n" , |
358 | rate); |
359 | return -EINVAL; |
360 | } |
361 | |
362 | for (i = 0; i < mc->soc->num_emem_regs; ++i) |
363 | mc_writel(mc, value: timing->emem_data[i], offset: mc->soc->emem_regs[i]); |
364 | |
365 | return 0; |
366 | } |
367 | EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration); |
368 | |
369 | unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) |
370 | { |
371 | u8 dram_count; |
372 | |
373 | dram_count = mc_readl(mc, MC_EMEM_ADR_CFG); |
374 | dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV; |
375 | dram_count++; |
376 | |
377 | return dram_count; |
378 | } |
379 | EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count); |
380 | |
381 | #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \ |
382 | defined(CONFIG_ARCH_TEGRA_114_SOC) || \ |
383 | defined(CONFIG_ARCH_TEGRA_124_SOC) || \ |
384 | defined(CONFIG_ARCH_TEGRA_132_SOC) || \ |
385 | defined(CONFIG_ARCH_TEGRA_210_SOC) |
386 | static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) |
387 | { |
388 | unsigned long long tick; |
389 | unsigned int i; |
390 | u32 value; |
391 | |
392 | /* compute the number of MC clock cycles per tick */ |
393 | tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); |
394 | do_div(tick, NSEC_PER_SEC); |
395 | |
396 | value = mc_readl(mc, MC_EMEM_ARB_CFG); |
397 | value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; |
398 | value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); |
399 | mc_writel(mc, value, MC_EMEM_ARB_CFG); |
400 | |
401 | /* write latency allowance defaults */ |
402 | for (i = 0; i < mc->soc->num_clients; i++) { |
403 | const struct tegra_mc_client *client = &mc->soc->clients[i]; |
404 | u32 value; |
405 | |
406 | value = mc_readl(mc, client->regs.la.reg); |
407 | value &= ~(client->regs.la.mask << client->regs.la.shift); |
408 | value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift; |
409 | mc_writel(mc, value, client->regs.la.reg); |
410 | } |
411 | |
412 | /* latch new values */ |
413 | mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); |
414 | |
415 | return 0; |
416 | } |
417 | |
418 | static int load_one_timing(struct tegra_mc *mc, |
419 | struct tegra_mc_timing *timing, |
420 | struct device_node *node) |
421 | { |
422 | int err; |
423 | u32 tmp; |
424 | |
425 | err = of_property_read_u32(node, "clock-frequency" , &tmp); |
426 | if (err) { |
427 | dev_err(mc->dev, |
428 | "timing %pOFn: failed to read rate\n" , node); |
429 | return err; |
430 | } |
431 | |
432 | timing->rate = tmp; |
433 | timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs, |
434 | sizeof(u32), GFP_KERNEL); |
435 | if (!timing->emem_data) |
436 | return -ENOMEM; |
437 | |
438 | err = of_property_read_u32_array(node, "nvidia,emem-configuration" , |
439 | timing->emem_data, |
440 | mc->soc->num_emem_regs); |
441 | if (err) { |
442 | dev_err(mc->dev, |
443 | "timing %pOFn: failed to read EMEM configuration\n" , |
444 | node); |
445 | return err; |
446 | } |
447 | |
448 | return 0; |
449 | } |
450 | |
451 | static int load_timings(struct tegra_mc *mc, struct device_node *node) |
452 | { |
453 | struct device_node *child; |
454 | struct tegra_mc_timing *timing; |
455 | int child_count = of_get_child_count(node); |
456 | int i = 0, err; |
457 | |
458 | mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing), |
459 | GFP_KERNEL); |
460 | if (!mc->timings) |
461 | return -ENOMEM; |
462 | |
463 | mc->num_timings = child_count; |
464 | |
465 | for_each_child_of_node(node, child) { |
466 | timing = &mc->timings[i++]; |
467 | |
468 | err = load_one_timing(mc, timing, child); |
469 | if (err) { |
470 | of_node_put(child); |
471 | return err; |
472 | } |
473 | } |
474 | |
475 | return 0; |
476 | } |
477 | |
478 | static int tegra_mc_setup_timings(struct tegra_mc *mc) |
479 | { |
480 | struct device_node *node; |
481 | u32 ram_code, node_ram_code; |
482 | int err; |
483 | |
484 | ram_code = tegra_read_ram_code(); |
485 | |
486 | mc->num_timings = 0; |
487 | |
488 | for_each_child_of_node(mc->dev->of_node, node) { |
489 | err = of_property_read_u32(node, "nvidia,ram-code" , |
490 | &node_ram_code); |
491 | if (err || (node_ram_code != ram_code)) |
492 | continue; |
493 | |
494 | err = load_timings(mc, node); |
495 | of_node_put(node); |
496 | if (err) |
497 | return err; |
498 | break; |
499 | } |
500 | |
501 | if (mc->num_timings == 0) |
502 | dev_warn(mc->dev, |
503 | "no memory timings for RAM code %u registered\n" , |
504 | ram_code); |
505 | |
506 | return 0; |
507 | } |
508 | |
509 | int tegra30_mc_probe(struct tegra_mc *mc) |
510 | { |
511 | int err; |
512 | |
513 | mc->clk = devm_clk_get_optional(mc->dev, "mc" ); |
514 | if (IS_ERR(mc->clk)) { |
515 | dev_err(mc->dev, "failed to get MC clock: %ld\n" , PTR_ERR(mc->clk)); |
516 | return PTR_ERR(mc->clk); |
517 | } |
518 | |
519 | /* ensure that debug features are disabled */ |
520 | mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); |
521 | |
522 | err = tegra_mc_setup_latency_allowance(mc); |
523 | if (err < 0) { |
524 | dev_err(mc->dev, "failed to setup latency allowance: %d\n" , err); |
525 | return err; |
526 | } |
527 | |
528 | err = tegra_mc_setup_timings(mc); |
529 | if (err < 0) { |
530 | dev_err(mc->dev, "failed to setup timings: %d\n" , err); |
531 | return err; |
532 | } |
533 | |
534 | return 0; |
535 | } |
536 | |
537 | const struct tegra_mc_ops tegra30_mc_ops = { |
538 | .probe = tegra30_mc_probe, |
539 | .handle_irq = tegra30_mc_handle_irq, |
540 | }; |
541 | #endif |
542 | |
543 | static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status, |
544 | unsigned int *mc_channel) |
545 | { |
546 | if ((status & mc->soc->ch_intmask) == 0) |
547 | return -EINVAL; |
548 | |
549 | *mc_channel = __ffs((status & mc->soc->ch_intmask) >> |
550 | mc->soc->global_intstatus_channel_shift); |
551 | |
552 | return 0; |
553 | } |
554 | |
555 | static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc, |
556 | unsigned int channel) |
557 | { |
558 | return BIT(channel) << mc->soc->global_intstatus_channel_shift; |
559 | } |
560 | |
561 | irqreturn_t tegra30_mc_handle_irq(int irq, void *data) |
562 | { |
563 | struct tegra_mc *mc = data; |
564 | unsigned int bit, channel; |
565 | unsigned long status; |
566 | |
567 | if (mc->soc->num_channels) { |
568 | u32 global_status; |
569 | int err; |
570 | |
571 | global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS); |
572 | err = mc_global_intstatus_to_channel(mc, status: global_status, mc_channel: &channel); |
573 | if (err < 0) { |
574 | dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n" , |
575 | global_status); |
576 | return IRQ_NONE; |
577 | } |
578 | |
579 | /* mask all interrupts to avoid flooding */ |
580 | status = mc_ch_readl(mc, ch: channel, MC_INTSTATUS) & mc->soc->intmask; |
581 | } else { |
582 | status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; |
583 | } |
584 | |
585 | if (!status) |
586 | return IRQ_NONE; |
587 | |
588 | for_each_set_bit(bit, &status, 32) { |
589 | const char *error = tegra_mc_status_names[bit] ?: "unknown" ; |
590 | const char *client = "unknown" , *desc; |
591 | const char *direction, *secure; |
592 | u32 status_reg, addr_reg; |
593 | u32 intmask = BIT(bit); |
594 | phys_addr_t addr = 0; |
595 | #ifdef CONFIG_PHYS_ADDR_T_64BIT |
596 | u32 addr_hi_reg = 0; |
597 | #endif |
598 | unsigned int i; |
599 | char perm[7]; |
600 | u8 id, type; |
601 | u32 value; |
602 | |
603 | switch (intmask) { |
604 | case MC_INT_DECERR_VPR: |
605 | status_reg = MC_ERR_VPR_STATUS; |
606 | addr_reg = MC_ERR_VPR_ADR; |
607 | break; |
608 | |
609 | case MC_INT_SECERR_SEC: |
610 | status_reg = MC_ERR_SEC_STATUS; |
611 | addr_reg = MC_ERR_SEC_ADR; |
612 | break; |
613 | |
614 | case MC_INT_DECERR_MTS: |
615 | status_reg = MC_ERR_MTS_STATUS; |
616 | addr_reg = MC_ERR_MTS_ADR; |
617 | break; |
618 | |
619 | case MC_INT_DECERR_GENERALIZED_CARVEOUT: |
620 | status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS; |
621 | addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR; |
622 | break; |
623 | |
624 | case MC_INT_DECERR_ROUTE_SANITY: |
625 | status_reg = MC_ERR_ROUTE_SANITY_STATUS; |
626 | addr_reg = MC_ERR_ROUTE_SANITY_ADR; |
627 | break; |
628 | |
629 | default: |
630 | status_reg = MC_ERR_STATUS; |
631 | addr_reg = MC_ERR_ADR; |
632 | |
633 | #ifdef CONFIG_PHYS_ADDR_T_64BIT |
634 | if (mc->soc->has_addr_hi_reg) |
635 | addr_hi_reg = MC_ERR_ADR_HI; |
636 | #endif |
637 | break; |
638 | } |
639 | |
640 | if (mc->soc->num_channels) |
641 | value = mc_ch_readl(mc, ch: channel, offset: status_reg); |
642 | else |
643 | value = mc_readl(mc, offset: status_reg); |
644 | |
645 | #ifdef CONFIG_PHYS_ADDR_T_64BIT |
646 | if (mc->soc->num_address_bits > 32) { |
647 | if (addr_hi_reg) { |
648 | if (mc->soc->num_channels) |
649 | addr = mc_ch_readl(mc, ch: channel, offset: addr_hi_reg); |
650 | else |
651 | addr = mc_readl(mc, offset: addr_hi_reg); |
652 | } else { |
653 | addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & |
654 | MC_ERR_STATUS_ADR_HI_MASK); |
655 | } |
656 | addr <<= 32; |
657 | } |
658 | #endif |
659 | |
660 | if (value & MC_ERR_STATUS_RW) |
661 | direction = "write" ; |
662 | else |
663 | direction = "read" ; |
664 | |
665 | if (value & MC_ERR_STATUS_SECURITY) |
666 | secure = "secure " ; |
667 | else |
668 | secure = "" ; |
669 | |
670 | id = value & mc->soc->client_id_mask; |
671 | |
672 | for (i = 0; i < mc->soc->num_clients; i++) { |
673 | if (mc->soc->clients[i].id == id) { |
674 | client = mc->soc->clients[i].name; |
675 | break; |
676 | } |
677 | } |
678 | |
679 | type = (value & MC_ERR_STATUS_TYPE_MASK) >> |
680 | MC_ERR_STATUS_TYPE_SHIFT; |
681 | desc = tegra_mc_error_names[type]; |
682 | |
683 | switch (value & MC_ERR_STATUS_TYPE_MASK) { |
684 | case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE: |
685 | perm[0] = ' '; |
686 | perm[1] = '['; |
687 | |
688 | if (value & MC_ERR_STATUS_READABLE) |
689 | perm[2] = 'R'; |
690 | else |
691 | perm[2] = '-'; |
692 | |
693 | if (value & MC_ERR_STATUS_WRITABLE) |
694 | perm[3] = 'W'; |
695 | else |
696 | perm[3] = '-'; |
697 | |
698 | if (value & MC_ERR_STATUS_NONSECURE) |
699 | perm[4] = '-'; |
700 | else |
701 | perm[4] = 'S'; |
702 | |
703 | perm[5] = ']'; |
704 | perm[6] = '\0'; |
705 | break; |
706 | |
707 | default: |
708 | perm[0] = '\0'; |
709 | break; |
710 | } |
711 | |
712 | if (mc->soc->num_channels) |
713 | value = mc_ch_readl(mc, ch: channel, offset: addr_reg); |
714 | else |
715 | value = mc_readl(mc, offset: addr_reg); |
716 | addr |= value; |
717 | |
718 | dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n" , |
719 | client, secure, direction, &addr, error, |
720 | desc, perm); |
721 | } |
722 | |
723 | /* clear interrupts */ |
724 | if (mc->soc->num_channels) { |
725 | mc_ch_writel(mc, ch: channel, value: status, MC_INTSTATUS); |
726 | mc_ch_writel(mc, MC_BROADCAST_CHANNEL, |
727 | value: mc_channel_to_global_intstatus(mc, channel), |
728 | MC_GLOBAL_INTSTATUS); |
729 | } else { |
730 | mc_writel(mc, value: status, MC_INTSTATUS); |
731 | } |
732 | |
733 | return IRQ_HANDLED; |
734 | } |
735 | |
736 | const char *const tegra_mc_status_names[32] = { |
737 | [ 1] = "External interrupt" , |
738 | [ 6] = "EMEM address decode error" , |
739 | [ 7] = "GART page fault" , |
740 | [ 8] = "Security violation" , |
741 | [ 9] = "EMEM arbitration error" , |
742 | [10] = "Page fault" , |
743 | [11] = "Invalid APB ASID update" , |
744 | [12] = "VPR violation" , |
745 | [13] = "Secure carveout violation" , |
746 | [16] = "MTS carveout violation" , |
747 | [17] = "Generalized carveout violation" , |
748 | [20] = "Route Sanity error" , |
749 | }; |
750 | |
751 | const char *const tegra_mc_error_names[8] = { |
752 | [2] = "EMEM decode error" , |
753 | [3] = "TrustZone violation" , |
754 | [4] = "Carveout violation" , |
755 | [6] = "SMMU translation error" , |
756 | }; |
757 | |
758 | struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data) |
759 | { |
760 | struct tegra_mc *mc = icc_provider_to_tegra_mc(provider: data); |
761 | struct icc_node *node; |
762 | |
763 | list_for_each_entry(node, &mc->provider.nodes, node_list) { |
764 | if (node->id == spec->args[0]) |
765 | return node; |
766 | } |
767 | |
768 | /* |
769 | * If a client driver calls devm_of_icc_get() before the MC driver |
770 | * is probed, then return EPROBE_DEFER to the client driver. |
771 | */ |
772 | return ERR_PTR(error: -EPROBE_DEFER); |
773 | } |
774 | |
775 | static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak) |
776 | { |
777 | *average = 0; |
778 | *peak = 0; |
779 | |
780 | return 0; |
781 | } |
782 | |
783 | static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst) |
784 | { |
785 | return 0; |
786 | } |
787 | |
788 | const struct tegra_mc_icc_ops tegra_mc_icc_ops = { |
789 | .xlate = tegra_mc_icc_xlate, |
790 | .aggregate = icc_std_aggregate, |
791 | .get_bw = tegra_mc_icc_get, |
792 | .set = tegra_mc_icc_set, |
793 | }; |
794 | |
795 | /* |
796 | * Memory Controller (MC) has few Memory Clients that are issuing memory |
797 | * bandwidth allocation requests to the MC interconnect provider. The MC |
798 | * provider aggregates the requests and then sends the aggregated request |
799 | * up to the External Memory Controller (EMC) interconnect provider which |
800 | * re-configures hardware interface to External Memory (EMEM) in accordance |
801 | * to the required bandwidth. Each MC interconnect node represents an |
802 | * individual Memory Client. |
803 | * |
804 | * Memory interconnect topology: |
805 | * |
806 | * +----+ |
807 | * +--------+ | | |
808 | * | TEXSRD +--->+ | |
809 | * +--------+ | | |
810 | * | | +-----+ +------+ |
811 | * ... | MC +--->+ EMC +--->+ EMEM | |
812 | * | | +-----+ +------+ |
813 | * +--------+ | | |
814 | * | DISP.. +--->+ | |
815 | * +--------+ | | |
816 | * +----+ |
817 | */ |
818 | static int tegra_mc_interconnect_setup(struct tegra_mc *mc) |
819 | { |
820 | struct icc_node *node; |
821 | unsigned int i; |
822 | int err; |
823 | |
824 | /* older device-trees don't have interconnect properties */ |
825 | if (!device_property_present(dev: mc->dev, propname: "#interconnect-cells" ) || |
826 | !mc->soc->icc_ops) |
827 | return 0; |
828 | |
829 | mc->provider.dev = mc->dev; |
830 | mc->provider.data = &mc->provider; |
831 | mc->provider.set = mc->soc->icc_ops->set; |
832 | mc->provider.aggregate = mc->soc->icc_ops->aggregate; |
833 | mc->provider.get_bw = mc->soc->icc_ops->get_bw; |
834 | mc->provider.xlate = mc->soc->icc_ops->xlate; |
835 | mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended; |
836 | |
837 | icc_provider_init(provider: &mc->provider); |
838 | |
839 | /* create Memory Controller node */ |
840 | node = icc_node_create(TEGRA_ICC_MC); |
841 | if (IS_ERR(ptr: node)) |
842 | return PTR_ERR(ptr: node); |
843 | |
844 | node->name = "Memory Controller" ; |
845 | icc_node_add(node, provider: &mc->provider); |
846 | |
847 | /* link Memory Controller to External Memory Controller */ |
848 | err = icc_link_create(node, TEGRA_ICC_EMC); |
849 | if (err) |
850 | goto remove_nodes; |
851 | |
852 | for (i = 0; i < mc->soc->num_clients; i++) { |
853 | /* create MC client node */ |
854 | node = icc_node_create(id: mc->soc->clients[i].id); |
855 | if (IS_ERR(ptr: node)) { |
856 | err = PTR_ERR(ptr: node); |
857 | goto remove_nodes; |
858 | } |
859 | |
860 | node->name = mc->soc->clients[i].name; |
861 | icc_node_add(node, provider: &mc->provider); |
862 | |
863 | /* link Memory Client to Memory Controller */ |
864 | err = icc_link_create(node, TEGRA_ICC_MC); |
865 | if (err) |
866 | goto remove_nodes; |
867 | |
868 | node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]); |
869 | } |
870 | |
871 | err = icc_provider_register(provider: &mc->provider); |
872 | if (err) |
873 | goto remove_nodes; |
874 | |
875 | return 0; |
876 | |
877 | remove_nodes: |
878 | icc_nodes_remove(provider: &mc->provider); |
879 | |
880 | return err; |
881 | } |
882 | |
883 | static void tegra_mc_num_channel_enabled(struct tegra_mc *mc) |
884 | { |
885 | unsigned int i; |
886 | u32 value; |
887 | |
888 | value = mc_ch_readl(mc, ch: 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE); |
889 | if (value <= 0) { |
890 | mc->num_channels = mc->soc->num_channels; |
891 | return; |
892 | } |
893 | |
894 | for (i = 0; i < 32; i++) { |
895 | if (value & BIT(i)) |
896 | mc->num_channels++; |
897 | } |
898 | } |
899 | |
900 | static int tegra_mc_probe(struct platform_device *pdev) |
901 | { |
902 | struct tegra_mc *mc; |
903 | u64 mask; |
904 | int err; |
905 | |
906 | mc = devm_kzalloc(dev: &pdev->dev, size: sizeof(*mc), GFP_KERNEL); |
907 | if (!mc) |
908 | return -ENOMEM; |
909 | |
910 | platform_set_drvdata(pdev, data: mc); |
911 | spin_lock_init(&mc->lock); |
912 | mc->soc = of_device_get_match_data(dev: &pdev->dev); |
913 | mc->dev = &pdev->dev; |
914 | |
915 | mask = DMA_BIT_MASK(mc->soc->num_address_bits); |
916 | |
917 | err = dma_coerce_mask_and_coherent(dev: &pdev->dev, mask); |
918 | if (err < 0) { |
919 | dev_err(&pdev->dev, "failed to set DMA mask: %d\n" , err); |
920 | return err; |
921 | } |
922 | |
923 | /* length of MC tick in nanoseconds */ |
924 | mc->tick = 30; |
925 | |
926 | mc->regs = devm_platform_ioremap_resource(pdev, index: 0); |
927 | if (IS_ERR(ptr: mc->regs)) |
928 | return PTR_ERR(ptr: mc->regs); |
929 | |
930 | mc->debugfs.root = debugfs_create_dir(name: "mc" , NULL); |
931 | |
932 | if (mc->soc->ops && mc->soc->ops->probe) { |
933 | err = mc->soc->ops->probe(mc); |
934 | if (err < 0) |
935 | return err; |
936 | } |
937 | |
938 | tegra_mc_num_channel_enabled(mc); |
939 | |
940 | if (mc->soc->ops && mc->soc->ops->handle_irq) { |
941 | mc->irq = platform_get_irq(pdev, 0); |
942 | if (mc->irq < 0) |
943 | return mc->irq; |
944 | |
945 | WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n" ); |
946 | |
947 | if (mc->soc->num_channels) |
948 | mc_ch_writel(mc, MC_BROADCAST_CHANNEL, value: mc->soc->intmask, |
949 | MC_INTMASK); |
950 | else |
951 | mc_writel(mc, value: mc->soc->intmask, MC_INTMASK); |
952 | |
953 | err = devm_request_irq(dev: &pdev->dev, irq: mc->irq, handler: mc->soc->ops->handle_irq, irqflags: 0, |
954 | devname: dev_name(dev: &pdev->dev), dev_id: mc); |
955 | if (err < 0) { |
956 | dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n" , mc->irq, |
957 | err); |
958 | return err; |
959 | } |
960 | } |
961 | |
962 | if (mc->soc->reset_ops) { |
963 | err = tegra_mc_reset_setup(mc); |
964 | if (err < 0) |
965 | dev_err(&pdev->dev, "failed to register reset controller: %d\n" , err); |
966 | } |
967 | |
968 | err = tegra_mc_interconnect_setup(mc); |
969 | if (err < 0) |
970 | dev_err(&pdev->dev, "failed to initialize interconnect: %d\n" , |
971 | err); |
972 | |
973 | if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) { |
974 | mc->smmu = tegra_smmu_probe(dev: &pdev->dev, soc: mc->soc->smmu, mc); |
975 | if (IS_ERR(ptr: mc->smmu)) { |
976 | dev_err(&pdev->dev, "failed to probe SMMU: %ld\n" , |
977 | PTR_ERR(mc->smmu)); |
978 | mc->smmu = NULL; |
979 | } |
980 | } |
981 | |
982 | return 0; |
983 | } |
984 | |
985 | static void tegra_mc_sync_state(struct device *dev) |
986 | { |
987 | struct tegra_mc *mc = dev_get_drvdata(dev); |
988 | |
989 | /* check whether ICC provider is registered */ |
990 | if (mc->provider.dev == dev) |
991 | icc_sync_state(dev); |
992 | } |
993 | |
994 | static struct platform_driver tegra_mc_driver = { |
995 | .driver = { |
996 | .name = "tegra-mc" , |
997 | .of_match_table = tegra_mc_of_match, |
998 | .suppress_bind_attrs = true, |
999 | .sync_state = tegra_mc_sync_state, |
1000 | }, |
1001 | .prevent_deferred_probe = true, |
1002 | .probe = tegra_mc_probe, |
1003 | }; |
1004 | |
1005 | static int tegra_mc_init(void) |
1006 | { |
1007 | return platform_driver_register(&tegra_mc_driver); |
1008 | } |
1009 | arch_initcall(tegra_mc_init); |
1010 | |
1011 | MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>" ); |
1012 | MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver" ); |
1013 | |