1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2015-2016 MediaTek Inc. |
4 | * Author: Yong Wu <yong.wu@mediatek.com> |
5 | */ |
6 | #include <linux/arm-smccc.h> |
7 | #include <linux/bitfield.h> |
8 | #include <linux/bug.h> |
9 | #include <linux/clk.h> |
10 | #include <linux/component.h> |
11 | #include <linux/device.h> |
12 | #include <linux/err.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/io.h> |
15 | #include <linux/iommu.h> |
16 | #include <linux/iopoll.h> |
17 | #include <linux/io-pgtable.h> |
18 | #include <linux/list.h> |
19 | #include <linux/mfd/syscon.h> |
20 | #include <linux/module.h> |
21 | #include <linux/of_address.h> |
22 | #include <linux/of_irq.h> |
23 | #include <linux/of_platform.h> |
24 | #include <linux/pci.h> |
25 | #include <linux/platform_device.h> |
26 | #include <linux/pm_runtime.h> |
27 | #include <linux/regmap.h> |
28 | #include <linux/slab.h> |
29 | #include <linux/spinlock.h> |
30 | #include <linux/soc/mediatek/infracfg.h> |
31 | #include <linux/soc/mediatek/mtk_sip_svc.h> |
32 | #include <asm/barrier.h> |
33 | #include <soc/mediatek/smi.h> |
34 | |
35 | #include <dt-bindings/memory/mtk-memory-port.h> |
36 | |
37 | #define REG_MMU_PT_BASE_ADDR 0x000 |
38 | |
39 | #define REG_MMU_INVALIDATE 0x020 |
40 | #define F_ALL_INVLD 0x2 |
41 | #define F_MMU_INV_RANGE 0x1 |
42 | |
43 | #define REG_MMU_INVLD_START_A 0x024 |
44 | #define REG_MMU_INVLD_END_A 0x028 |
45 | |
46 | #define REG_MMU_INV_SEL_GEN2 0x02c |
47 | #define REG_MMU_INV_SEL_GEN1 0x038 |
48 | #define F_INVLD_EN0 BIT(0) |
49 | #define F_INVLD_EN1 BIT(1) |
50 | |
51 | #define REG_MMU_MISC_CTRL 0x048 |
52 | #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17)) |
53 | #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) |
54 | |
55 | #define REG_MMU_DCM_DIS 0x050 |
56 | #define F_MMU_DCM BIT(8) |
57 | |
58 | #define REG_MMU_WR_LEN_CTRL 0x054 |
59 | #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) |
60 | |
61 | #define REG_MMU_CTRL_REG 0x110 |
62 | #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4) |
63 | #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) |
64 | #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5) |
65 | |
66 | #define REG_MMU_IVRP_PADDR 0x114 |
67 | |
68 | #define REG_MMU_VLD_PA_RNG 0x118 |
69 | #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) |
70 | |
71 | #define REG_MMU_INT_CONTROL0 0x120 |
72 | #define F_L2_MULIT_HIT_EN BIT(0) |
73 | #define F_TABLE_WALK_FAULT_INT_EN BIT(1) |
74 | #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) |
75 | #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) |
76 | #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) |
77 | #define F_MISS_FIFO_ERR_INT_EN BIT(6) |
78 | #define F_INT_CLR_BIT BIT(12) |
79 | |
80 | #define REG_MMU_INT_MAIN_CONTROL 0x124 |
81 | /* mmu0 | mmu1 */ |
82 | #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7)) |
83 | #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8)) |
84 | #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9)) |
85 | #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10)) |
86 | #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11)) |
87 | #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12)) |
88 | #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13)) |
89 | |
90 | #define REG_MMU_CPE_DONE 0x12C |
91 | |
92 | #define REG_MMU_FAULT_ST1 0x134 |
93 | #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0) |
94 | #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) |
95 | |
96 | #define REG_MMU0_FAULT_VA 0x13c |
97 | #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) |
98 | #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) |
99 | #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) |
100 | #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) |
101 | #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) |
102 | |
103 | #define REG_MMU0_INVLD_PA 0x140 |
104 | #define REG_MMU1_FAULT_VA 0x144 |
105 | #define REG_MMU1_INVLD_PA 0x148 |
106 | #define REG_MMU0_INT_ID 0x150 |
107 | #define REG_MMU1_INT_ID 0x154 |
108 | #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) |
109 | #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) |
110 | #define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7) |
111 | #define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7) |
112 | /* Macro for 5 bits length port ID field (default) */ |
113 | #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) |
114 | #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) |
115 | /* Macro for 6 bits length port ID field */ |
116 | #define F_MMU_INT_ID_LARB_ID_WID_6(a) (((a) >> 8) & 0x7) |
117 | #define F_MMU_INT_ID_PORT_ID_WID_6(a) (((a) >> 2) & 0x3f) |
118 | |
119 | #define MTK_PROTECT_PA_ALIGN 256 |
120 | #define MTK_IOMMU_BANK_SZ 0x1000 |
121 | |
122 | #define PERICFG_IOMMU_1 0x714 |
123 | |
124 | #define HAS_4GB_MODE BIT(0) |
125 | /* HW will use the EMI clock if there isn't the "bclk". */ |
126 | #define HAS_BCLK BIT(1) |
127 | #define HAS_VLD_PA_RNG BIT(2) |
128 | #define RESET_AXI BIT(3) |
129 | #define OUT_ORDER_WR_EN BIT(4) |
130 | #define HAS_SUB_COMM_2BITS BIT(5) |
131 | #define HAS_SUB_COMM_3BITS BIT(6) |
132 | #define WR_THROT_EN BIT(7) |
133 | #define HAS_LEGACY_IVRP_PADDR BIT(8) |
134 | #define IOVA_34_EN BIT(9) |
135 | #define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */ |
136 | #define DCM_DISABLE BIT(11) |
137 | #define STD_AXI_MODE BIT(12) /* For non MM iommu */ |
138 | /* 2 bits: iommu type */ |
139 | #define MTK_IOMMU_TYPE_MM (0x0 << 13) |
140 | #define MTK_IOMMU_TYPE_INFRA (0x1 << 13) |
141 | #define MTK_IOMMU_TYPE_MASK (0x3 << 13) |
142 | /* PM and clock always on. e.g. infra iommu */ |
143 | #define PM_CLK_AO BIT(15) |
144 | #define IFA_IOMMU_PCIE_SUPPORT BIT(16) |
145 | #define PGTABLE_PA_35_EN BIT(17) |
146 | #define TF_PORT_TO_ADDR_MT8173 BIT(18) |
147 | #define INT_ID_PORT_WIDTH_6 BIT(19) |
148 | #define CFG_IFA_MASTER_IN_ATF BIT(20) |
149 | |
150 | #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ |
151 | ((((pdata)->flags) & (mask)) == (_x)) |
152 | |
153 | #define MTK_IOMMU_HAS_FLAG(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, _x) |
154 | #define MTK_IOMMU_IS_TYPE(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x,\ |
155 | MTK_IOMMU_TYPE_MASK) |
156 | |
157 | #define MTK_INVALID_LARBID MTK_LARB_NR_MAX |
158 | |
159 | #define MTK_LARB_COM_MAX 8 |
160 | #define MTK_LARB_SUBCOM_MAX 8 |
161 | |
162 | #define MTK_IOMMU_GROUP_MAX 8 |
163 | #define MTK_IOMMU_BANK_MAX 5 |
164 | |
165 | enum mtk_iommu_plat { |
166 | M4U_MT2712, |
167 | M4U_MT6779, |
168 | M4U_MT6795, |
169 | M4U_MT8167, |
170 | M4U_MT8173, |
171 | M4U_MT8183, |
172 | M4U_MT8186, |
173 | M4U_MT8188, |
174 | M4U_MT8192, |
175 | M4U_MT8195, |
176 | M4U_MT8365, |
177 | }; |
178 | |
179 | struct mtk_iommu_iova_region { |
180 | dma_addr_t iova_base; |
181 | unsigned long long size; |
182 | }; |
183 | |
184 | struct mtk_iommu_suspend_reg { |
185 | u32 misc_ctrl; |
186 | u32 dcm_dis; |
187 | u32 ctrl_reg; |
188 | u32 vld_pa_rng; |
189 | u32 wr_len_ctrl; |
190 | |
191 | u32 int_control[MTK_IOMMU_BANK_MAX]; |
192 | u32 int_main_control[MTK_IOMMU_BANK_MAX]; |
193 | u32 ivrp_paddr[MTK_IOMMU_BANK_MAX]; |
194 | }; |
195 | |
196 | struct mtk_iommu_plat_data { |
197 | enum mtk_iommu_plat m4u_plat; |
198 | u32 flags; |
199 | u32 inv_sel_reg; |
200 | |
201 | char *pericfg_comp_str; |
202 | struct list_head *hw_list; |
203 | |
204 | /* |
205 | * The IOMMU HW may support 16GB iova. In order to balance the IOVA ranges, |
206 | * different masters will be put in different iova ranges, for example vcodec |
207 | * is in 4G-8G and cam is in 8G-12G. Meanwhile, some masters may have the |
208 | * special IOVA range requirement, like CCU can only support the address |
209 | * 0x40000000-0x44000000. |
210 | * Here list the iova ranges this SoC supports and which larbs/ports are in |
211 | * which region. |
212 | * |
213 | * 16GB iova all use one pgtable, but each a region is a iommu group. |
214 | */ |
215 | struct { |
216 | unsigned int iova_region_nr; |
217 | const struct mtk_iommu_iova_region *iova_region; |
218 | /* |
219 | * Indicate the correspondance between larbs, ports and regions. |
220 | * |
221 | * The index is the same as iova_region and larb port numbers are |
222 | * described as bit positions. |
223 | * For example, storing BIT(0) at index 2,1 means "larb 1, port0 is in region 2". |
224 | * [2] = { [1] = BIT(0) } |
225 | */ |
226 | const u32 (*iova_region_larb_msk)[MTK_LARB_NR_MAX]; |
227 | }; |
228 | |
229 | /* |
230 | * The IOMMU HW may have 5 banks. Each bank has a independent pgtable. |
231 | * Here list how many banks this SoC supports/enables and which ports are in which bank. |
232 | */ |
233 | struct { |
234 | u8 banks_num; |
235 | bool banks_enable[MTK_IOMMU_BANK_MAX]; |
236 | unsigned int banks_portmsk[MTK_IOMMU_BANK_MAX]; |
237 | }; |
238 | |
239 | unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; |
240 | }; |
241 | |
242 | struct mtk_iommu_bank_data { |
243 | void __iomem *base; |
244 | int irq; |
245 | u8 id; |
246 | struct device *parent_dev; |
247 | struct mtk_iommu_data *parent_data; |
248 | spinlock_t tlb_lock; /* lock for tlb range flush */ |
249 | struct mtk_iommu_domain *m4u_dom; /* Each bank has a domain */ |
250 | }; |
251 | |
252 | struct mtk_iommu_data { |
253 | struct device *dev; |
254 | struct clk *bclk; |
255 | phys_addr_t protect_base; /* protect memory base */ |
256 | struct mtk_iommu_suspend_reg reg; |
257 | struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX]; |
258 | bool enable_4GB; |
259 | |
260 | struct iommu_device iommu; |
261 | const struct mtk_iommu_plat_data *plat_data; |
262 | struct device *smicomm_dev; |
263 | |
264 | struct mtk_iommu_bank_data *bank; |
265 | struct mtk_iommu_domain *share_dom; |
266 | |
267 | struct regmap *pericfg; |
268 | struct mutex mutex; /* Protect m4u_group/m4u_dom above */ |
269 | |
270 | /* |
271 | * In the sharing pgtable case, list data->list to the global list like m4ulist. |
272 | * In the non-sharing pgtable case, list data->list to the itself hw_list_head. |
273 | */ |
274 | struct list_head *hw_list; |
275 | struct list_head hw_list_head; |
276 | struct list_head list; |
277 | struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; |
278 | }; |
279 | |
280 | struct mtk_iommu_domain { |
281 | struct io_pgtable_cfg cfg; |
282 | struct io_pgtable_ops *iop; |
283 | |
284 | struct mtk_iommu_bank_data *bank; |
285 | struct iommu_domain domain; |
286 | |
287 | struct mutex mutex; /* Protect "data" in this structure */ |
288 | }; |
289 | |
290 | static int mtk_iommu_bind(struct device *dev) |
291 | { |
292 | struct mtk_iommu_data *data = dev_get_drvdata(dev); |
293 | |
294 | return component_bind_all(parent: dev, data: &data->larb_imu); |
295 | } |
296 | |
297 | static void mtk_iommu_unbind(struct device *dev) |
298 | { |
299 | struct mtk_iommu_data *data = dev_get_drvdata(dev); |
300 | |
301 | component_unbind_all(parent: dev, data: &data->larb_imu); |
302 | } |
303 | |
304 | static const struct iommu_ops mtk_iommu_ops; |
305 | |
306 | static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid); |
307 | |
308 | #define MTK_IOMMU_TLB_ADDR(iova) ({ \ |
309 | dma_addr_t _addr = iova; \ |
310 | ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ |
311 | }) |
312 | |
313 | /* |
314 | * In M4U 4GB mode, the physical address is remapped as below: |
315 | * |
316 | * CPU Physical address: |
317 | * ==================== |
318 | * |
319 | * 0 1G 2G 3G 4G 5G |
320 | * |---A---|---B---|---C---|---D---|---E---| |
321 | * +--I/O--+------------Memory-------------+ |
322 | * |
323 | * IOMMU output physical address: |
324 | * ============================= |
325 | * |
326 | * 4G 5G 6G 7G 8G |
327 | * |---E---|---B---|---C---|---D---| |
328 | * +------------Memory-------------+ |
329 | * |
330 | * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the |
331 | * bit32 of the CPU physical address always is needed to set, and for Region |
332 | * 'E', the CPU physical address keep as is. |
333 | * Additionally, The iommu consumers always use the CPU phyiscal address. |
334 | */ |
335 | #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL |
336 | |
337 | static LIST_HEAD(m4ulist); /* List all the M4U HWs */ |
338 | |
339 | #define for_each_m4u(data, head) list_for_each_entry(data, head, list) |
340 | |
341 | #define MTK_IOMMU_IOVA_SZ_4G (SZ_4G - SZ_8M) /* 8M as gap */ |
342 | |
343 | static const struct mtk_iommu_iova_region single_domain[] = { |
344 | {.iova_base = 0, .size = MTK_IOMMU_IOVA_SZ_4G}, |
345 | }; |
346 | |
347 | #define MT8192_MULTI_REGION_NR_MAX 6 |
348 | |
349 | #define MT8192_MULTI_REGION_NR (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) ? \ |
350 | MT8192_MULTI_REGION_NR_MAX : 1) |
351 | |
352 | static const struct mtk_iommu_iova_region mt8192_multi_dom[MT8192_MULTI_REGION_NR] = { |
353 | { .iova_base = 0x0, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 0 ~ 4G, */ |
354 | #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) |
355 | { .iova_base = SZ_4G, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 4G ~ 8G */ |
356 | { .iova_base = SZ_4G * 2, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 8G ~ 12G */ |
357 | { .iova_base = SZ_4G * 3, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 12G ~ 16G */ |
358 | |
359 | { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ |
360 | { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ |
361 | #endif |
362 | }; |
363 | |
364 | /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ |
365 | static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist) |
366 | { |
367 | return list_first_entry(hwlist, struct mtk_iommu_data, list); |
368 | } |
369 | |
370 | static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) |
371 | { |
372 | return container_of(dom, struct mtk_iommu_domain, domain); |
373 | } |
374 | |
375 | static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) |
376 | { |
377 | /* Tlb flush all always is in bank0. */ |
378 | struct mtk_iommu_bank_data *bank = &data->bank[0]; |
379 | void __iomem *base = bank->base; |
380 | unsigned long flags; |
381 | |
382 | spin_lock_irqsave(&bank->tlb_lock, flags); |
383 | writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, base + data->plat_data->inv_sel_reg); |
384 | writel_relaxed(F_ALL_INVLD, base + REG_MMU_INVALIDATE); |
385 | wmb(); /* Make sure the tlb flush all done */ |
386 | spin_unlock_irqrestore(lock: &bank->tlb_lock, flags); |
387 | } |
388 | |
389 | static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, |
390 | struct mtk_iommu_bank_data *bank) |
391 | { |
392 | struct list_head *head = bank->parent_data->hw_list; |
393 | struct mtk_iommu_bank_data *curbank; |
394 | struct mtk_iommu_data *data; |
395 | bool check_pm_status; |
396 | unsigned long flags; |
397 | void __iomem *base; |
398 | int ret; |
399 | u32 tmp; |
400 | |
401 | for_each_m4u(data, head) { |
402 | /* |
403 | * To avoid resume the iommu device frequently when the iommu device |
404 | * is not active, it doesn't always call pm_runtime_get here, then tlb |
405 | * flush depends on the tlb flush all in the runtime resume. |
406 | * |
407 | * There are 2 special cases: |
408 | * |
409 | * Case1: The iommu dev doesn't have power domain but has bclk. This case |
410 | * should also avoid the tlb flush while the dev is not active to mute |
411 | * the tlb timeout log. like mt8173. |
412 | * |
413 | * Case2: The power/clock of infra iommu is always on, and it doesn't |
414 | * have the device link with the master devices. This case should avoid |
415 | * the PM status check. |
416 | */ |
417 | check_pm_status = !MTK_IOMMU_HAS_FLAG(data->plat_data, PM_CLK_AO); |
418 | |
419 | if (check_pm_status) { |
420 | if (pm_runtime_get_if_in_use(dev: data->dev) <= 0) |
421 | continue; |
422 | } |
423 | |
424 | curbank = &data->bank[bank->id]; |
425 | base = curbank->base; |
426 | |
427 | spin_lock_irqsave(&curbank->tlb_lock, flags); |
428 | writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, |
429 | base + data->plat_data->inv_sel_reg); |
430 | |
431 | writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), base + REG_MMU_INVLD_START_A); |
432 | writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), |
433 | base + REG_MMU_INVLD_END_A); |
434 | writel_relaxed(F_MMU_INV_RANGE, base + REG_MMU_INVALIDATE); |
435 | |
436 | /* tlb sync */ |
437 | ret = readl_poll_timeout_atomic(base + REG_MMU_CPE_DONE, |
438 | tmp, tmp != 0, 10, 1000); |
439 | |
440 | /* Clear the CPE status */ |
441 | writel_relaxed(0, base + REG_MMU_CPE_DONE); |
442 | spin_unlock_irqrestore(lock: &curbank->tlb_lock, flags); |
443 | |
444 | if (ret) { |
445 | dev_warn(data->dev, |
446 | "Partial TLB flush timed out, falling back to full flush\n" ); |
447 | mtk_iommu_tlb_flush_all(data); |
448 | } |
449 | |
450 | if (check_pm_status) |
451 | pm_runtime_put(dev: data->dev); |
452 | } |
453 | } |
454 | |
455 | static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) |
456 | { |
457 | struct mtk_iommu_bank_data *bank = dev_id; |
458 | struct mtk_iommu_data *data = bank->parent_data; |
459 | struct mtk_iommu_domain *dom = bank->m4u_dom; |
460 | unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0; |
461 | u32 int_state, regval, va34_32, pa34_32; |
462 | const struct mtk_iommu_plat_data *plat_data = data->plat_data; |
463 | void __iomem *base = bank->base; |
464 | u64 fault_iova, fault_pa; |
465 | bool layer, write; |
466 | |
467 | /* Read error info from registers */ |
468 | int_state = readl_relaxed(base + REG_MMU_FAULT_ST1); |
469 | if (int_state & F_REG_MMU0_FAULT_MASK) { |
470 | regval = readl_relaxed(base + REG_MMU0_INT_ID); |
471 | fault_iova = readl_relaxed(base + REG_MMU0_FAULT_VA); |
472 | fault_pa = readl_relaxed(base + REG_MMU0_INVLD_PA); |
473 | } else { |
474 | regval = readl_relaxed(base + REG_MMU1_INT_ID); |
475 | fault_iova = readl_relaxed(base + REG_MMU1_FAULT_VA); |
476 | fault_pa = readl_relaxed(base + REG_MMU1_INVLD_PA); |
477 | } |
478 | layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; |
479 | write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; |
480 | if (MTK_IOMMU_HAS_FLAG(plat_data, IOVA_34_EN)) { |
481 | va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); |
482 | fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; |
483 | fault_iova |= (u64)va34_32 << 32; |
484 | } |
485 | pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); |
486 | fault_pa |= (u64)pa34_32 << 32; |
487 | |
488 | if (MTK_IOMMU_IS_TYPE(plat_data, MTK_IOMMU_TYPE_MM)) { |
489 | if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_2BITS)) { |
490 | fault_larb = F_MMU_INT_ID_COMM_ID(regval); |
491 | sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); |
492 | fault_port = F_MMU_INT_ID_PORT_ID(regval); |
493 | } else if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_3BITS)) { |
494 | fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); |
495 | sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); |
496 | fault_port = F_MMU_INT_ID_PORT_ID(regval); |
497 | } else if (MTK_IOMMU_HAS_FLAG(plat_data, INT_ID_PORT_WIDTH_6)) { |
498 | fault_port = F_MMU_INT_ID_PORT_ID_WID_6(regval); |
499 | fault_larb = F_MMU_INT_ID_LARB_ID_WID_6(regval); |
500 | } else { |
501 | fault_port = F_MMU_INT_ID_PORT_ID(regval); |
502 | fault_larb = F_MMU_INT_ID_LARB_ID(regval); |
503 | } |
504 | fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; |
505 | } |
506 | |
507 | if (!dom || report_iommu_fault(domain: &dom->domain, dev: bank->parent_dev, iova: fault_iova, |
508 | flags: write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { |
509 | dev_err_ratelimited( |
510 | bank->parent_dev, |
511 | "fault type=0x%x iova=0x%llx pa=0x%llx master=0x%x(larb=%d port=%d) layer=%d %s\n" , |
512 | int_state, fault_iova, fault_pa, regval, fault_larb, fault_port, |
513 | layer, write ? "write" : "read" ); |
514 | } |
515 | |
516 | /* Interrupt clear */ |
517 | regval = readl_relaxed(base + REG_MMU_INT_CONTROL0); |
518 | regval |= F_INT_CLR_BIT; |
519 | writel_relaxed(regval, base + REG_MMU_INT_CONTROL0); |
520 | |
521 | mtk_iommu_tlb_flush_all(data); |
522 | |
523 | return IRQ_HANDLED; |
524 | } |
525 | |
526 | static unsigned int mtk_iommu_get_bank_id(struct device *dev, |
527 | const struct mtk_iommu_plat_data *plat_data) |
528 | { |
529 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
530 | unsigned int i, portmsk = 0, bankid = 0; |
531 | |
532 | if (plat_data->banks_num == 1) |
533 | return bankid; |
534 | |
535 | for (i = 0; i < fwspec->num_ids; i++) |
536 | portmsk |= BIT(MTK_M4U_TO_PORT(fwspec->ids[i])); |
537 | |
538 | for (i = 0; i < plat_data->banks_num && i < MTK_IOMMU_BANK_MAX; i++) { |
539 | if (!plat_data->banks_enable[i]) |
540 | continue; |
541 | |
542 | if (portmsk & plat_data->banks_portmsk[i]) { |
543 | bankid = i; |
544 | break; |
545 | } |
546 | } |
547 | return bankid; /* default is 0 */ |
548 | } |
549 | |
550 | static int mtk_iommu_get_iova_region_id(struct device *dev, |
551 | const struct mtk_iommu_plat_data *plat_data) |
552 | { |
553 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
554 | unsigned int portidmsk = 0, larbid; |
555 | const u32 *rgn_larb_msk; |
556 | int i; |
557 | |
558 | if (plat_data->iova_region_nr == 1) |
559 | return 0; |
560 | |
561 | larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); |
562 | for (i = 0; i < fwspec->num_ids; i++) |
563 | portidmsk |= BIT(MTK_M4U_TO_PORT(fwspec->ids[i])); |
564 | |
565 | for (i = 0; i < plat_data->iova_region_nr; i++) { |
566 | rgn_larb_msk = plat_data->iova_region_larb_msk[i]; |
567 | if (!rgn_larb_msk) |
568 | continue; |
569 | |
570 | if ((rgn_larb_msk[larbid] & portidmsk) == portidmsk) |
571 | return i; |
572 | } |
573 | |
574 | dev_err(dev, "Can NOT find the region for larb(%d-%x).\n" , |
575 | larbid, portidmsk); |
576 | return -EINVAL; |
577 | } |
578 | |
579 | static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, |
580 | bool enable, unsigned int regionid) |
581 | { |
582 | struct mtk_smi_larb_iommu *larb_mmu; |
583 | unsigned int larbid, portid; |
584 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
585 | const struct mtk_iommu_iova_region *region; |
586 | unsigned long portid_msk = 0; |
587 | struct arm_smccc_res res; |
588 | int i, ret = 0; |
589 | |
590 | for (i = 0; i < fwspec->num_ids; ++i) { |
591 | portid = MTK_M4U_TO_PORT(fwspec->ids[i]); |
592 | portid_msk |= BIT(portid); |
593 | } |
594 | |
595 | if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { |
596 | /* All ports should be in the same larb. just use 0 here */ |
597 | larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); |
598 | larb_mmu = &data->larb_imu[larbid]; |
599 | region = data->plat_data->iova_region + regionid; |
600 | |
601 | for_each_set_bit(portid, &portid_msk, 32) |
602 | larb_mmu->bank[portid] = upper_32_bits(region->iova_base); |
603 | |
604 | dev_dbg(dev, "%s iommu for larb(%s) port 0x%lx region %d rgn-bank %d.\n" , |
605 | enable ? "enable" : "disable" , dev_name(larb_mmu->dev), |
606 | portid_msk, regionid, upper_32_bits(region->iova_base)); |
607 | |
608 | if (enable) |
609 | larb_mmu->mmu |= portid_msk; |
610 | else |
611 | larb_mmu->mmu &= ~portid_msk; |
612 | } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { |
613 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, CFG_IFA_MASTER_IN_ATF)) { |
614 | arm_smccc_smc(MTK_SIP_KERNEL_IOMMU_CONTROL, |
615 | IOMMU_ATF_CMD_CONFIG_INFRA_IOMMU, |
616 | portid_msk, enable, 0, 0, 0, 0, &res); |
617 | ret = res.a0; |
618 | } else { |
619 | /* PCI dev has only one output id, enable the next writing bit for PCIe */ |
620 | if (dev_is_pci(dev)) { |
621 | if (fwspec->num_ids != 1) { |
622 | dev_err(dev, "PCI dev can only have one port.\n" ); |
623 | return -ENODEV; |
624 | } |
625 | portid_msk |= BIT(portid + 1); |
626 | } |
627 | |
628 | ret = regmap_update_bits(map: data->pericfg, PERICFG_IOMMU_1, |
629 | mask: (u32)portid_msk, val: enable ? (u32)portid_msk : 0); |
630 | } |
631 | if (ret) |
632 | dev_err(dev, "%s iommu(%s) inframaster 0x%lx fail(%d).\n" , |
633 | enable ? "enable" : "disable" , |
634 | dev_name(data->dev), portid_msk, ret); |
635 | } |
636 | return ret; |
637 | } |
638 | |
639 | static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, |
640 | struct mtk_iommu_data *data, |
641 | unsigned int region_id) |
642 | { |
643 | struct mtk_iommu_domain *share_dom = data->share_dom; |
644 | const struct mtk_iommu_iova_region *region; |
645 | |
646 | /* Share pgtable when 2 MM IOMMU share the pgtable or one IOMMU use multiple iova ranges */ |
647 | if (share_dom) { |
648 | dom->iop = share_dom->iop; |
649 | dom->cfg = share_dom->cfg; |
650 | dom->domain.pgsize_bitmap = share_dom->cfg.pgsize_bitmap; |
651 | goto update_iova_region; |
652 | } |
653 | |
654 | dom->cfg = (struct io_pgtable_cfg) { |
655 | .quirks = IO_PGTABLE_QUIRK_ARM_NS | |
656 | IO_PGTABLE_QUIRK_NO_PERMS | |
657 | IO_PGTABLE_QUIRK_ARM_MTK_EXT, |
658 | .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, |
659 | .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, |
660 | .iommu_dev = data->dev, |
661 | }; |
662 | |
663 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN)) |
664 | dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT; |
665 | |
666 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) |
667 | dom->cfg.oas = data->enable_4GB ? 33 : 32; |
668 | else |
669 | dom->cfg.oas = 35; |
670 | |
671 | dom->iop = alloc_io_pgtable_ops(fmt: ARM_V7S, cfg: &dom->cfg, cookie: data); |
672 | if (!dom->iop) { |
673 | dev_err(data->dev, "Failed to alloc io pgtable\n" ); |
674 | return -ENOMEM; |
675 | } |
676 | |
677 | /* Update our support page sizes bitmap */ |
678 | dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; |
679 | |
680 | data->share_dom = dom; |
681 | |
682 | update_iova_region: |
683 | /* Update the iova region for this domain */ |
684 | region = data->plat_data->iova_region + region_id; |
685 | dom->domain.geometry.aperture_start = region->iova_base; |
686 | dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; |
687 | dom->domain.geometry.force_aperture = true; |
688 | return 0; |
689 | } |
690 | |
691 | static struct iommu_domain *mtk_iommu_domain_alloc_paging(struct device *dev) |
692 | { |
693 | struct mtk_iommu_domain *dom; |
694 | |
695 | dom = kzalloc(size: sizeof(*dom), GFP_KERNEL); |
696 | if (!dom) |
697 | return NULL; |
698 | mutex_init(&dom->mutex); |
699 | |
700 | return &dom->domain; |
701 | } |
702 | |
703 | static void mtk_iommu_domain_free(struct iommu_domain *domain) |
704 | { |
705 | kfree(objp: to_mtk_domain(dom: domain)); |
706 | } |
707 | |
708 | static int mtk_iommu_attach_device(struct iommu_domain *domain, |
709 | struct device *dev) |
710 | { |
711 | struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; |
712 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
713 | struct list_head *hw_list = data->hw_list; |
714 | struct device *m4udev = data->dev; |
715 | struct mtk_iommu_bank_data *bank; |
716 | unsigned int bankid; |
717 | int ret, region_id; |
718 | |
719 | region_id = mtk_iommu_get_iova_region_id(dev, plat_data: data->plat_data); |
720 | if (region_id < 0) |
721 | return region_id; |
722 | |
723 | bankid = mtk_iommu_get_bank_id(dev, plat_data: data->plat_data); |
724 | mutex_lock(&dom->mutex); |
725 | if (!dom->bank) { |
726 | /* Data is in the frstdata in sharing pgtable case. */ |
727 | frstdata = mtk_iommu_get_frst_data(hwlist: hw_list); |
728 | |
729 | mutex_lock(&frstdata->mutex); |
730 | ret = mtk_iommu_domain_finalise(dom, data: frstdata, region_id); |
731 | mutex_unlock(lock: &frstdata->mutex); |
732 | if (ret) { |
733 | mutex_unlock(lock: &dom->mutex); |
734 | return ret; |
735 | } |
736 | dom->bank = &data->bank[bankid]; |
737 | } |
738 | mutex_unlock(lock: &dom->mutex); |
739 | |
740 | mutex_lock(&data->mutex); |
741 | bank = &data->bank[bankid]; |
742 | if (!bank->m4u_dom) { /* Initialize the M4U HW for each a BANK */ |
743 | ret = pm_runtime_resume_and_get(dev: m4udev); |
744 | if (ret < 0) { |
745 | dev_err(m4udev, "pm get fail(%d) in attach.\n" , ret); |
746 | goto err_unlock; |
747 | } |
748 | |
749 | ret = mtk_iommu_hw_init(data, bankid); |
750 | if (ret) { |
751 | pm_runtime_put(dev: m4udev); |
752 | goto err_unlock; |
753 | } |
754 | bank->m4u_dom = dom; |
755 | writel(val: dom->cfg.arm_v7s_cfg.ttbr, addr: bank->base + REG_MMU_PT_BASE_ADDR); |
756 | |
757 | pm_runtime_put(dev: m4udev); |
758 | } |
759 | mutex_unlock(lock: &data->mutex); |
760 | |
761 | if (region_id > 0) { |
762 | ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(34)); |
763 | if (ret) { |
764 | dev_err(m4udev, "Failed to set dma_mask for %s(%d).\n" , dev_name(dev), ret); |
765 | return ret; |
766 | } |
767 | } |
768 | |
769 | return mtk_iommu_config(data, dev, enable: true, regionid: region_id); |
770 | |
771 | err_unlock: |
772 | mutex_unlock(lock: &data->mutex); |
773 | return ret; |
774 | } |
775 | |
776 | static int mtk_iommu_identity_attach(struct iommu_domain *identity_domain, |
777 | struct device *dev) |
778 | { |
779 | struct iommu_domain *domain = iommu_get_domain_for_dev(dev); |
780 | struct mtk_iommu_data *data = dev_iommu_priv_get(dev); |
781 | |
782 | if (domain == identity_domain || !domain) |
783 | return 0; |
784 | |
785 | mtk_iommu_config(data, dev, enable: false, regionid: 0); |
786 | return 0; |
787 | } |
788 | |
789 | static struct iommu_domain_ops mtk_iommu_identity_ops = { |
790 | .attach_dev = mtk_iommu_identity_attach, |
791 | }; |
792 | |
793 | static struct iommu_domain mtk_iommu_identity_domain = { |
794 | .type = IOMMU_DOMAIN_IDENTITY, |
795 | .ops = &mtk_iommu_identity_ops, |
796 | }; |
797 | |
798 | static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, |
799 | phys_addr_t paddr, size_t pgsize, size_t pgcount, |
800 | int prot, gfp_t gfp, size_t *mapped) |
801 | { |
802 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
803 | |
804 | /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ |
805 | if (dom->bank->parent_data->enable_4GB) |
806 | paddr |= BIT_ULL(32); |
807 | |
808 | /* Synchronize with the tlb_lock */ |
809 | return dom->iop->map_pages(dom->iop, iova, paddr, pgsize, pgcount, prot, gfp, mapped); |
810 | } |
811 | |
812 | static size_t mtk_iommu_unmap(struct iommu_domain *domain, |
813 | unsigned long iova, size_t pgsize, size_t pgcount, |
814 | struct iommu_iotlb_gather *gather) |
815 | { |
816 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
817 | |
818 | iommu_iotlb_gather_add_range(gather, iova, size: pgsize * pgcount); |
819 | return dom->iop->unmap_pages(dom->iop, iova, pgsize, pgcount, gather); |
820 | } |
821 | |
822 | static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) |
823 | { |
824 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
825 | |
826 | if (dom->bank) |
827 | mtk_iommu_tlb_flush_all(data: dom->bank->parent_data); |
828 | } |
829 | |
830 | static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, |
831 | struct iommu_iotlb_gather *gather) |
832 | { |
833 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
834 | size_t length = gather->end - gather->start + 1; |
835 | |
836 | mtk_iommu_tlb_flush_range_sync(iova: gather->start, size: length, bank: dom->bank); |
837 | } |
838 | |
839 | static int mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, |
840 | size_t size) |
841 | { |
842 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
843 | |
844 | mtk_iommu_tlb_flush_range_sync(iova, size, bank: dom->bank); |
845 | return 0; |
846 | } |
847 | |
848 | static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, |
849 | dma_addr_t iova) |
850 | { |
851 | struct mtk_iommu_domain *dom = to_mtk_domain(dom: domain); |
852 | phys_addr_t pa; |
853 | |
854 | pa = dom->iop->iova_to_phys(dom->iop, iova); |
855 | if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && |
856 | dom->bank->parent_data->enable_4GB && |
857 | pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) |
858 | pa &= ~BIT_ULL(32); |
859 | |
860 | return pa; |
861 | } |
862 | |
863 | static struct iommu_device *mtk_iommu_probe_device(struct device *dev) |
864 | { |
865 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
866 | struct mtk_iommu_data *data = dev_iommu_priv_get(dev); |
867 | struct device_link *link; |
868 | struct device *larbdev; |
869 | unsigned int larbid, larbidx, i; |
870 | |
871 | if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) |
872 | return &data->iommu; |
873 | |
874 | /* |
875 | * Link the consumer device with the smi-larb device(supplier). |
876 | * The device that connects with each a larb is a independent HW. |
877 | * All the ports in each a device should be in the same larbs. |
878 | */ |
879 | larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); |
880 | if (larbid >= MTK_LARB_NR_MAX) |
881 | return ERR_PTR(error: -EINVAL); |
882 | |
883 | for (i = 1; i < fwspec->num_ids; i++) { |
884 | larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); |
885 | if (larbid != larbidx) { |
886 | dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n" , |
887 | larbid, larbidx); |
888 | return ERR_PTR(error: -EINVAL); |
889 | } |
890 | } |
891 | larbdev = data->larb_imu[larbid].dev; |
892 | if (!larbdev) |
893 | return ERR_PTR(error: -EINVAL); |
894 | |
895 | link = device_link_add(consumer: dev, supplier: larbdev, |
896 | DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); |
897 | if (!link) |
898 | dev_err(dev, "Unable to link %s\n" , dev_name(larbdev)); |
899 | return &data->iommu; |
900 | } |
901 | |
902 | static void mtk_iommu_release_device(struct device *dev) |
903 | { |
904 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
905 | struct mtk_iommu_data *data; |
906 | struct device *larbdev; |
907 | unsigned int larbid; |
908 | |
909 | data = dev_iommu_priv_get(dev); |
910 | if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { |
911 | larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); |
912 | larbdev = data->larb_imu[larbid].dev; |
913 | device_link_remove(consumer: dev, supplier: larbdev); |
914 | } |
915 | } |
916 | |
917 | static int mtk_iommu_get_group_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) |
918 | { |
919 | unsigned int bankid; |
920 | |
921 | /* |
922 | * If the bank function is enabled, each bank is a iommu group/domain. |
923 | * Otherwise, each iova region is a iommu group/domain. |
924 | */ |
925 | bankid = mtk_iommu_get_bank_id(dev, plat_data); |
926 | if (bankid) |
927 | return bankid; |
928 | |
929 | return mtk_iommu_get_iova_region_id(dev, plat_data); |
930 | } |
931 | |
932 | static struct iommu_group *mtk_iommu_device_group(struct device *dev) |
933 | { |
934 | struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; |
935 | struct list_head *hw_list = c_data->hw_list; |
936 | struct iommu_group *group; |
937 | int groupid; |
938 | |
939 | data = mtk_iommu_get_frst_data(hwlist: hw_list); |
940 | if (!data) |
941 | return ERR_PTR(error: -ENODEV); |
942 | |
943 | groupid = mtk_iommu_get_group_id(dev, plat_data: data->plat_data); |
944 | if (groupid < 0) |
945 | return ERR_PTR(error: groupid); |
946 | |
947 | mutex_lock(&data->mutex); |
948 | group = data->m4u_group[groupid]; |
949 | if (!group) { |
950 | group = iommu_group_alloc(); |
951 | if (!IS_ERR(ptr: group)) |
952 | data->m4u_group[groupid] = group; |
953 | } else { |
954 | iommu_group_ref_get(group); |
955 | } |
956 | mutex_unlock(lock: &data->mutex); |
957 | return group; |
958 | } |
959 | |
960 | static int mtk_iommu_of_xlate(struct device *dev, |
961 | const struct of_phandle_args *args) |
962 | { |
963 | struct platform_device *m4updev; |
964 | |
965 | if (args->args_count != 1) { |
966 | dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n" , |
967 | args->args_count); |
968 | return -EINVAL; |
969 | } |
970 | |
971 | if (!dev_iommu_priv_get(dev)) { |
972 | /* Get the m4u device */ |
973 | m4updev = of_find_device_by_node(np: args->np); |
974 | if (WARN_ON(!m4updev)) |
975 | return -EINVAL; |
976 | |
977 | dev_iommu_priv_set(dev, priv: platform_get_drvdata(pdev: m4updev)); |
978 | } |
979 | |
980 | return iommu_fwspec_add_ids(dev, ids: args->args, num_ids: 1); |
981 | } |
982 | |
983 | static void mtk_iommu_get_resv_regions(struct device *dev, |
984 | struct list_head *head) |
985 | { |
986 | struct mtk_iommu_data *data = dev_iommu_priv_get(dev); |
987 | unsigned int regionid = mtk_iommu_get_iova_region_id(dev, plat_data: data->plat_data), i; |
988 | const struct mtk_iommu_iova_region *resv, *curdom; |
989 | struct iommu_resv_region *region; |
990 | int prot = IOMMU_WRITE | IOMMU_READ; |
991 | |
992 | if ((int)regionid < 0) |
993 | return; |
994 | curdom = data->plat_data->iova_region + regionid; |
995 | for (i = 0; i < data->plat_data->iova_region_nr; i++) { |
996 | resv = data->plat_data->iova_region + i; |
997 | |
998 | /* Only reserve when the region is inside the current domain */ |
999 | if (resv->iova_base <= curdom->iova_base || |
1000 | resv->iova_base + resv->size >= curdom->iova_base + curdom->size) |
1001 | continue; |
1002 | |
1003 | region = iommu_alloc_resv_region(start: resv->iova_base, length: resv->size, |
1004 | prot, type: IOMMU_RESV_RESERVED, |
1005 | GFP_KERNEL); |
1006 | if (!region) |
1007 | return; |
1008 | |
1009 | list_add_tail(new: ®ion->list, head); |
1010 | } |
1011 | } |
1012 | |
1013 | static const struct iommu_ops mtk_iommu_ops = { |
1014 | .identity_domain = &mtk_iommu_identity_domain, |
1015 | .domain_alloc_paging = mtk_iommu_domain_alloc_paging, |
1016 | .probe_device = mtk_iommu_probe_device, |
1017 | .release_device = mtk_iommu_release_device, |
1018 | .device_group = mtk_iommu_device_group, |
1019 | .of_xlate = mtk_iommu_of_xlate, |
1020 | .get_resv_regions = mtk_iommu_get_resv_regions, |
1021 | .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, |
1022 | .owner = THIS_MODULE, |
1023 | .default_domain_ops = &(const struct iommu_domain_ops) { |
1024 | .attach_dev = mtk_iommu_attach_device, |
1025 | .map_pages = mtk_iommu_map, |
1026 | .unmap_pages = mtk_iommu_unmap, |
1027 | .flush_iotlb_all = mtk_iommu_flush_iotlb_all, |
1028 | .iotlb_sync = mtk_iommu_iotlb_sync, |
1029 | .iotlb_sync_map = mtk_iommu_sync_map, |
1030 | .iova_to_phys = mtk_iommu_iova_to_phys, |
1031 | .free = mtk_iommu_domain_free, |
1032 | } |
1033 | }; |
1034 | |
1035 | static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid) |
1036 | { |
1037 | const struct mtk_iommu_bank_data *bankx = &data->bank[bankid]; |
1038 | const struct mtk_iommu_bank_data *bank0 = &data->bank[0]; |
1039 | u32 regval; |
1040 | |
1041 | /* |
1042 | * Global control settings are in bank0. May re-init these global registers |
1043 | * since no sure if there is bank0 consumers. |
1044 | */ |
1045 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, TF_PORT_TO_ADDR_MT8173)) { |
1046 | regval = F_MMU_PREFETCH_RT_REPLACE_MOD | |
1047 | F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; |
1048 | } else { |
1049 | regval = readl_relaxed(bank0->base + REG_MMU_CTRL_REG); |
1050 | regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; |
1051 | } |
1052 | writel_relaxed(regval, bank0->base + REG_MMU_CTRL_REG); |
1053 | |
1054 | if (data->enable_4GB && |
1055 | MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { |
1056 | /* |
1057 | * If 4GB mode is enabled, the validate PA range is from |
1058 | * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. |
1059 | */ |
1060 | regval = F_MMU_VLD_PA_RNG(7, 4); |
1061 | writel_relaxed(regval, bank0->base + REG_MMU_VLD_PA_RNG); |
1062 | } |
1063 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE)) |
1064 | writel_relaxed(F_MMU_DCM, bank0->base + REG_MMU_DCM_DIS); |
1065 | else |
1066 | writel_relaxed(0, bank0->base + REG_MMU_DCM_DIS); |
1067 | |
1068 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { |
1069 | /* write command throttling mode */ |
1070 | regval = readl_relaxed(bank0->base + REG_MMU_WR_LEN_CTRL); |
1071 | regval &= ~F_MMU_WR_THROT_DIS_MASK; |
1072 | writel_relaxed(regval, bank0->base + REG_MMU_WR_LEN_CTRL); |
1073 | } |
1074 | |
1075 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { |
1076 | /* The register is called STANDARD_AXI_MODE in this case */ |
1077 | regval = 0; |
1078 | } else { |
1079 | regval = readl_relaxed(bank0->base + REG_MMU_MISC_CTRL); |
1080 | if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE)) |
1081 | regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; |
1082 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) |
1083 | regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; |
1084 | } |
1085 | writel_relaxed(regval, bank0->base + REG_MMU_MISC_CTRL); |
1086 | |
1087 | /* Independent settings for each bank */ |
1088 | regval = F_L2_MULIT_HIT_EN | |
1089 | F_TABLE_WALK_FAULT_INT_EN | |
1090 | F_PREETCH_FIFO_OVERFLOW_INT_EN | |
1091 | F_MISS_FIFO_OVERFLOW_INT_EN | |
1092 | F_PREFETCH_FIFO_ERR_INT_EN | |
1093 | F_MISS_FIFO_ERR_INT_EN; |
1094 | writel_relaxed(regval, bankx->base + REG_MMU_INT_CONTROL0); |
1095 | |
1096 | regval = F_INT_TRANSLATION_FAULT | |
1097 | F_INT_MAIN_MULTI_HIT_FAULT | |
1098 | F_INT_INVALID_PA_FAULT | |
1099 | F_INT_ENTRY_REPLACEMENT_FAULT | |
1100 | F_INT_TLB_MISS_FAULT | |
1101 | F_INT_MISS_TRANSACTION_FIFO_FAULT | |
1102 | F_INT_PRETETCH_TRANSATION_FIFO_FAULT; |
1103 | writel_relaxed(regval, bankx->base + REG_MMU_INT_MAIN_CONTROL); |
1104 | |
1105 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) |
1106 | regval = (data->protect_base >> 1) | (data->enable_4GB << 31); |
1107 | else |
1108 | regval = lower_32_bits(data->protect_base) | |
1109 | upper_32_bits(data->protect_base); |
1110 | writel_relaxed(regval, bankx->base + REG_MMU_IVRP_PADDR); |
1111 | |
1112 | if (devm_request_irq(dev: bankx->parent_dev, irq: bankx->irq, handler: mtk_iommu_isr, irqflags: 0, |
1113 | devname: dev_name(dev: bankx->parent_dev), dev_id: (void *)bankx)) { |
1114 | writel_relaxed(0, bankx->base + REG_MMU_PT_BASE_ADDR); |
1115 | dev_err(bankx->parent_dev, "Failed @ IRQ-%d Request\n" , bankx->irq); |
1116 | return -ENODEV; |
1117 | } |
1118 | |
1119 | return 0; |
1120 | } |
1121 | |
1122 | static const struct component_master_ops mtk_iommu_com_ops = { |
1123 | .bind = mtk_iommu_bind, |
1124 | .unbind = mtk_iommu_unbind, |
1125 | }; |
1126 | |
1127 | static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match, |
1128 | struct mtk_iommu_data *data) |
1129 | { |
1130 | struct device_node *larbnode, *frst_avail_smicomm_node = NULL; |
1131 | struct platform_device *plarbdev, *pcommdev; |
1132 | struct device_link *link; |
1133 | int i, larb_nr, ret; |
1134 | |
1135 | larb_nr = of_count_phandle_with_args(np: dev->of_node, list_name: "mediatek,larbs" , NULL); |
1136 | if (larb_nr < 0) |
1137 | return larb_nr; |
1138 | if (larb_nr == 0 || larb_nr > MTK_LARB_NR_MAX) |
1139 | return -EINVAL; |
1140 | |
1141 | for (i = 0; i < larb_nr; i++) { |
1142 | struct device_node *smicomm_node, *smi_subcomm_node; |
1143 | u32 id; |
1144 | |
1145 | larbnode = of_parse_phandle(np: dev->of_node, phandle_name: "mediatek,larbs" , index: i); |
1146 | if (!larbnode) { |
1147 | ret = -EINVAL; |
1148 | goto err_larbdev_put; |
1149 | } |
1150 | |
1151 | if (!of_device_is_available(device: larbnode)) { |
1152 | of_node_put(node: larbnode); |
1153 | continue; |
1154 | } |
1155 | |
1156 | ret = of_property_read_u32(np: larbnode, propname: "mediatek,larb-id" , out_value: &id); |
1157 | if (ret)/* The id is consecutive if there is no this property */ |
1158 | id = i; |
1159 | if (id >= MTK_LARB_NR_MAX) { |
1160 | of_node_put(node: larbnode); |
1161 | ret = -EINVAL; |
1162 | goto err_larbdev_put; |
1163 | } |
1164 | |
1165 | plarbdev = of_find_device_by_node(np: larbnode); |
1166 | of_node_put(node: larbnode); |
1167 | if (!plarbdev) { |
1168 | ret = -ENODEV; |
1169 | goto err_larbdev_put; |
1170 | } |
1171 | if (data->larb_imu[id].dev) { |
1172 | platform_device_put(pdev: plarbdev); |
1173 | ret = -EEXIST; |
1174 | goto err_larbdev_put; |
1175 | } |
1176 | data->larb_imu[id].dev = &plarbdev->dev; |
1177 | |
1178 | if (!plarbdev->dev.driver) { |
1179 | ret = -EPROBE_DEFER; |
1180 | goto err_larbdev_put; |
1181 | } |
1182 | |
1183 | /* Get smi-(sub)-common dev from the last larb. */ |
1184 | smi_subcomm_node = of_parse_phandle(np: larbnode, phandle_name: "mediatek,smi" , index: 0); |
1185 | if (!smi_subcomm_node) { |
1186 | ret = -EINVAL; |
1187 | goto err_larbdev_put; |
1188 | } |
1189 | |
1190 | /* |
1191 | * It may have two level smi-common. the node is smi-sub-common if it |
1192 | * has a new mediatek,smi property. otherwise it is smi-commmon. |
1193 | */ |
1194 | smicomm_node = of_parse_phandle(np: smi_subcomm_node, phandle_name: "mediatek,smi" , index: 0); |
1195 | if (smicomm_node) |
1196 | of_node_put(node: smi_subcomm_node); |
1197 | else |
1198 | smicomm_node = smi_subcomm_node; |
1199 | |
1200 | /* |
1201 | * All the larbs that connect to one IOMMU must connect with the same |
1202 | * smi-common. |
1203 | */ |
1204 | if (!frst_avail_smicomm_node) { |
1205 | frst_avail_smicomm_node = smicomm_node; |
1206 | } else if (frst_avail_smicomm_node != smicomm_node) { |
1207 | dev_err(dev, "mediatek,smi property is not right @larb%d." , id); |
1208 | of_node_put(node: smicomm_node); |
1209 | ret = -EINVAL; |
1210 | goto err_larbdev_put; |
1211 | } else { |
1212 | of_node_put(node: smicomm_node); |
1213 | } |
1214 | |
1215 | component_match_add(parent: dev, matchptr: match, compare: component_compare_dev, compare_data: &plarbdev->dev); |
1216 | platform_device_put(pdev: plarbdev); |
1217 | } |
1218 | |
1219 | if (!frst_avail_smicomm_node) |
1220 | return -EINVAL; |
1221 | |
1222 | pcommdev = of_find_device_by_node(np: frst_avail_smicomm_node); |
1223 | of_node_put(node: frst_avail_smicomm_node); |
1224 | if (!pcommdev) |
1225 | return -ENODEV; |
1226 | data->smicomm_dev = &pcommdev->dev; |
1227 | |
1228 | link = device_link_add(consumer: data->smicomm_dev, supplier: dev, |
1229 | DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); |
1230 | platform_device_put(pdev: pcommdev); |
1231 | if (!link) { |
1232 | dev_err(dev, "Unable to link %s.\n" , dev_name(data->smicomm_dev)); |
1233 | return -EINVAL; |
1234 | } |
1235 | return 0; |
1236 | |
1237 | err_larbdev_put: |
1238 | for (i = MTK_LARB_NR_MAX - 1; i >= 0; i--) { |
1239 | if (!data->larb_imu[i].dev) |
1240 | continue; |
1241 | put_device(dev: data->larb_imu[i].dev); |
1242 | } |
1243 | return ret; |
1244 | } |
1245 | |
1246 | static int mtk_iommu_probe(struct platform_device *pdev) |
1247 | { |
1248 | struct mtk_iommu_data *data; |
1249 | struct device *dev = &pdev->dev; |
1250 | struct resource *res; |
1251 | resource_size_t ioaddr; |
1252 | struct component_match *match = NULL; |
1253 | struct regmap *infracfg; |
1254 | void *protect; |
1255 | int ret, banks_num, i = 0; |
1256 | u32 val; |
1257 | char *p; |
1258 | struct mtk_iommu_bank_data *bank; |
1259 | void __iomem *base; |
1260 | |
1261 | data = devm_kzalloc(dev, size: sizeof(*data), GFP_KERNEL); |
1262 | if (!data) |
1263 | return -ENOMEM; |
1264 | data->dev = dev; |
1265 | data->plat_data = of_device_get_match_data(dev); |
1266 | |
1267 | /* Protect memory. HW will access here while translation fault.*/ |
1268 | protect = devm_kcalloc(dev, n: 2, MTK_PROTECT_PA_ALIGN, GFP_KERNEL); |
1269 | if (!protect) |
1270 | return -ENOMEM; |
1271 | data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); |
1272 | |
1273 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) { |
1274 | infracfg = syscon_regmap_lookup_by_phandle(np: dev->of_node, property: "mediatek,infracfg" ); |
1275 | if (IS_ERR(ptr: infracfg)) { |
1276 | /* |
1277 | * Legacy devicetrees will not specify a phandle to |
1278 | * mediatek,infracfg: in that case, we use the older |
1279 | * way to retrieve a syscon to infra. |
1280 | * |
1281 | * This is for retrocompatibility purposes only, hence |
1282 | * no more compatibles shall be added to this. |
1283 | */ |
1284 | switch (data->plat_data->m4u_plat) { |
1285 | case M4U_MT2712: |
1286 | p = "mediatek,mt2712-infracfg" ; |
1287 | break; |
1288 | case M4U_MT8173: |
1289 | p = "mediatek,mt8173-infracfg" ; |
1290 | break; |
1291 | default: |
1292 | p = NULL; |
1293 | } |
1294 | |
1295 | infracfg = syscon_regmap_lookup_by_compatible(s: p); |
1296 | if (IS_ERR(ptr: infracfg)) |
1297 | return PTR_ERR(ptr: infracfg); |
1298 | } |
1299 | |
1300 | ret = regmap_read(map: infracfg, REG_INFRA_MISC, val: &val); |
1301 | if (ret) |
1302 | return ret; |
1303 | data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); |
1304 | } |
1305 | |
1306 | banks_num = data->plat_data->banks_num; |
1307 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
1308 | if (!res) |
1309 | return -EINVAL; |
1310 | if (resource_size(res) < banks_num * MTK_IOMMU_BANK_SZ) { |
1311 | dev_err(dev, "banknr %d. res %pR is not enough.\n" , banks_num, res); |
1312 | return -EINVAL; |
1313 | } |
1314 | base = devm_ioremap_resource(dev, res); |
1315 | if (IS_ERR(ptr: base)) |
1316 | return PTR_ERR(ptr: base); |
1317 | ioaddr = res->start; |
1318 | |
1319 | data->bank = devm_kmalloc(dev, size: banks_num * sizeof(*data->bank), GFP_KERNEL); |
1320 | if (!data->bank) |
1321 | return -ENOMEM; |
1322 | |
1323 | do { |
1324 | if (!data->plat_data->banks_enable[i]) |
1325 | continue; |
1326 | bank = &data->bank[i]; |
1327 | bank->id = i; |
1328 | bank->base = base + i * MTK_IOMMU_BANK_SZ; |
1329 | bank->m4u_dom = NULL; |
1330 | |
1331 | bank->irq = platform_get_irq(pdev, i); |
1332 | if (bank->irq < 0) |
1333 | return bank->irq; |
1334 | bank->parent_dev = dev; |
1335 | bank->parent_data = data; |
1336 | spin_lock_init(&bank->tlb_lock); |
1337 | } while (++i < banks_num); |
1338 | |
1339 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { |
1340 | data->bclk = devm_clk_get(dev, id: "bclk" ); |
1341 | if (IS_ERR(ptr: data->bclk)) |
1342 | return PTR_ERR(ptr: data->bclk); |
1343 | } |
1344 | |
1345 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN)) { |
1346 | ret = dma_set_mask(dev, DMA_BIT_MASK(35)); |
1347 | if (ret) { |
1348 | dev_err(dev, "Failed to set dma_mask 35.\n" ); |
1349 | return ret; |
1350 | } |
1351 | } |
1352 | |
1353 | pm_runtime_enable(dev); |
1354 | |
1355 | if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { |
1356 | ret = mtk_iommu_mm_dts_parse(dev, match: &match, data); |
1357 | if (ret) { |
1358 | dev_err_probe(dev, err: ret, fmt: "mm dts parse fail\n" ); |
1359 | goto out_runtime_disable; |
1360 | } |
1361 | } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && |
1362 | !MTK_IOMMU_HAS_FLAG(data->plat_data, CFG_IFA_MASTER_IN_ATF)) { |
1363 | p = data->plat_data->pericfg_comp_str; |
1364 | data->pericfg = syscon_regmap_lookup_by_compatible(s: p); |
1365 | if (IS_ERR(ptr: data->pericfg)) { |
1366 | ret = PTR_ERR(ptr: data->pericfg); |
1367 | goto out_runtime_disable; |
1368 | } |
1369 | } |
1370 | |
1371 | platform_set_drvdata(pdev, data); |
1372 | mutex_init(&data->mutex); |
1373 | |
1374 | ret = iommu_device_sysfs_add(iommu: &data->iommu, parent: dev, NULL, |
1375 | fmt: "mtk-iommu.%pa" , &ioaddr); |
1376 | if (ret) |
1377 | goto out_link_remove; |
1378 | |
1379 | ret = iommu_device_register(iommu: &data->iommu, ops: &mtk_iommu_ops, hwdev: dev); |
1380 | if (ret) |
1381 | goto out_sysfs_remove; |
1382 | |
1383 | if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { |
1384 | list_add_tail(new: &data->list, head: data->plat_data->hw_list); |
1385 | data->hw_list = data->plat_data->hw_list; |
1386 | } else { |
1387 | INIT_LIST_HEAD(list: &data->hw_list_head); |
1388 | list_add_tail(new: &data->list, head: &data->hw_list_head); |
1389 | data->hw_list = &data->hw_list_head; |
1390 | } |
1391 | |
1392 | if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { |
1393 | ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); |
1394 | if (ret) |
1395 | goto out_list_del; |
1396 | } |
1397 | return ret; |
1398 | |
1399 | out_list_del: |
1400 | list_del(entry: &data->list); |
1401 | iommu_device_unregister(iommu: &data->iommu); |
1402 | out_sysfs_remove: |
1403 | iommu_device_sysfs_remove(iommu: &data->iommu); |
1404 | out_link_remove: |
1405 | if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) |
1406 | device_link_remove(consumer: data->smicomm_dev, supplier: dev); |
1407 | out_runtime_disable: |
1408 | pm_runtime_disable(dev); |
1409 | return ret; |
1410 | } |
1411 | |
1412 | static void mtk_iommu_remove(struct platform_device *pdev) |
1413 | { |
1414 | struct mtk_iommu_data *data = platform_get_drvdata(pdev); |
1415 | struct mtk_iommu_bank_data *bank; |
1416 | int i; |
1417 | |
1418 | iommu_device_sysfs_remove(iommu: &data->iommu); |
1419 | iommu_device_unregister(iommu: &data->iommu); |
1420 | |
1421 | list_del(entry: &data->list); |
1422 | |
1423 | if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { |
1424 | device_link_remove(consumer: data->smicomm_dev, supplier: &pdev->dev); |
1425 | component_master_del(&pdev->dev, &mtk_iommu_com_ops); |
1426 | } |
1427 | pm_runtime_disable(dev: &pdev->dev); |
1428 | for (i = 0; i < data->plat_data->banks_num; i++) { |
1429 | bank = &data->bank[i]; |
1430 | if (!bank->m4u_dom) |
1431 | continue; |
1432 | devm_free_irq(dev: &pdev->dev, irq: bank->irq, dev_id: bank); |
1433 | } |
1434 | } |
1435 | |
1436 | static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) |
1437 | { |
1438 | struct mtk_iommu_data *data = dev_get_drvdata(dev); |
1439 | struct mtk_iommu_suspend_reg *reg = &data->reg; |
1440 | void __iomem *base; |
1441 | int i = 0; |
1442 | |
1443 | base = data->bank[i].base; |
1444 | reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); |
1445 | reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); |
1446 | reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); |
1447 | reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); |
1448 | reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); |
1449 | do { |
1450 | if (!data->plat_data->banks_enable[i]) |
1451 | continue; |
1452 | base = data->bank[i].base; |
1453 | reg->int_control[i] = readl_relaxed(base + REG_MMU_INT_CONTROL0); |
1454 | reg->int_main_control[i] = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); |
1455 | reg->ivrp_paddr[i] = readl_relaxed(base + REG_MMU_IVRP_PADDR); |
1456 | } while (++i < data->plat_data->banks_num); |
1457 | clk_disable_unprepare(clk: data->bclk); |
1458 | return 0; |
1459 | } |
1460 | |
1461 | static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) |
1462 | { |
1463 | struct mtk_iommu_data *data = dev_get_drvdata(dev); |
1464 | struct mtk_iommu_suspend_reg *reg = &data->reg; |
1465 | struct mtk_iommu_domain *m4u_dom; |
1466 | void __iomem *base; |
1467 | int ret, i = 0; |
1468 | |
1469 | ret = clk_prepare_enable(clk: data->bclk); |
1470 | if (ret) { |
1471 | dev_err(data->dev, "Failed to enable clk(%d) in resume\n" , ret); |
1472 | return ret; |
1473 | } |
1474 | |
1475 | /* |
1476 | * Uppon first resume, only enable the clk and return, since the values of the |
1477 | * registers are not yet set. |
1478 | */ |
1479 | if (!reg->wr_len_ctrl) |
1480 | return 0; |
1481 | |
1482 | base = data->bank[i].base; |
1483 | writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); |
1484 | writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); |
1485 | writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); |
1486 | writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); |
1487 | writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); |
1488 | do { |
1489 | m4u_dom = data->bank[i].m4u_dom; |
1490 | if (!data->plat_data->banks_enable[i] || !m4u_dom) |
1491 | continue; |
1492 | base = data->bank[i].base; |
1493 | writel_relaxed(reg->int_control[i], base + REG_MMU_INT_CONTROL0); |
1494 | writel_relaxed(reg->int_main_control[i], base + REG_MMU_INT_MAIN_CONTROL); |
1495 | writel_relaxed(reg->ivrp_paddr[i], base + REG_MMU_IVRP_PADDR); |
1496 | writel(val: m4u_dom->cfg.arm_v7s_cfg.ttbr, addr: base + REG_MMU_PT_BASE_ADDR); |
1497 | } while (++i < data->plat_data->banks_num); |
1498 | |
1499 | /* |
1500 | * Users may allocate dma buffer before they call pm_runtime_get, |
1501 | * in which case it will lack the necessary tlb flush. |
1502 | * Thus, make sure to update the tlb after each PM resume. |
1503 | */ |
1504 | mtk_iommu_tlb_flush_all(data); |
1505 | return 0; |
1506 | } |
1507 | |
1508 | static const struct dev_pm_ops mtk_iommu_pm_ops = { |
1509 | SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) |
1510 | SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, |
1511 | pm_runtime_force_resume) |
1512 | }; |
1513 | |
1514 | static const struct mtk_iommu_plat_data mt2712_data = { |
1515 | .m4u_plat = M4U_MT2712, |
1516 | .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE | |
1517 | MTK_IOMMU_TYPE_MM, |
1518 | .hw_list = &m4ulist, |
1519 | .inv_sel_reg = REG_MMU_INV_SEL_GEN1, |
1520 | .iova_region = single_domain, |
1521 | .banks_num = 1, |
1522 | .banks_enable = {true}, |
1523 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1524 | .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, |
1525 | }; |
1526 | |
1527 | static const struct mtk_iommu_plat_data mt6779_data = { |
1528 | .m4u_plat = M4U_MT6779, |
1529 | .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | |
1530 | MTK_IOMMU_TYPE_MM | PGTABLE_PA_35_EN, |
1531 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1532 | .banks_num = 1, |
1533 | .banks_enable = {true}, |
1534 | .iova_region = single_domain, |
1535 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1536 | .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, |
1537 | }; |
1538 | |
1539 | static const struct mtk_iommu_plat_data mt6795_data = { |
1540 | .m4u_plat = M4U_MT6795, |
1541 | .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | |
1542 | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM | |
1543 | TF_PORT_TO_ADDR_MT8173, |
1544 | .inv_sel_reg = REG_MMU_INV_SEL_GEN1, |
1545 | .banks_num = 1, |
1546 | .banks_enable = {true}, |
1547 | .iova_region = single_domain, |
1548 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1549 | .larbid_remap = {{0}, {1}, {2}, {3}, {4}}, /* Linear mapping. */ |
1550 | }; |
1551 | |
1552 | static const struct mtk_iommu_plat_data mt8167_data = { |
1553 | .m4u_plat = M4U_MT8167, |
1554 | .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, |
1555 | .inv_sel_reg = REG_MMU_INV_SEL_GEN1, |
1556 | .banks_num = 1, |
1557 | .banks_enable = {true}, |
1558 | .iova_region = single_domain, |
1559 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1560 | .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ |
1561 | }; |
1562 | |
1563 | static const struct mtk_iommu_plat_data mt8173_data = { |
1564 | .m4u_plat = M4U_MT8173, |
1565 | .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | |
1566 | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM | |
1567 | TF_PORT_TO_ADDR_MT8173, |
1568 | .inv_sel_reg = REG_MMU_INV_SEL_GEN1, |
1569 | .banks_num = 1, |
1570 | .banks_enable = {true}, |
1571 | .iova_region = single_domain, |
1572 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1573 | .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ |
1574 | }; |
1575 | |
1576 | static const struct mtk_iommu_plat_data mt8183_data = { |
1577 | .m4u_plat = M4U_MT8183, |
1578 | .flags = RESET_AXI | MTK_IOMMU_TYPE_MM, |
1579 | .inv_sel_reg = REG_MMU_INV_SEL_GEN1, |
1580 | .banks_num = 1, |
1581 | .banks_enable = {true}, |
1582 | .iova_region = single_domain, |
1583 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1584 | .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, |
1585 | }; |
1586 | |
1587 | static const unsigned int mt8186_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { |
1588 | [0] = {~0, ~0, ~0}, /* Region0: all ports for larb0/1/2 */ |
1589 | [1] = {0, 0, 0, 0, ~0, 0, 0, ~0}, /* Region1: larb4/7 */ |
1590 | [2] = {0, 0, 0, 0, 0, 0, 0, 0, /* Region2: larb8/9/11/13/16/17/19/20 */ |
1591 | ~0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), 0, 0, |
1592 | /* larb13: the other ports except port9/10 */ |
1593 | ~0, ~0, 0, ~0, ~0}, |
1594 | [3] = {0}, |
1595 | [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ |
1596 | [5] = {[14] = ~0}, /* larb14 */ |
1597 | }; |
1598 | |
1599 | static const struct mtk_iommu_plat_data mt8186_data_mm = { |
1600 | .m4u_plat = M4U_MT8186, |
1601 | .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | |
1602 | WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, |
1603 | .larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20}, |
1604 | {MTK_INVALID_LARBID, 14, 16}, |
1605 | {MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}}, |
1606 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1607 | .banks_num = 1, |
1608 | .banks_enable = {true}, |
1609 | .iova_region = mt8192_multi_dom, |
1610 | .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), |
1611 | .iova_region_larb_msk = mt8186_larb_region_msk, |
1612 | }; |
1613 | |
1614 | static const struct mtk_iommu_plat_data mt8188_data_infra = { |
1615 | .m4u_plat = M4U_MT8188, |
1616 | .flags = WR_THROT_EN | DCM_DISABLE | STD_AXI_MODE | PM_CLK_AO | |
1617 | MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT | |
1618 | PGTABLE_PA_35_EN | CFG_IFA_MASTER_IN_ATF, |
1619 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1620 | .banks_num = 1, |
1621 | .banks_enable = {true}, |
1622 | .iova_region = single_domain, |
1623 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1624 | }; |
1625 | |
1626 | static const u32 mt8188_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { |
1627 | [0] = {~0, ~0, ~0, ~0}, /* Region0: all ports for larb0/1/2/3 */ |
1628 | [1] = {0, 0, 0, 0, 0, 0, 0, 0, |
1629 | 0, 0, 0, 0, 0, 0, 0, 0, |
1630 | 0, 0, 0, 0, 0, ~0, ~0, ~0}, /* Region1: larb19(21)/21(22)/23 */ |
1631 | [2] = {0, 0, 0, 0, ~0, ~0, ~0, ~0, /* Region2: the other larbs. */ |
1632 | ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, |
1633 | ~0, ~0, ~0, ~0, ~0, 0, 0, 0, |
1634 | 0, ~0}, |
1635 | [3] = {0}, |
1636 | [4] = {[24] = BIT(0) | BIT(1)}, /* Only larb27(24) port0/1 */ |
1637 | [5] = {[24] = BIT(2) | BIT(3)}, /* Only larb27(24) port2/3 */ |
1638 | }; |
1639 | |
1640 | static const struct mtk_iommu_plat_data mt8188_data_vdo = { |
1641 | .m4u_plat = M4U_MT8188, |
1642 | .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | |
1643 | WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | |
1644 | PGTABLE_PA_35_EN | MTK_IOMMU_TYPE_MM, |
1645 | .hw_list = &m4ulist, |
1646 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1647 | .banks_num = 1, |
1648 | .banks_enable = {true}, |
1649 | .iova_region = mt8192_multi_dom, |
1650 | .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), |
1651 | .iova_region_larb_msk = mt8188_larb_region_msk, |
1652 | .larbid_remap = {{2}, {0}, {21}, {0}, {19}, {9, 10, |
1653 | 11 /* 11a */, 25 /* 11c */}, |
1654 | {13, 0, 29 /* 16b */, 30 /* 17b */, 0}, {5}}, |
1655 | }; |
1656 | |
1657 | static const struct mtk_iommu_plat_data mt8188_data_vpp = { |
1658 | .m4u_plat = M4U_MT8188, |
1659 | .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | |
1660 | WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | |
1661 | PGTABLE_PA_35_EN | MTK_IOMMU_TYPE_MM, |
1662 | .hw_list = &m4ulist, |
1663 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1664 | .banks_num = 1, |
1665 | .banks_enable = {true}, |
1666 | .iova_region = mt8192_multi_dom, |
1667 | .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), |
1668 | .iova_region_larb_msk = mt8188_larb_region_msk, |
1669 | .larbid_remap = {{1}, {3}, {23}, {7}, {MTK_INVALID_LARBID}, |
1670 | {12, 15, 24 /* 11b */}, {14, MTK_INVALID_LARBID, |
1671 | 16 /* 16a */, 17 /* 17a */, MTK_INVALID_LARBID, |
1672 | 27, 28 /* ccu0 */, MTK_INVALID_LARBID}, {4, 6}}, |
1673 | }; |
1674 | |
1675 | static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { |
1676 | [0] = {~0, ~0}, /* Region0: larb0/1 */ |
1677 | [1] = {0, 0, 0, 0, ~0, ~0, 0, ~0}, /* Region1: larb4/5/7 */ |
1678 | [2] = {0, 0, ~0, 0, 0, 0, 0, 0, /* Region2: larb2/9/11/13/14/16/17/18/19/20 */ |
1679 | 0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), ~(u32)(BIT(4) | BIT(5)), 0, |
1680 | ~0, ~0, ~0, ~0, ~0}, |
1681 | [3] = {0}, |
1682 | [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ |
1683 | [5] = {[14] = BIT(4) | BIT(5)}, /* larb14 port4/5 */ |
1684 | }; |
1685 | |
1686 | static const struct mtk_iommu_plat_data mt8192_data = { |
1687 | .m4u_plat = M4U_MT8192, |
1688 | .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | |
1689 | WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, |
1690 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1691 | .banks_num = 1, |
1692 | .banks_enable = {true}, |
1693 | .iova_region = mt8192_multi_dom, |
1694 | .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), |
1695 | .iova_region_larb_msk = mt8192_larb_region_msk, |
1696 | .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, |
1697 | {0, 14, 16}, {0, 13, 18, 17}}, |
1698 | }; |
1699 | |
1700 | static const struct mtk_iommu_plat_data mt8195_data_infra = { |
1701 | .m4u_plat = M4U_MT8195, |
1702 | .flags = WR_THROT_EN | DCM_DISABLE | STD_AXI_MODE | PM_CLK_AO | |
1703 | MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT, |
1704 | .pericfg_comp_str = "mediatek,mt8195-pericfg_ao" , |
1705 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1706 | .banks_num = 5, |
1707 | .banks_enable = {true, false, false, false, true}, |
1708 | .banks_portmsk = {[0] = GENMASK(19, 16), /* PCIe */ |
1709 | [4] = GENMASK(31, 20), /* USB */ |
1710 | }, |
1711 | .iova_region = single_domain, |
1712 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1713 | }; |
1714 | |
1715 | static const unsigned int mt8195_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { |
1716 | [0] = {~0, ~0, ~0, ~0}, /* Region0: all ports for larb0/1/2/3 */ |
1717 | [1] = {0, 0, 0, 0, 0, 0, 0, 0, |
1718 | 0, 0, 0, 0, 0, 0, 0, 0, |
1719 | 0, 0, 0, ~0, ~0, ~0, ~0, ~0, /* Region1: larb19/20/21/22/23/24 */ |
1720 | ~0}, |
1721 | [2] = {0, 0, 0, 0, ~0, ~0, ~0, ~0, /* Region2: the other larbs. */ |
1722 | ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, |
1723 | ~0, ~0, 0, 0, 0, 0, 0, 0, |
1724 | 0, ~0, ~0, ~0, ~0}, |
1725 | [3] = {0}, |
1726 | [4] = {[18] = BIT(0) | BIT(1)}, /* Only larb18 port0/1 */ |
1727 | [5] = {[18] = BIT(2) | BIT(3)}, /* Only larb18 port2/3 */ |
1728 | }; |
1729 | |
1730 | static const struct mtk_iommu_plat_data mt8195_data_vdo = { |
1731 | .m4u_plat = M4U_MT8195, |
1732 | .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | |
1733 | WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, |
1734 | .hw_list = &m4ulist, |
1735 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1736 | .banks_num = 1, |
1737 | .banks_enable = {true}, |
1738 | .iova_region = mt8192_multi_dom, |
1739 | .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), |
1740 | .iova_region_larb_msk = mt8195_larb_region_msk, |
1741 | .larbid_remap = {{2, 0}, {21}, {24}, {7}, {19}, {9, 10, 11}, |
1742 | {13, 17, 15/* 17b */, 25}, {5}}, |
1743 | }; |
1744 | |
1745 | static const struct mtk_iommu_plat_data mt8195_data_vpp = { |
1746 | .m4u_plat = M4U_MT8195, |
1747 | .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | |
1748 | WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, |
1749 | .hw_list = &m4ulist, |
1750 | .inv_sel_reg = REG_MMU_INV_SEL_GEN2, |
1751 | .banks_num = 1, |
1752 | .banks_enable = {true}, |
1753 | .iova_region = mt8192_multi_dom, |
1754 | .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), |
1755 | .iova_region_larb_msk = mt8195_larb_region_msk, |
1756 | .larbid_remap = {{1}, {3}, |
1757 | {22, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 23}, |
1758 | {8}, {20}, {12}, |
1759 | /* 16: 16a; 29: 16b; 30: CCUtop0; 31: CCUtop1 */ |
1760 | {14, 16, 29, 26, 30, 31, 18}, |
1761 | {4, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 6}}, |
1762 | }; |
1763 | |
1764 | static const struct mtk_iommu_plat_data mt8365_data = { |
1765 | .m4u_plat = M4U_MT8365, |
1766 | .flags = RESET_AXI | INT_ID_PORT_WIDTH_6, |
1767 | .inv_sel_reg = REG_MMU_INV_SEL_GEN1, |
1768 | .banks_num = 1, |
1769 | .banks_enable = {true}, |
1770 | .iova_region = single_domain, |
1771 | .iova_region_nr = ARRAY_SIZE(single_domain), |
1772 | .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ |
1773 | }; |
1774 | |
1775 | static const struct of_device_id mtk_iommu_of_ids[] = { |
1776 | { .compatible = "mediatek,mt2712-m4u" , .data = &mt2712_data}, |
1777 | { .compatible = "mediatek,mt6779-m4u" , .data = &mt6779_data}, |
1778 | { .compatible = "mediatek,mt6795-m4u" , .data = &mt6795_data}, |
1779 | { .compatible = "mediatek,mt8167-m4u" , .data = &mt8167_data}, |
1780 | { .compatible = "mediatek,mt8173-m4u" , .data = &mt8173_data}, |
1781 | { .compatible = "mediatek,mt8183-m4u" , .data = &mt8183_data}, |
1782 | { .compatible = "mediatek,mt8186-iommu-mm" , .data = &mt8186_data_mm}, /* mm: m4u */ |
1783 | { .compatible = "mediatek,mt8188-iommu-infra" , .data = &mt8188_data_infra}, |
1784 | { .compatible = "mediatek,mt8188-iommu-vdo" , .data = &mt8188_data_vdo}, |
1785 | { .compatible = "mediatek,mt8188-iommu-vpp" , .data = &mt8188_data_vpp}, |
1786 | { .compatible = "mediatek,mt8192-m4u" , .data = &mt8192_data}, |
1787 | { .compatible = "mediatek,mt8195-iommu-infra" , .data = &mt8195_data_infra}, |
1788 | { .compatible = "mediatek,mt8195-iommu-vdo" , .data = &mt8195_data_vdo}, |
1789 | { .compatible = "mediatek,mt8195-iommu-vpp" , .data = &mt8195_data_vpp}, |
1790 | { .compatible = "mediatek,mt8365-m4u" , .data = &mt8365_data}, |
1791 | {} |
1792 | }; |
1793 | MODULE_DEVICE_TABLE(of, mtk_iommu_of_ids); |
1794 | |
1795 | static struct platform_driver mtk_iommu_driver = { |
1796 | .probe = mtk_iommu_probe, |
1797 | .remove_new = mtk_iommu_remove, |
1798 | .driver = { |
1799 | .name = "mtk-iommu" , |
1800 | .of_match_table = mtk_iommu_of_ids, |
1801 | .pm = &mtk_iommu_pm_ops, |
1802 | } |
1803 | }; |
1804 | module_platform_driver(mtk_iommu_driver); |
1805 | |
1806 | MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations" ); |
1807 | MODULE_LICENSE("GPL v2" ); |
1808 | |