1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2020-2023 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/circ_buf.h> |
7 | #include <linux/highmem.h> |
8 | |
9 | #include "ivpu_drv.h" |
10 | #include "ivpu_hw.h" |
11 | #include "ivpu_hw_reg_io.h" |
12 | #include "ivpu_mmu.h" |
13 | #include "ivpu_mmu_context.h" |
14 | #include "ivpu_pm.h" |
15 | |
16 | #define IVPU_MMU_REG_IDR0 0x00200000u |
17 | #define IVPU_MMU_REG_IDR1 0x00200004u |
18 | #define IVPU_MMU_REG_IDR3 0x0020000cu |
19 | #define IVPU_MMU_REG_IDR5 0x00200014u |
20 | #define IVPU_MMU_REG_CR0 0x00200020u |
21 | #define IVPU_MMU_REG_CR0ACK 0x00200024u |
22 | #define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0) |
23 | #define IVPU_MMU_REG_CR1 0x00200028u |
24 | #define IVPU_MMU_REG_CR2 0x0020002cu |
25 | #define IVPU_MMU_REG_IRQ_CTRL 0x00200050u |
26 | #define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u |
27 | #define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0) |
28 | |
29 | #define IVPU_MMU_REG_GERROR 0x00200060u |
30 | #define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0) |
31 | #define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2) |
32 | #define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3) |
33 | #define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4) |
34 | #define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5) |
35 | #define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6) |
36 | #define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7) |
37 | |
38 | #define IVPU_MMU_REG_GERRORN 0x00200064u |
39 | |
40 | #define IVPU_MMU_REG_STRTAB_BASE 0x00200080u |
41 | #define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u |
42 | #define IVPU_MMU_REG_CMDQ_BASE 0x00200090u |
43 | #define IVPU_MMU_REG_CMDQ_PROD 0x00200098u |
44 | #define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu |
45 | #define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0) |
46 | #define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24) |
47 | #define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u |
48 | #define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u |
49 | #define IVPU_MMU_REG_EVTQ_CONS 0x002000acu |
50 | #define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K) |
51 | #define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K) |
52 | |
53 | #define IVPU_MMU_IDR0_REF 0x080f3e0f |
54 | #define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f |
55 | #define IVPU_MMU_IDR1_REF 0x0e739d18 |
56 | #define IVPU_MMU_IDR3_REF 0x0000003c |
57 | #define IVPU_MMU_IDR5_REF 0x00040070 |
58 | #define IVPU_MMU_IDR5_REF_SIMICS 0x00000075 |
59 | #define IVPU_MMU_IDR5_REF_FPGA 0x00800075 |
60 | |
61 | #define IVPU_MMU_CDTAB_ENT_SIZE 64 |
62 | #define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */ |
63 | #define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2) |
64 | |
65 | #define IVPU_MMU_STREAM_ID0 0 |
66 | #define IVPU_MMU_STREAM_ID3 3 |
67 | |
68 | #define IVPU_MMU_STRTAB_ENT_SIZE 64 |
69 | #define IVPU_MMU_STRTAB_ENT_COUNT 4 |
70 | #define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2 |
71 | #define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE |
72 | |
73 | #define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */ |
74 | #define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2) |
75 | #define IVPU_MMU_Q_WRAP_MASK GENMASK(IVPU_MMU_Q_COUNT_LOG2, 0) |
76 | #define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1) |
77 | #define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK) |
78 | #define IVPU_MMU_Q_WRP(val) ((val) & IVPU_MMU_Q_COUNT) |
79 | |
80 | #define IVPU_MMU_CMDQ_CMD_SIZE 16 |
81 | #define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE) |
82 | |
83 | #define IVPU_MMU_EVTQ_CMD_SIZE 32 |
84 | #define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE) |
85 | |
86 | #define IVPU_MMU_CMD_OPCODE GENMASK(7, 0) |
87 | |
88 | #define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12) |
89 | #define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22) |
90 | #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) |
91 | #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) |
92 | #define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32) |
93 | |
94 | #define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10) |
95 | #define IVPU_MMU_CMD_CFGI_0_SSV BIT(11) |
96 | #define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12) |
97 | #define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32) |
98 | #define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0) |
99 | |
100 | #define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48) |
101 | #define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32) |
102 | |
103 | #define CMD_PREFETCH_CFG 0x1 |
104 | #define CMD_CFGI_STE 0x3 |
105 | #define CMD_CFGI_ALL 0x4 |
106 | #define CMD_CFGI_CD 0x5 |
107 | #define CMD_CFGI_CD_ALL 0x6 |
108 | #define CMD_TLBI_NH_ASID 0x11 |
109 | #define CMD_TLBI_EL2_ALL 0x20 |
110 | #define CMD_TLBI_NSNH_ALL 0x30 |
111 | #define CMD_SYNC 0x46 |
112 | |
113 | #define IVPU_MMU_EVT_F_UUT 0x01 |
114 | #define IVPU_MMU_EVT_C_BAD_STREAMID 0x02 |
115 | #define IVPU_MMU_EVT_F_STE_FETCH 0x03 |
116 | #define IVPU_MMU_EVT_C_BAD_STE 0x04 |
117 | #define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05 |
118 | #define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06 |
119 | #define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07 |
120 | #define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08 |
121 | #define IVPU_MMU_EVT_F_CD_FETCH 0x09 |
122 | #define IVPU_MMU_EVT_C_BAD_CD 0x0a |
123 | #define IVPU_MMU_EVT_F_WALK_EABT 0x0b |
124 | #define IVPU_MMU_EVT_F_TRANSLATION 0x10 |
125 | #define IVPU_MMU_EVT_F_ADDR_SIZE 0x11 |
126 | #define IVPU_MMU_EVT_F_ACCESS 0x12 |
127 | #define IVPU_MMU_EVT_F_PERMISSION 0x13 |
128 | #define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20 |
129 | #define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21 |
130 | #define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24 |
131 | #define IVPU_MMU_EVT_F_VMS_FETCH 0x25 |
132 | |
133 | #define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0) |
134 | #define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12) |
135 | |
136 | #define IVPU_MMU_Q_BASE_RWA BIT(62) |
137 | #define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) |
138 | #define IVPU_MMU_STRTAB_BASE_RA BIT(62) |
139 | #define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) |
140 | |
141 | #define IVPU_MMU_IRQ_EVTQ_EN BIT(2) |
142 | #define IVPU_MMU_IRQ_GERROR_EN BIT(0) |
143 | |
144 | #define IVPU_MMU_CR0_ATSCHK BIT(4) |
145 | #define IVPU_MMU_CR0_CMDQEN BIT(3) |
146 | #define IVPU_MMU_CR0_EVTQEN BIT(2) |
147 | #define IVPU_MMU_CR0_PRIQEN BIT(1) |
148 | #define IVPU_MMU_CR0_SMMUEN BIT(0) |
149 | |
150 | #define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10) |
151 | #define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8) |
152 | #define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6) |
153 | #define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4) |
154 | #define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2) |
155 | #define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0) |
156 | #define IVPU_MMU_CACHE_NC 0 |
157 | #define IVPU_MMU_CACHE_WB 1 |
158 | #define IVPU_MMU_CACHE_WT 2 |
159 | #define IVPU_MMU_SH_NSH 0 |
160 | #define IVPU_MMU_SH_OSH 2 |
161 | #define IVPU_MMU_SH_ISH 3 |
162 | |
163 | #define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0) |
164 | |
165 | #define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) |
166 | #define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6) |
167 | #define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) |
168 | #define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) |
169 | #define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12) |
170 | #define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14) |
171 | #define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30) |
172 | #define IVPU_MMU_CD_0_ENDI BIT(15) |
173 | #define IVPU_MMU_CD_0_V BIT(31) |
174 | #define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32) |
175 | #define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38) |
176 | #define IVPU_MMU_CD_0_AA64 BIT(41) |
177 | #define IVPU_MMU_CD_0_S BIT(44) |
178 | #define IVPU_MMU_CD_0_R BIT(45) |
179 | #define IVPU_MMU_CD_0_A BIT(46) |
180 | #define IVPU_MMU_CD_0_ASET BIT(47) |
181 | #define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) |
182 | |
183 | #define IVPU_MMU_T0SZ_48BIT 16 |
184 | #define IVPU_MMU_T0SZ_38BIT 26 |
185 | |
186 | #define IVPU_MMU_IPS_48BIT 5 |
187 | #define IVPU_MMU_IPS_44BIT 4 |
188 | #define IVPU_MMU_IPS_42BIT 3 |
189 | #define IVPU_MMU_IPS_40BIT 2 |
190 | #define IVPU_MMU_IPS_36BIT 1 |
191 | #define IVPU_MMU_IPS_32BIT 0 |
192 | |
193 | #define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) |
194 | |
195 | #define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) |
196 | #define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4) |
197 | #define IVPU_MMU_STE_0_S1FMT_LINEAR 0 |
198 | #define IVPU_MMU_STE_DWORDS 8 |
199 | #define IVPU_MMU_STE_0_CFG_S1_TRANS 5 |
200 | #define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1) |
201 | #define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) |
202 | #define IVPU_MMU_STE_0_V BIT(0) |
203 | |
204 | #define IVPU_MMU_STE_1_STRW_NSEL1 0ul |
205 | #define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13) |
206 | #define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30) |
207 | #define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48) |
208 | #define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul |
209 | #define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50) |
210 | #define IVPU_MMU_STE_1_INSTCFG_DATA 2ul |
211 | #define IVPU_MMU_STE_1_MEV BIT(19) |
212 | #define IVPU_MMU_STE_1_S1STALLD BIT(27) |
213 | #define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul |
214 | #define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul |
215 | #define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul |
216 | #define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul |
217 | #define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2) |
218 | #define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4) |
219 | #define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6) |
220 | #define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0) |
221 | #define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0 |
222 | |
223 | #define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC) |
224 | #define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC) |
225 | |
226 | #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \ |
227 | (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \ |
228 | (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \ |
229 | (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \ |
230 | (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \ |
231 | (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \ |
232 | (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT))) |
233 | |
234 | #define IVPU_MMU_CERROR_NONE 0x0 |
235 | #define IVPU_MMU_CERROR_ILL 0x1 |
236 | #define IVPU_MMU_CERROR_ABT 0x2 |
237 | #define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3 |
238 | |
239 | static const char *ivpu_mmu_event_to_str(u32 cmd) |
240 | { |
241 | switch (cmd) { |
242 | case IVPU_MMU_EVT_F_UUT: |
243 | return "Unsupported Upstream Transaction" ; |
244 | case IVPU_MMU_EVT_C_BAD_STREAMID: |
245 | return "Transaction StreamID out of range" ; |
246 | case IVPU_MMU_EVT_F_STE_FETCH: |
247 | return "Fetch of STE caused external abort" ; |
248 | case IVPU_MMU_EVT_C_BAD_STE: |
249 | return "Used STE invalid" ; |
250 | case IVPU_MMU_EVT_F_BAD_ATS_TREQ: |
251 | return "Address Request disallowed for a StreamID" ; |
252 | case IVPU_MMU_EVT_F_STREAM_DISABLED: |
253 | return "Transaction marks non-substream disabled" ; |
254 | case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN: |
255 | return "MMU bypass is disallowed for this StreamID" ; |
256 | case IVPU_MMU_EVT_C_BAD_SUBSTREAMID: |
257 | return "Invalid StreamID" ; |
258 | case IVPU_MMU_EVT_F_CD_FETCH: |
259 | return "Fetch of CD caused external abort" ; |
260 | case IVPU_MMU_EVT_C_BAD_CD: |
261 | return "Fetched CD invalid" ; |
262 | case IVPU_MMU_EVT_F_WALK_EABT: |
263 | return " An external abort occurred fetching a TLB" ; |
264 | case IVPU_MMU_EVT_F_TRANSLATION: |
265 | return "Translation fault" ; |
266 | case IVPU_MMU_EVT_F_ADDR_SIZE: |
267 | return " Output address caused address size fault" ; |
268 | case IVPU_MMU_EVT_F_ACCESS: |
269 | return "Access flag fault" ; |
270 | case IVPU_MMU_EVT_F_PERMISSION: |
271 | return "Permission fault occurred on page access" ; |
272 | case IVPU_MMU_EVT_F_TLB_CONFLICT: |
273 | return "A TLB conflict" ; |
274 | case IVPU_MMU_EVT_F_CFG_CONFLICT: |
275 | return "A configuration cache conflict" ; |
276 | case IVPU_MMU_EVT_E_PAGE_REQUEST: |
277 | return "Page request hint from a client device" ; |
278 | case IVPU_MMU_EVT_F_VMS_FETCH: |
279 | return "Fetch of VMS caused external abort" ; |
280 | default: |
281 | return "Unknown event" ; |
282 | } |
283 | } |
284 | |
285 | static const char *ivpu_mmu_cmdq_err_to_str(u32 err) |
286 | { |
287 | switch (err) { |
288 | case IVPU_MMU_CERROR_NONE: |
289 | return "No error" ; |
290 | case IVPU_MMU_CERROR_ILL: |
291 | return "Illegal command" ; |
292 | case IVPU_MMU_CERROR_ABT: |
293 | return "External abort on command queue read" ; |
294 | case IVPU_MMU_CERROR_ATC_INV_SYNC: |
295 | return "Sync failed to complete ATS invalidation" ; |
296 | default: |
297 | return "Unknown error" ; |
298 | } |
299 | } |
300 | |
301 | static void ivpu_mmu_config_check(struct ivpu_device *vdev) |
302 | { |
303 | u32 val_ref; |
304 | u32 val; |
305 | |
306 | if (ivpu_is_simics(vdev)) |
307 | val_ref = IVPU_MMU_IDR0_REF_SIMICS; |
308 | else |
309 | val_ref = IVPU_MMU_IDR0_REF; |
310 | |
311 | val = REGV_RD32(IVPU_MMU_REG_IDR0); |
312 | if (val != val_ref) |
313 | ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n" , val, val_ref); |
314 | |
315 | val = REGV_RD32(IVPU_MMU_REG_IDR1); |
316 | if (val != IVPU_MMU_IDR1_REF) |
317 | ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n" , val, IVPU_MMU_IDR1_REF); |
318 | |
319 | val = REGV_RD32(IVPU_MMU_REG_IDR3); |
320 | if (val != IVPU_MMU_IDR3_REF) |
321 | ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n" , val, IVPU_MMU_IDR3_REF); |
322 | |
323 | if (ivpu_is_simics(vdev)) |
324 | val_ref = IVPU_MMU_IDR5_REF_SIMICS; |
325 | else if (ivpu_is_fpga(vdev)) |
326 | val_ref = IVPU_MMU_IDR5_REF_FPGA; |
327 | else |
328 | val_ref = IVPU_MMU_IDR5_REF; |
329 | |
330 | val = REGV_RD32(IVPU_MMU_REG_IDR5); |
331 | if (val != val_ref) |
332 | ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n" , val, val_ref); |
333 | } |
334 | |
335 | static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) |
336 | { |
337 | struct ivpu_mmu_info *mmu = vdev->mmu; |
338 | struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; |
339 | size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE; |
340 | |
341 | cdtab->base = dmam_alloc_coherent(dev: vdev->drm.dev, size, dma_handle: &cdtab->dma, GFP_KERNEL); |
342 | if (!cdtab->base) |
343 | return -ENOMEM; |
344 | |
345 | ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n" , &cdtab->dma, size); |
346 | |
347 | return 0; |
348 | } |
349 | |
350 | static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) |
351 | { |
352 | struct ivpu_mmu_info *mmu = vdev->mmu; |
353 | struct ivpu_mmu_strtab *strtab = &mmu->strtab; |
354 | size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE; |
355 | |
356 | strtab->base = dmam_alloc_coherent(dev: vdev->drm.dev, size, dma_handle: &strtab->dma, GFP_KERNEL); |
357 | if (!strtab->base) |
358 | return -ENOMEM; |
359 | |
360 | strtab->base_cfg = IVPU_MMU_STRTAB_CFG; |
361 | strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA; |
362 | strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK; |
363 | |
364 | ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n" , |
365 | &strtab->dma, &strtab->dma_q, size); |
366 | |
367 | return 0; |
368 | } |
369 | |
370 | static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) |
371 | { |
372 | struct ivpu_mmu_info *mmu = vdev->mmu; |
373 | struct ivpu_mmu_queue *q = &mmu->cmdq; |
374 | |
375 | q->base = dmam_alloc_coherent(dev: vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, dma_handle: &q->dma, GFP_KERNEL); |
376 | if (!q->base) |
377 | return -ENOMEM; |
378 | |
379 | q->dma_q = IVPU_MMU_Q_BASE_RWA; |
380 | q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; |
381 | q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; |
382 | |
383 | ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n" , |
384 | &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE); |
385 | |
386 | return 0; |
387 | } |
388 | |
389 | static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) |
390 | { |
391 | struct ivpu_mmu_info *mmu = vdev->mmu; |
392 | struct ivpu_mmu_queue *q = &mmu->evtq; |
393 | |
394 | q->base = dmam_alloc_coherent(dev: vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, dma_handle: &q->dma, GFP_KERNEL); |
395 | if (!q->base) |
396 | return -ENOMEM; |
397 | |
398 | q->dma_q = IVPU_MMU_Q_BASE_RWA; |
399 | q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; |
400 | q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; |
401 | |
402 | ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n" , |
403 | &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE); |
404 | |
405 | return 0; |
406 | } |
407 | |
408 | static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) |
409 | { |
410 | int ret; |
411 | |
412 | ret = ivpu_mmu_cdtab_alloc(vdev); |
413 | if (ret) { |
414 | ivpu_err(vdev, "Failed to allocate cdtab: %d\n" , ret); |
415 | return ret; |
416 | } |
417 | |
418 | ret = ivpu_mmu_strtab_alloc(vdev); |
419 | if (ret) { |
420 | ivpu_err(vdev, "Failed to allocate strtab: %d\n" , ret); |
421 | return ret; |
422 | } |
423 | |
424 | ret = ivpu_mmu_cmdq_alloc(vdev); |
425 | if (ret) { |
426 | ivpu_err(vdev, "Failed to allocate cmdq: %d\n" , ret); |
427 | return ret; |
428 | } |
429 | |
430 | ret = ivpu_mmu_evtq_alloc(vdev); |
431 | if (ret) |
432 | ivpu_err(vdev, "Failed to allocate evtq: %d\n" , ret); |
433 | |
434 | return ret; |
435 | } |
436 | |
437 | static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val) |
438 | { |
439 | REGV_WR32(IVPU_MMU_REG_CR0, val); |
440 | |
441 | return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); |
442 | } |
443 | |
444 | static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val) |
445 | { |
446 | REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val); |
447 | |
448 | return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); |
449 | } |
450 | |
451 | static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) |
452 | { |
453 | u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN; |
454 | int ret; |
455 | |
456 | ret = ivpu_mmu_reg_write_irq_ctrl(vdev, val: 0); |
457 | if (ret) |
458 | return ret; |
459 | |
460 | return ivpu_mmu_reg_write_irq_ctrl(vdev, val: irq_ctrl); |
461 | } |
462 | |
463 | static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) |
464 | { |
465 | struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; |
466 | int ret; |
467 | |
468 | ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod, |
469 | IVPU_MMU_QUEUE_TIMEOUT_US); |
470 | if (ret) |
471 | return ret; |
472 | |
473 | cmdq->cons = cmdq->prod; |
474 | |
475 | return 0; |
476 | } |
477 | |
478 | static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q) |
479 | { |
480 | return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) && |
481 | (IVPU_MMU_Q_WRP(q->prod) != IVPU_MMU_Q_WRP(q->cons))); |
482 | } |
483 | |
484 | static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q) |
485 | { |
486 | return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) && |
487 | (IVPU_MMU_Q_WRP(q->prod) == IVPU_MMU_Q_WRP(q->cons))); |
488 | } |
489 | |
490 | static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) |
491 | { |
492 | struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; |
493 | u64 *queue_buffer = cmdq->base; |
494 | int idx = IVPU_MMU_Q_IDX(cmdq->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer)); |
495 | |
496 | if (ivpu_mmu_queue_is_full(q: cmdq)) { |
497 | ivpu_err(vdev, "Failed to write MMU CMD %s\n" , name); |
498 | return -EBUSY; |
499 | } |
500 | |
501 | queue_buffer[idx] = data0; |
502 | queue_buffer[idx + 1] = data1; |
503 | cmdq->prod = (cmdq->prod + 1) & IVPU_MMU_Q_WRAP_MASK; |
504 | |
505 | ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n" , name, data0, data1); |
506 | |
507 | return 0; |
508 | } |
509 | |
510 | static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) |
511 | { |
512 | struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; |
513 | u64 val; |
514 | int ret; |
515 | |
516 | val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC); |
517 | |
518 | ret = ivpu_mmu_cmdq_cmd_write(vdev, name: "SYNC" , data0: val, data1: 0); |
519 | if (ret) |
520 | return ret; |
521 | |
522 | clflush_cache_range(addr: q->base, IVPU_MMU_CMDQ_SIZE); |
523 | REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod); |
524 | |
525 | ret = ivpu_mmu_cmdq_wait_for_cons(vdev); |
526 | if (ret) { |
527 | u32 err; |
528 | |
529 | val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS); |
530 | err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val); |
531 | |
532 | ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n" , ret, |
533 | ivpu_mmu_cmdq_err_to_str(err)); |
534 | ivpu_hw_diagnose_failure(vdev); |
535 | } |
536 | |
537 | return ret; |
538 | } |
539 | |
540 | static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) |
541 | { |
542 | u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL); |
543 | u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f); |
544 | |
545 | return ivpu_mmu_cmdq_cmd_write(vdev, name: "CFGI_ALL" , data0, data1); |
546 | } |
547 | |
548 | static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) |
549 | { |
550 | u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) | |
551 | FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid); |
552 | |
553 | return ivpu_mmu_cmdq_cmd_write(vdev, name: "TLBI_NH_ASID" , data0: val, data1: 0); |
554 | } |
555 | |
556 | static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) |
557 | { |
558 | u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL); |
559 | |
560 | return ivpu_mmu_cmdq_cmd_write(vdev, name: "TLBI_NSNH_ALL" , data0: val, data1: 0); |
561 | } |
562 | |
563 | static int ivpu_mmu_reset(struct ivpu_device *vdev) |
564 | { |
565 | struct ivpu_mmu_info *mmu = vdev->mmu; |
566 | u32 val; |
567 | int ret; |
568 | |
569 | memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE); |
570 | clflush_cache_range(addr: mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE); |
571 | mmu->cmdq.prod = 0; |
572 | mmu->cmdq.cons = 0; |
573 | |
574 | memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE); |
575 | mmu->evtq.prod = 0; |
576 | mmu->evtq.cons = 0; |
577 | |
578 | ret = ivpu_mmu_reg_write_cr0(vdev, val: 0); |
579 | if (ret) |
580 | return ret; |
581 | |
582 | val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) | |
583 | FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) | |
584 | FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) | |
585 | FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) | |
586 | FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) | |
587 | FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB); |
588 | REGV_WR32(IVPU_MMU_REG_CR1, val); |
589 | |
590 | REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q); |
591 | REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg); |
592 | |
593 | REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q); |
594 | REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0); |
595 | REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0); |
596 | |
597 | val = IVPU_MMU_CR0_CMDQEN; |
598 | ret = ivpu_mmu_reg_write_cr0(vdev, val); |
599 | if (ret) |
600 | return ret; |
601 | |
602 | ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); |
603 | if (ret) |
604 | return ret; |
605 | |
606 | ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); |
607 | if (ret) |
608 | return ret; |
609 | |
610 | ret = ivpu_mmu_cmdq_sync(vdev); |
611 | if (ret) |
612 | return ret; |
613 | |
614 | REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q); |
615 | REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0); |
616 | REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0); |
617 | |
618 | val |= IVPU_MMU_CR0_EVTQEN; |
619 | ret = ivpu_mmu_reg_write_cr0(vdev, val); |
620 | if (ret) |
621 | return ret; |
622 | |
623 | val |= IVPU_MMU_CR0_ATSCHK; |
624 | ret = ivpu_mmu_reg_write_cr0(vdev, val); |
625 | if (ret) |
626 | return ret; |
627 | |
628 | ret = ivpu_mmu_irqs_setup(vdev); |
629 | if (ret) |
630 | return ret; |
631 | |
632 | val |= IVPU_MMU_CR0_SMMUEN; |
633 | return ivpu_mmu_reg_write_cr0(vdev, val); |
634 | } |
635 | |
636 | static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) |
637 | { |
638 | struct ivpu_mmu_info *mmu = vdev->mmu; |
639 | struct ivpu_mmu_strtab *strtab = &mmu->strtab; |
640 | struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; |
641 | u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE); |
642 | u64 str[2]; |
643 | |
644 | str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) | |
645 | FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) | |
646 | FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) | |
647 | IVPU_MMU_STE_0_V | |
648 | (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK); |
649 | |
650 | str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) | |
651 | FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) | |
652 | FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) | |
653 | FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) | |
654 | FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) | |
655 | FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) | |
656 | FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) | |
657 | FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) | |
658 | IVPU_MMU_STE_1_MEV | |
659 | IVPU_MMU_STE_1_S1STALLD; |
660 | |
661 | WRITE_ONCE(entry[1], str[1]); |
662 | WRITE_ONCE(entry[0], str[0]); |
663 | |
664 | clflush_cache_range(addr: entry, IVPU_MMU_STRTAB_ENT_SIZE); |
665 | |
666 | ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n" , sid, str[0], str[1]); |
667 | } |
668 | |
669 | static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) |
670 | { |
671 | ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); |
672 | ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); |
673 | |
674 | return 0; |
675 | } |
676 | |
677 | int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) |
678 | { |
679 | struct ivpu_mmu_info *mmu = vdev->mmu; |
680 | int ret = 0; |
681 | |
682 | mutex_lock(&mmu->lock); |
683 | if (!mmu->on) |
684 | goto unlock; |
685 | |
686 | ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); |
687 | if (ret) |
688 | goto unlock; |
689 | |
690 | ret = ivpu_mmu_cmdq_sync(vdev); |
691 | unlock: |
692 | mutex_unlock(lock: &mmu->lock); |
693 | return ret; |
694 | } |
695 | |
696 | static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) |
697 | { |
698 | struct ivpu_mmu_info *mmu = vdev->mmu; |
699 | struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; |
700 | u64 *entry; |
701 | u64 cd[4]; |
702 | int ret = 0; |
703 | |
704 | if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) |
705 | return -EINVAL; |
706 | |
707 | entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); |
708 | |
709 | if (cd_dma != 0) { |
710 | cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) | |
711 | FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | |
712 | FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | |
713 | FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | |
714 | FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | |
715 | FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) | |
716 | FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | |
717 | IVPU_MMU_CD_0_TCR_EPD1 | |
718 | IVPU_MMU_CD_0_AA64 | |
719 | IVPU_MMU_CD_0_R | |
720 | IVPU_MMU_CD_0_ASET | |
721 | IVPU_MMU_CD_0_V; |
722 | cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK; |
723 | cd[2] = 0; |
724 | cd[3] = 0x0000000000007444; |
725 | |
726 | /* For global context generate memory fault on VPU */ |
727 | if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) |
728 | cd[0] |= IVPU_MMU_CD_0_A; |
729 | } else { |
730 | memset(cd, 0, sizeof(cd)); |
731 | } |
732 | |
733 | WRITE_ONCE(entry[1], cd[1]); |
734 | WRITE_ONCE(entry[2], cd[2]); |
735 | WRITE_ONCE(entry[3], cd[3]); |
736 | WRITE_ONCE(entry[0], cd[0]); |
737 | |
738 | clflush_cache_range(addr: entry, IVPU_MMU_CDTAB_ENT_SIZE); |
739 | |
740 | ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n" , |
741 | cd_dma ? "write" : "clear" , ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]); |
742 | |
743 | mutex_lock(&mmu->lock); |
744 | if (!mmu->on) |
745 | goto unlock; |
746 | |
747 | ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); |
748 | if (ret) |
749 | goto unlock; |
750 | |
751 | ret = ivpu_mmu_cmdq_sync(vdev); |
752 | unlock: |
753 | mutex_unlock(lock: &mmu->lock); |
754 | return ret; |
755 | } |
756 | |
757 | static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev) |
758 | { |
759 | int ret; |
760 | |
761 | ret = ivpu_mmu_cd_add(vdev, ssid: 0, cd_dma: vdev->gctx.pgtable.pgd_dma); |
762 | if (ret) |
763 | ivpu_err(vdev, "Failed to add global CD entry: %d\n" , ret); |
764 | |
765 | return ret; |
766 | } |
767 | |
768 | static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma) |
769 | { |
770 | int ret; |
771 | |
772 | if (ssid == 0) { |
773 | ivpu_err(vdev, "Invalid SSID: %u\n" , ssid); |
774 | return -EINVAL; |
775 | } |
776 | |
777 | ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma); |
778 | if (ret) |
779 | ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n" , ssid, ret); |
780 | |
781 | return ret; |
782 | } |
783 | |
784 | int ivpu_mmu_init(struct ivpu_device *vdev) |
785 | { |
786 | struct ivpu_mmu_info *mmu = vdev->mmu; |
787 | int ret; |
788 | |
789 | ivpu_dbg(vdev, MMU, "Init..\n" ); |
790 | |
791 | ivpu_mmu_config_check(vdev); |
792 | |
793 | ret = drmm_mutex_init(&vdev->drm, &mmu->lock); |
794 | if (ret) |
795 | return ret; |
796 | |
797 | ret = ivpu_mmu_structs_alloc(vdev); |
798 | if (ret) |
799 | return ret; |
800 | |
801 | ret = ivpu_mmu_strtab_init(vdev); |
802 | if (ret) { |
803 | ivpu_err(vdev, "Failed to initialize strtab: %d\n" , ret); |
804 | return ret; |
805 | } |
806 | |
807 | ret = ivpu_mmu_cd_add_gbl(vdev); |
808 | if (ret) { |
809 | ivpu_err(vdev, "Failed to initialize strtab: %d\n" , ret); |
810 | return ret; |
811 | } |
812 | |
813 | ret = ivpu_mmu_enable(vdev); |
814 | if (ret) { |
815 | ivpu_err(vdev, "Failed to resume MMU: %d\n" , ret); |
816 | return ret; |
817 | } |
818 | |
819 | ivpu_dbg(vdev, MMU, "Init done\n" ); |
820 | |
821 | return 0; |
822 | } |
823 | |
824 | int ivpu_mmu_enable(struct ivpu_device *vdev) |
825 | { |
826 | struct ivpu_mmu_info *mmu = vdev->mmu; |
827 | int ret; |
828 | |
829 | mutex_lock(&mmu->lock); |
830 | |
831 | mmu->on = true; |
832 | |
833 | ret = ivpu_mmu_reset(vdev); |
834 | if (ret) { |
835 | ivpu_err(vdev, "Failed to reset MMU: %d\n" , ret); |
836 | goto err; |
837 | } |
838 | |
839 | ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); |
840 | if (ret) |
841 | goto err; |
842 | |
843 | ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); |
844 | if (ret) |
845 | goto err; |
846 | |
847 | ret = ivpu_mmu_cmdq_sync(vdev); |
848 | if (ret) |
849 | goto err; |
850 | |
851 | mutex_unlock(lock: &mmu->lock); |
852 | |
853 | return 0; |
854 | err: |
855 | mmu->on = false; |
856 | mutex_unlock(lock: &mmu->lock); |
857 | return ret; |
858 | } |
859 | |
860 | void ivpu_mmu_disable(struct ivpu_device *vdev) |
861 | { |
862 | struct ivpu_mmu_info *mmu = vdev->mmu; |
863 | |
864 | mutex_lock(&mmu->lock); |
865 | mmu->on = false; |
866 | mutex_unlock(lock: &mmu->lock); |
867 | } |
868 | |
869 | static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) |
870 | { |
871 | u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); |
872 | u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]); |
873 | u64 fetch_addr = ((u64)event[7]) << 32 | event[6]; |
874 | u64 in_addr = ((u64)event[5]) << 32 | event[4]; |
875 | u32 sid = event[1]; |
876 | |
877 | ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n" , |
878 | op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr); |
879 | } |
880 | |
881 | static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) |
882 | { |
883 | struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; |
884 | u32 idx = IVPU_MMU_Q_IDX(evtq->cons); |
885 | u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE); |
886 | |
887 | evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); |
888 | if (ivpu_mmu_queue_is_empty(q: evtq)) |
889 | return NULL; |
890 | |
891 | evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK; |
892 | return evt; |
893 | } |
894 | |
895 | void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) |
896 | { |
897 | u32 *event; |
898 | u32 ssid; |
899 | |
900 | ivpu_dbg(vdev, IRQ, "MMU event queue\n" ); |
901 | |
902 | while ((event = ivpu_mmu_get_event(vdev)) != NULL) { |
903 | ivpu_mmu_dump_event(vdev, event); |
904 | |
905 | ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); |
906 | if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) { |
907 | ivpu_pm_trigger_recovery(vdev, reason: "MMU event" ); |
908 | return; |
909 | } |
910 | |
911 | ivpu_mmu_user_context_mark_invalid(vdev, ssid); |
912 | REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons); |
913 | } |
914 | } |
915 | |
916 | void ivpu_mmu_evtq_dump(struct ivpu_device *vdev) |
917 | { |
918 | u32 *event; |
919 | |
920 | while ((event = ivpu_mmu_get_event(vdev)) != NULL) |
921 | ivpu_mmu_dump_event(vdev, event); |
922 | } |
923 | |
924 | void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) |
925 | { |
926 | u32 gerror_val, gerrorn_val, active; |
927 | |
928 | ivpu_dbg(vdev, IRQ, "MMU error\n" ); |
929 | |
930 | gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR); |
931 | gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN); |
932 | |
933 | active = gerror_val ^ gerrorn_val; |
934 | if (!(active & IVPU_MMU_GERROR_ERR_MASK)) |
935 | return; |
936 | |
937 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active)) |
938 | ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n" ); |
939 | |
940 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active)) |
941 | ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n" ); |
942 | |
943 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active)) |
944 | ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n" ); |
945 | |
946 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active)) |
947 | ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n" ); |
948 | |
949 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active)) |
950 | ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n" ); |
951 | |
952 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active)) |
953 | ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n" ); |
954 | |
955 | if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active)) |
956 | ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n" ); |
957 | |
958 | REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val); |
959 | } |
960 | |
961 | int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) |
962 | { |
963 | return ivpu_mmu_cd_add_user(vdev, ssid, cd_dma: pgtable->pgd_dma); |
964 | } |
965 | |
966 | void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid) |
967 | { |
968 | ivpu_mmu_cd_add_user(vdev, ssid, cd_dma: 0); /* 0 will clear CD entry */ |
969 | } |
970 | |