1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2008-2009 ST-Ericsson AB |
4 | * TCM memory handling for ARM systems |
5 | * |
6 | * Author: Linus Walleij <linus.walleij@stericsson.com> |
7 | * Author: Rickard Andersson <rickard.andersson@stericsson.com> |
8 | */ |
9 | #include <linux/init.h> |
10 | #include <linux/kernel.h> |
11 | #include <linux/module.h> |
12 | #include <linux/stddef.h> |
13 | #include <linux/ioport.h> |
14 | #include <linux/genalloc.h> |
15 | #include <linux/string.h> /* memcpy */ |
16 | #include <asm/cputype.h> |
17 | #include <asm/mach/map.h> |
18 | #include <asm/page.h> |
19 | #include <asm/system_info.h> |
20 | #include <asm/traps.h> |
21 | #include <asm/tcm.h> |
22 | |
23 | #define TCMTR_FORMAT_MASK 0xe0000000U |
24 | |
25 | static struct gen_pool *tcm_pool; |
26 | static bool dtcm_present; |
27 | static bool itcm_present; |
28 | |
29 | /* TCM section definitions from the linker */ |
30 | extern char __itcm_start, __sitcm_text, __eitcm_text; |
31 | extern char __dtcm_start, __sdtcm_data, __edtcm_data; |
32 | |
33 | /* These will be increased as we run */ |
34 | static u32 dtcm_end = DTCM_OFFSET; |
35 | static u32 itcm_end = ITCM_OFFSET; |
36 | |
37 | /* |
38 | * TCM memory resources |
39 | */ |
40 | static struct resource dtcm_res = { |
41 | .name = "DTCM RAM" , |
42 | .start = DTCM_OFFSET, |
43 | .end = DTCM_OFFSET, |
44 | .flags = IORESOURCE_MEM |
45 | }; |
46 | |
47 | static struct resource itcm_res = { |
48 | .name = "ITCM RAM" , |
49 | .start = ITCM_OFFSET, |
50 | .end = ITCM_OFFSET, |
51 | .flags = IORESOURCE_MEM |
52 | }; |
53 | |
54 | static struct map_desc dtcm_iomap[] __initdata = { |
55 | { |
56 | .virtual = DTCM_OFFSET, |
57 | .pfn = __phys_to_pfn(DTCM_OFFSET), |
58 | .length = 0, |
59 | .type = MT_MEMORY_RW_DTCM |
60 | } |
61 | }; |
62 | |
63 | static struct map_desc itcm_iomap[] __initdata = { |
64 | { |
65 | .virtual = ITCM_OFFSET, |
66 | .pfn = __phys_to_pfn(ITCM_OFFSET), |
67 | .length = 0, |
68 | .type = MT_MEMORY_RWX_ITCM, |
69 | } |
70 | }; |
71 | |
72 | /* |
73 | * Allocate a chunk of TCM memory |
74 | */ |
75 | void *tcm_alloc(size_t len) |
76 | { |
77 | unsigned long vaddr; |
78 | |
79 | if (!tcm_pool) |
80 | return NULL; |
81 | |
82 | vaddr = gen_pool_alloc(pool: tcm_pool, size: len); |
83 | if (!vaddr) |
84 | return NULL; |
85 | |
86 | return (void *) vaddr; |
87 | } |
88 | EXPORT_SYMBOL(tcm_alloc); |
89 | |
90 | /* |
91 | * Free a chunk of TCM memory |
92 | */ |
93 | void tcm_free(void *addr, size_t len) |
94 | { |
95 | gen_pool_free(pool: tcm_pool, addr: (unsigned long) addr, size: len); |
96 | } |
97 | EXPORT_SYMBOL(tcm_free); |
98 | |
99 | bool tcm_dtcm_present(void) |
100 | { |
101 | return dtcm_present; |
102 | } |
103 | EXPORT_SYMBOL(tcm_dtcm_present); |
104 | |
105 | bool tcm_itcm_present(void) |
106 | { |
107 | return itcm_present; |
108 | } |
109 | EXPORT_SYMBOL(tcm_itcm_present); |
110 | |
111 | static int __init setup_tcm_bank(u8 type, u8 bank, u8 banks, |
112 | u32 *offset) |
113 | { |
114 | const int tcm_sizes[16] = { 0, -1, -1, 4, 8, 16, 32, 64, 128, |
115 | 256, 512, 1024, -1, -1, -1, -1 }; |
116 | u32 tcm_region; |
117 | int tcm_size; |
118 | |
119 | /* |
120 | * If there are more than one TCM bank of this type, |
121 | * select the TCM bank to operate on in the TCM selection |
122 | * register. |
123 | */ |
124 | if (banks > 1) |
125 | asm("mcr p15, 0, %0, c9, c2, 0" |
126 | : /* No output operands */ |
127 | : "r" (bank)); |
128 | |
129 | /* Read the special TCM region register c9, 0 */ |
130 | if (!type) |
131 | asm("mrc p15, 0, %0, c9, c1, 0" |
132 | : "=r" (tcm_region)); |
133 | else |
134 | asm("mrc p15, 0, %0, c9, c1, 1" |
135 | : "=r" (tcm_region)); |
136 | |
137 | tcm_size = tcm_sizes[(tcm_region >> 2) & 0x0f]; |
138 | if (tcm_size < 0) { |
139 | pr_err("CPU: %sTCM%d of unknown size\n" , |
140 | type ? "I" : "D" , bank); |
141 | return -EINVAL; |
142 | } else if (tcm_size > 32) { |
143 | pr_err("CPU: %sTCM%d larger than 32k found\n" , |
144 | type ? "I" : "D" , bank); |
145 | return -EINVAL; |
146 | } else { |
147 | pr_info("CPU: found %sTCM%d %dk @ %08x, %senabled\n" , |
148 | type ? "I" : "D" , |
149 | bank, |
150 | tcm_size, |
151 | (tcm_region & 0xfffff000U), |
152 | (tcm_region & 1) ? "" : "not " ); |
153 | } |
154 | |
155 | /* Not much fun you can do with a size 0 bank */ |
156 | if (tcm_size == 0) |
157 | return 0; |
158 | |
159 | /* Force move the TCM bank to where we want it, enable */ |
160 | tcm_region = *offset | (tcm_region & 0x00000ffeU) | 1; |
161 | |
162 | if (!type) |
163 | asm("mcr p15, 0, %0, c9, c1, 0" |
164 | : /* No output operands */ |
165 | : "r" (tcm_region)); |
166 | else |
167 | asm("mcr p15, 0, %0, c9, c1, 1" |
168 | : /* No output operands */ |
169 | : "r" (tcm_region)); |
170 | |
171 | /* Increase offset */ |
172 | *offset += (tcm_size << 10); |
173 | |
174 | pr_info("CPU: moved %sTCM%d %dk to %08x, enabled\n" , |
175 | type ? "I" : "D" , |
176 | bank, |
177 | tcm_size, |
178 | (tcm_region & 0xfffff000U)); |
179 | return 0; |
180 | } |
181 | |
182 | /* |
183 | * When we are running in the non-secure world and the secure world |
184 | * has not explicitly given us access to the TCM we will get an |
185 | * undefined error when reading the TCM region register in the |
186 | * setup_tcm_bank function (above). |
187 | * |
188 | * There are two variants of this register read that we need to trap, |
189 | * the read for the data TCM and the read for the instruction TCM: |
190 | * c0370628: ee196f11 mrc 15, 0, r6, cr9, cr1, {0} |
191 | * c0370674: ee196f31 mrc 15, 0, r6, cr9, cr1, {1} |
192 | * |
193 | * Our undef hook mask explicitly matches all fields of the encoded |
194 | * instruction other than the destination register. The mask also |
195 | * only allows operand 2 to have the values 0 or 1. |
196 | * |
197 | * The undefined hook is defined as __init and __initdata, and therefore |
198 | * must be removed before tcm_init returns. |
199 | * |
200 | * In this particular case (MRC with ARM condition code ALways) the |
201 | * Thumb-2 and ARM instruction encoding are identical, so this hook |
202 | * will work on a Thumb-2 kernel. |
203 | * |
204 | * See A8.8.107, DDI0406C_C ARM Architecture Reference Manual, Encoding |
205 | * T1/A1 for the bit-by-bit details. |
206 | * |
207 | * mrc p15, 0, XX, c9, c1, 0 |
208 | * mrc p15, 0, XX, c9, c1, 1 |
209 | * | | | | | | | +---- opc2 0|1 = 000|001 |
210 | * | | | | | | +------- CRm 0 = 0001 |
211 | * | | | | | +----------- CRn 0 = 1001 |
212 | * | | | | +--------------- Rt ? = ???? |
213 | * | | | +------------------- opc1 0 = 000 |
214 | * | | +----------------------- coproc 15 = 1111 |
215 | * | +-------------------------- condition ALways = 1110 |
216 | * +----------------------------- instruction MRC = 1110 |
217 | * |
218 | * Encoding this as per A8.8.107 of DDI0406C, Encoding T1/A1, yields: |
219 | * 1111 1111 1111 1111 0000 1111 1101 1111 Required Mask |
220 | * 1110 1110 0001 1001 ???? 1111 0001 0001 mrc p15, 0, XX, c9, c1, 0 |
221 | * 1110 1110 0001 1001 ???? 1111 0011 0001 mrc p15, 0, XX, c9, c1, 1 |
222 | * [ ] [ ] [ ]| [ ] [ ] [ ] [ ]| +--- CRm |
223 | * | | | | | | | | +----- SBO |
224 | * | | | | | | | +------- opc2 |
225 | * | | | | | | +----------- coproc |
226 | * | | | | | +---------------- Rt |
227 | * | | | | +--------------------- CRn |
228 | * | | | +------------------------- SBO |
229 | * | | +--------------------------- opc1 |
230 | * | +------------------------------- instruction |
231 | * +------------------------------------ condition |
232 | */ |
233 | #define TCM_REGION_READ_MASK 0xffff0fdf |
234 | #define TCM_REGION_READ_INSTR 0xee190f11 |
235 | #define DEST_REG_SHIFT 12 |
236 | #define DEST_REG_MASK 0xf |
237 | |
238 | static int __init tcm_handler(struct pt_regs *regs, unsigned int instr) |
239 | { |
240 | regs->uregs[(instr >> DEST_REG_SHIFT) & DEST_REG_MASK] = 0; |
241 | regs->ARM_pc += 4; |
242 | return 0; |
243 | } |
244 | |
245 | static struct undef_hook tcm_hook __initdata = { |
246 | .instr_mask = TCM_REGION_READ_MASK, |
247 | .instr_val = TCM_REGION_READ_INSTR, |
248 | .cpsr_mask = MODE_MASK, |
249 | .cpsr_val = SVC_MODE, |
250 | .fn = tcm_handler |
251 | }; |
252 | |
253 | /* |
254 | * This initializes the TCM memory |
255 | */ |
256 | void __init tcm_init(void) |
257 | { |
258 | u32 tcm_status; |
259 | u8 dtcm_banks; |
260 | u8 itcm_banks; |
261 | size_t dtcm_code_sz = &__edtcm_data - &__sdtcm_data; |
262 | size_t itcm_code_sz = &__eitcm_text - &__sitcm_text; |
263 | char *start; |
264 | char *end; |
265 | char *ram; |
266 | int ret; |
267 | int i; |
268 | |
269 | /* |
270 | * Prior to ARMv5 there is no TCM, and trying to read the status |
271 | * register will hang the processor. |
272 | */ |
273 | if (cpu_architecture() < CPU_ARCH_ARMv5) { |
274 | if (dtcm_code_sz || itcm_code_sz) |
275 | pr_info("CPU TCM: %u bytes of DTCM and %u bytes of " |
276 | "ITCM code compiled in, but no TCM present " |
277 | "in pre-v5 CPU\n" , dtcm_code_sz, itcm_code_sz); |
278 | return; |
279 | } |
280 | |
281 | tcm_status = read_cpuid_tcmstatus(); |
282 | |
283 | /* |
284 | * This code only supports v6-compatible TCMTR implementations. |
285 | */ |
286 | if (tcm_status & TCMTR_FORMAT_MASK) |
287 | return; |
288 | |
289 | dtcm_banks = (tcm_status >> 16) & 0x03; |
290 | itcm_banks = (tcm_status & 0x03); |
291 | |
292 | register_undef_hook(&tcm_hook); |
293 | |
294 | /* Values greater than 2 for D/ITCM banks are "reserved" */ |
295 | if (dtcm_banks > 2) |
296 | dtcm_banks = 0; |
297 | if (itcm_banks > 2) |
298 | itcm_banks = 0; |
299 | |
300 | /* Setup DTCM if present */ |
301 | if (dtcm_banks > 0) { |
302 | for (i = 0; i < dtcm_banks; i++) { |
303 | ret = setup_tcm_bank(type: 0, bank: i, banks: dtcm_banks, offset: &dtcm_end); |
304 | if (ret) |
305 | goto unregister; |
306 | } |
307 | /* This means you compiled more code than fits into DTCM */ |
308 | if (dtcm_code_sz > (dtcm_end - DTCM_OFFSET)) { |
309 | pr_info("CPU DTCM: %u bytes of code compiled to " |
310 | "DTCM but only %lu bytes of DTCM present\n" , |
311 | dtcm_code_sz, (dtcm_end - DTCM_OFFSET)); |
312 | goto no_dtcm; |
313 | } |
314 | /* |
315 | * This means that the DTCM sizes were 0 or the DTCM banks |
316 | * were inaccessible due to TrustZone configuration. |
317 | */ |
318 | if (!(dtcm_end - DTCM_OFFSET)) |
319 | goto no_dtcm; |
320 | dtcm_res.end = dtcm_end - 1; |
321 | request_resource(root: &iomem_resource, new: &dtcm_res); |
322 | dtcm_iomap[0].length = dtcm_end - DTCM_OFFSET; |
323 | iotable_init(dtcm_iomap, 1); |
324 | /* Copy data from RAM to DTCM */ |
325 | start = &__sdtcm_data; |
326 | end = &__edtcm_data; |
327 | ram = &__dtcm_start; |
328 | memcpy(start, ram, dtcm_code_sz); |
329 | pr_debug("CPU DTCM: copied data from %p - %p\n" , |
330 | start, end); |
331 | dtcm_present = true; |
332 | } else if (dtcm_code_sz) { |
333 | pr_info("CPU DTCM: %u bytes of code compiled to DTCM but no " |
334 | "DTCM banks present in CPU\n" , dtcm_code_sz); |
335 | } |
336 | |
337 | no_dtcm: |
338 | /* Setup ITCM if present */ |
339 | if (itcm_banks > 0) { |
340 | for (i = 0; i < itcm_banks; i++) { |
341 | ret = setup_tcm_bank(type: 1, bank: i, banks: itcm_banks, offset: &itcm_end); |
342 | if (ret) |
343 | goto unregister; |
344 | } |
345 | /* This means you compiled more code than fits into ITCM */ |
346 | if (itcm_code_sz > (itcm_end - ITCM_OFFSET)) { |
347 | pr_info("CPU ITCM: %u bytes of code compiled to " |
348 | "ITCM but only %lu bytes of ITCM present\n" , |
349 | itcm_code_sz, (itcm_end - ITCM_OFFSET)); |
350 | goto unregister; |
351 | } |
352 | /* |
353 | * This means that the ITCM sizes were 0 or the ITCM banks |
354 | * were inaccessible due to TrustZone configuration. |
355 | */ |
356 | if (!(itcm_end - ITCM_OFFSET)) |
357 | goto unregister; |
358 | itcm_res.end = itcm_end - 1; |
359 | request_resource(root: &iomem_resource, new: &itcm_res); |
360 | itcm_iomap[0].length = itcm_end - ITCM_OFFSET; |
361 | iotable_init(itcm_iomap, 1); |
362 | /* Copy code from RAM to ITCM */ |
363 | start = &__sitcm_text; |
364 | end = &__eitcm_text; |
365 | ram = &__itcm_start; |
366 | memcpy(start, ram, itcm_code_sz); |
367 | pr_debug("CPU ITCM: copied code from %p - %p\n" , |
368 | start, end); |
369 | itcm_present = true; |
370 | } else if (itcm_code_sz) { |
371 | pr_info("CPU ITCM: %u bytes of code compiled to ITCM but no " |
372 | "ITCM banks present in CPU\n" , itcm_code_sz); |
373 | } |
374 | |
375 | unregister: |
376 | unregister_undef_hook(&tcm_hook); |
377 | } |
378 | |
379 | /* |
380 | * This creates the TCM memory pool and has to be done later, |
381 | * during the core_initicalls, since the allocator is not yet |
382 | * up and running when the first initialization runs. |
383 | */ |
384 | static int __init setup_tcm_pool(void) |
385 | { |
386 | u32 dtcm_pool_start = (u32) &__edtcm_data; |
387 | u32 itcm_pool_start = (u32) &__eitcm_text; |
388 | int ret; |
389 | |
390 | /* |
391 | * Set up malloc pool, 2^2 = 4 bytes granularity since |
392 | * the TCM is sometimes just 4 KiB. NB: pages and cache |
393 | * line alignments does not matter in TCM! |
394 | */ |
395 | tcm_pool = gen_pool_create(2, -1); |
396 | |
397 | pr_debug("Setting up TCM memory pool\n" ); |
398 | |
399 | /* Add the rest of DTCM to the TCM pool */ |
400 | if (dtcm_present) { |
401 | if (dtcm_pool_start < dtcm_end) { |
402 | ret = gen_pool_add(pool: tcm_pool, addr: dtcm_pool_start, |
403 | size: dtcm_end - dtcm_pool_start, nid: -1); |
404 | if (ret) { |
405 | pr_err("CPU DTCM: could not add DTCM " \ |
406 | "remainder to pool!\n" ); |
407 | return ret; |
408 | } |
409 | pr_debug("CPU DTCM: Added %08x bytes @ %08x to " \ |
410 | "the TCM memory pool\n" , |
411 | dtcm_end - dtcm_pool_start, |
412 | dtcm_pool_start); |
413 | } |
414 | } |
415 | |
416 | /* Add the rest of ITCM to the TCM pool */ |
417 | if (itcm_present) { |
418 | if (itcm_pool_start < itcm_end) { |
419 | ret = gen_pool_add(pool: tcm_pool, addr: itcm_pool_start, |
420 | size: itcm_end - itcm_pool_start, nid: -1); |
421 | if (ret) { |
422 | pr_err("CPU ITCM: could not add ITCM " \ |
423 | "remainder to pool!\n" ); |
424 | return ret; |
425 | } |
426 | pr_debug("CPU ITCM: Added %08x bytes @ %08x to " \ |
427 | "the TCM memory pool\n" , |
428 | itcm_end - itcm_pool_start, |
429 | itcm_pool_start); |
430 | } |
431 | } |
432 | return 0; |
433 | } |
434 | |
435 | core_initcall(setup_tcm_pool); |
436 | |