1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6#include <asm/asm-offsets.h>
7#include <asm/asm.h>
8#include <linux/init.h>
9#include <linux/linkage.h>
10#include <asm/thread_info.h>
11#include <asm/page.h>
12#include <asm/pgtable.h>
13#include <asm/csr.h>
14#include <asm/hwcap.h>
15#include <asm/image.h>
16#include <asm/scs.h>
17#include <asm/xip_fixup.h>
18#include "efi-header.S"
19
20__HEAD
21SYM_CODE_START(_start)
22 /*
23 * Image header expected by Linux boot-loaders. The image header data
24 * structure is described in asm/image.h.
25 * Do not modify it without modifying the structure and all bootloaders
26 * that expects this header format!!
27 */
28#ifdef CONFIG_EFI
29 /*
30 * This instruction decodes to "MZ" ASCII required by UEFI.
31 */
32 c.li s4,-13
33 j _start_kernel
34#else
35 /* jump to start kernel */
36 j _start_kernel
37 /* reserved */
38 .word 0
39#endif
40 .balign 8
41#ifdef CONFIG_RISCV_M_MODE
42 /* Image load offset (0MB) from start of RAM for M-mode */
43 .dword 0
44#else
45#if __riscv_xlen == 64
46 /* Image load offset(2MB) from start of RAM */
47 .dword 0x200000
48#else
49 /* Image load offset(4MB) from start of RAM */
50 .dword 0x400000
51#endif
52#endif
53 /* Effective size of kernel image */
54 .dword _end - _start
55 .dword __HEAD_FLAGS
56 .word RISCV_HEADER_VERSION
57 .word 0
58 .dword 0
59 .ascii RISCV_IMAGE_MAGIC
60 .balign 4
61 .ascii RISCV_IMAGE_MAGIC2
62#ifdef CONFIG_EFI
63 .word pe_head_start - _start
64pe_head_start:
65
66 __EFI_PE_HEADER
67#else
68 .word 0
69#endif
70
71.align 2
72#ifdef CONFIG_MMU
73 .global relocate_enable_mmu
74relocate_enable_mmu:
75 /* Relocate return address */
76 la a1, kernel_map
77 XIP_FIXUP_OFFSET a1
78 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1)
79 la a2, _start
80 sub a1, a1, a2
81 add ra, ra, a1
82
83 /* Point stvec to virtual address of intruction after satp write */
84 la a2, 1f
85 add a2, a2, a1
86 csrw CSR_TVEC, a2
87
88 /* Compute satp for kernel page tables, but don't load it yet */
89 srl a2, a0, PAGE_SHIFT
90 la a1, satp_mode
91 XIP_FIXUP_OFFSET a1
92 REG_L a1, 0(a1)
93 or a2, a2, a1
94
95 /*
96 * Load trampoline page directory, which will cause us to trap to
97 * stvec if VA != PA, or simply fall through if VA == PA. We need a
98 * full fence here because setup_vm() just wrote these PTEs and we need
99 * to ensure the new translations are in use.
100 */
101 la a0, trampoline_pg_dir
102 XIP_FIXUP_OFFSET a0
103 srl a0, a0, PAGE_SHIFT
104 or a0, a0, a1
105 sfence.vma
106 csrw CSR_SATP, a0
107.align 2
1081:
109 /* Set trap vector to spin forever to help debug */
110 la a0, .Lsecondary_park
111 csrw CSR_TVEC, a0
112
113 /* Reload the global pointer */
114 load_global_pointer
115
116 /*
117 * Switch to kernel page tables. A full fence is necessary in order to
118 * avoid using the trampoline translations, which are only correct for
119 * the first superpage. Fetching the fence is guaranteed to work
120 * because that first superpage is translated the same way.
121 */
122 csrw CSR_SATP, a2
123 sfence.vma
124
125 ret
126#endif /* CONFIG_MMU */
127#ifdef CONFIG_SMP
128 .global secondary_start_sbi
129secondary_start_sbi:
130 /* Mask all interrupts */
131 csrw CSR_IE, zero
132 csrw CSR_IP, zero
133
134#ifndef CONFIG_RISCV_M_MODE
135 /* Enable time CSR */
136 li t0, 0x2
137 csrw CSR_SCOUNTEREN, t0
138#endif
139
140 /* Load the global pointer */
141 load_global_pointer
142
143 /*
144 * Disable FPU & VECTOR to detect illegal usage of
145 * floating point or vector in kernel space
146 */
147 li t0, SR_FS_VS
148 csrc CSR_STATUS, t0
149
150 /* Set trap vector to spin forever to help debug */
151 la a3, .Lsecondary_park
152 csrw CSR_TVEC, a3
153
154 /* a0 contains the hartid & a1 contains boot data */
155 li a2, SBI_HART_BOOT_TASK_PTR_OFFSET
156 XIP_FIXUP_OFFSET a2
157 add a2, a2, a1
158 REG_L tp, (a2)
159 li a3, SBI_HART_BOOT_STACK_PTR_OFFSET
160 XIP_FIXUP_OFFSET a3
161 add a3, a3, a1
162 REG_L sp, (a3)
163
164.Lsecondary_start_common:
165
166#ifdef CONFIG_MMU
167 /* Enable virtual memory and relocate to virtual address */
168 la a0, swapper_pg_dir
169 XIP_FIXUP_OFFSET a0
170 call relocate_enable_mmu
171#endif
172 call .Lsetup_trap_vector
173 scs_load_current
174 call smp_callin
175#endif /* CONFIG_SMP */
176
177.align 2
178.Lsecondary_park:
179 /*
180 * Park this hart if we:
181 * - have too many harts on CONFIG_RISCV_BOOT_SPINWAIT
182 * - receive an early trap, before setup_trap_vector finished
183 * - fail in smp_callin(), as a successful one wouldn't return
184 */
185 wfi
186 j .Lsecondary_park
187
188.align 2
189.Lsetup_trap_vector:
190 /* Set trap vector to exception handler */
191 la a0, handle_exception
192 csrw CSR_TVEC, a0
193
194 /*
195 * Set sup0 scratch register to 0, indicating to exception vector that
196 * we are presently executing in kernel.
197 */
198 csrw CSR_SCRATCH, zero
199 ret
200
201SYM_CODE_END(_start)
202
203SYM_CODE_START(_start_kernel)
204 /* Mask all interrupts */
205 csrw CSR_IE, zero
206 csrw CSR_IP, zero
207
208#ifdef CONFIG_RISCV_M_MODE
209 /* flush the instruction cache */
210 fence.i
211
212 /* Reset all registers except ra, a0, a1 */
213 call reset_regs
214
215 /*
216 * Setup a PMP to permit access to all of memory. Some machines may
217 * not implement PMPs, so we set up a quick trap handler to just skip
218 * touching the PMPs on any trap.
219 */
220 la a0, .Lpmp_done
221 csrw CSR_TVEC, a0
222
223 li a0, -1
224 csrw CSR_PMPADDR0, a0
225 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X)
226 csrw CSR_PMPCFG0, a0
227.align 2
228.Lpmp_done:
229
230 /*
231 * The hartid in a0 is expected later on, and we have no firmware
232 * to hand it to us.
233 */
234 csrr a0, CSR_MHARTID
235#else
236 /* Enable time CSR */
237 li t0, 0x2
238 csrw CSR_SCOUNTEREN, t0
239#endif /* CONFIG_RISCV_M_MODE */
240
241 /* Load the global pointer */
242 load_global_pointer
243
244 /*
245 * Disable FPU & VECTOR to detect illegal usage of
246 * floating point or vector in kernel space
247 */
248 li t0, SR_FS_VS
249 csrc CSR_STATUS, t0
250
251#ifdef CONFIG_RISCV_BOOT_SPINWAIT
252 li t0, CONFIG_NR_CPUS
253 blt a0, t0, .Lgood_cores
254 tail .Lsecondary_park
255.Lgood_cores:
256
257 /* The lottery system is only required for spinwait booting method */
258#ifndef CONFIG_XIP_KERNEL
259 /* Pick one hart to run the main boot sequence */
260 la a3, hart_lottery
261 li a2, 1
262 amoadd.w a3, a2, (a3)
263 bnez a3, .Lsecondary_start
264
265#else
266 /* hart_lottery in flash contains a magic number */
267 la a3, hart_lottery
268 mv a2, a3
269 XIP_FIXUP_OFFSET a2
270 XIP_FIXUP_FLASH_OFFSET a3
271 lw t1, (a3)
272 amoswap.w t0, t1, (a2)
273 /* first time here if hart_lottery in RAM is not set */
274 beq t0, t1, .Lsecondary_start
275
276#endif /* CONFIG_XIP */
277#endif /* CONFIG_RISCV_BOOT_SPINWAIT */
278
279#ifdef CONFIG_XIP_KERNEL
280 la sp, _end + THREAD_SIZE
281 XIP_FIXUP_OFFSET sp
282 mv s0, a0
283 mv s1, a1
284 call __copy_data
285
286 /* Restore a0 & a1 copy */
287 mv a0, s0
288 mv a1, s1
289#endif
290
291#ifndef CONFIG_XIP_KERNEL
292 /* Clear BSS for flat non-ELF images */
293 la a3, __bss_start
294 la a4, __bss_stop
295 ble a4, a3, .Lclear_bss_done
296.Lclear_bss:
297 REG_S zero, (a3)
298 add a3, a3, RISCV_SZPTR
299 blt a3, a4, .Lclear_bss
300.Lclear_bss_done:
301#endif
302 la a2, boot_cpu_hartid
303 XIP_FIXUP_OFFSET a2
304 REG_S a0, (a2)
305
306 /* Initialize page tables and relocate to virtual addresses */
307 la tp, init_task
308 la sp, init_thread_union + THREAD_SIZE
309 XIP_FIXUP_OFFSET sp
310 addi sp, sp, -PT_SIZE_ON_STACK
311 scs_load_init_stack
312#ifdef CONFIG_BUILTIN_DTB
313 la a0, __dtb_start
314 XIP_FIXUP_OFFSET a0
315#else
316 mv a0, a1
317#endif /* CONFIG_BUILTIN_DTB */
318 /* Set trap vector to spin forever to help debug */
319 la a3, .Lsecondary_park
320 csrw CSR_TVEC, a3
321 call setup_vm
322#ifdef CONFIG_MMU
323 la a0, early_pg_dir
324 XIP_FIXUP_OFFSET a0
325 call relocate_enable_mmu
326#endif /* CONFIG_MMU */
327
328 call .Lsetup_trap_vector
329 /* Restore C environment */
330 la tp, init_task
331 la sp, init_thread_union + THREAD_SIZE
332 addi sp, sp, -PT_SIZE_ON_STACK
333 scs_load_current
334
335#ifdef CONFIG_KASAN
336 call kasan_early_init
337#endif
338 /* Start the kernel */
339 call soc_early_init
340 tail start_kernel
341
342#ifdef CONFIG_RISCV_BOOT_SPINWAIT
343.Lsecondary_start:
344 /* Set trap vector to spin forever to help debug */
345 la a3, .Lsecondary_park
346 csrw CSR_TVEC, a3
347
348 slli a3, a0, LGREG
349 la a1, __cpu_spinwait_stack_pointer
350 XIP_FIXUP_OFFSET a1
351 la a2, __cpu_spinwait_task_pointer
352 XIP_FIXUP_OFFSET a2
353 add a1, a3, a1
354 add a2, a3, a2
355
356 /*
357 * This hart didn't win the lottery, so we wait for the winning hart to
358 * get far enough along the boot process that it should continue.
359 */
360.Lwait_for_cpu_up:
361 /* FIXME: We should WFI to save some energy here. */
362 REG_L sp, (a1)
363 REG_L tp, (a2)
364 beqz sp, .Lwait_for_cpu_up
365 beqz tp, .Lwait_for_cpu_up
366 fence
367
368 tail .Lsecondary_start_common
369#endif /* CONFIG_RISCV_BOOT_SPINWAIT */
370
371SYM_CODE_END(_start_kernel)
372
373#ifdef CONFIG_RISCV_M_MODE
374SYM_CODE_START_LOCAL(reset_regs)
375 li sp, 0
376 li gp, 0
377 li tp, 0
378 li t0, 0
379 li t1, 0
380 li t2, 0
381 li s0, 0
382 li s1, 0
383 li a2, 0
384 li a3, 0
385 li a4, 0
386 li a5, 0
387 li a6, 0
388 li a7, 0
389 li s2, 0
390 li s3, 0
391 li s4, 0
392 li s5, 0
393 li s6, 0
394 li s7, 0
395 li s8, 0
396 li s9, 0
397 li s10, 0
398 li s11, 0
399 li t3, 0
400 li t4, 0
401 li t5, 0
402 li t6, 0
403 csrw CSR_SCRATCH, 0
404
405#ifdef CONFIG_FPU
406 csrr t0, CSR_MISA
407 andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)
408 beqz t0, .Lreset_regs_done_fpu
409
410 li t1, SR_FS
411 csrs CSR_STATUS, t1
412 fmv.s.x f0, zero
413 fmv.s.x f1, zero
414 fmv.s.x f2, zero
415 fmv.s.x f3, zero
416 fmv.s.x f4, zero
417 fmv.s.x f5, zero
418 fmv.s.x f6, zero
419 fmv.s.x f7, zero
420 fmv.s.x f8, zero
421 fmv.s.x f9, zero
422 fmv.s.x f10, zero
423 fmv.s.x f11, zero
424 fmv.s.x f12, zero
425 fmv.s.x f13, zero
426 fmv.s.x f14, zero
427 fmv.s.x f15, zero
428 fmv.s.x f16, zero
429 fmv.s.x f17, zero
430 fmv.s.x f18, zero
431 fmv.s.x f19, zero
432 fmv.s.x f20, zero
433 fmv.s.x f21, zero
434 fmv.s.x f22, zero
435 fmv.s.x f23, zero
436 fmv.s.x f24, zero
437 fmv.s.x f25, zero
438 fmv.s.x f26, zero
439 fmv.s.x f27, zero
440 fmv.s.x f28, zero
441 fmv.s.x f29, zero
442 fmv.s.x f30, zero
443 fmv.s.x f31, zero
444 csrw fcsr, 0
445 /* note that the caller must clear SR_FS */
446.Lreset_regs_done_fpu:
447#endif /* CONFIG_FPU */
448
449#ifdef CONFIG_RISCV_ISA_V
450 csrr t0, CSR_MISA
451 li t1, COMPAT_HWCAP_ISA_V
452 and t0, t0, t1
453 beqz t0, .Lreset_regs_done_vector
454
455 /*
456 * Clear vector registers and reset vcsr
457 * VLMAX has a defined value, VLEN is a constant,
458 * and this form of vsetvli is defined to set vl to VLMAX.
459 */
460 li t1, SR_VS
461 csrs CSR_STATUS, t1
462 csrs CSR_VCSR, x0
463 vsetvli t1, x0, e8, m8, ta, ma
464 vmv.v.i v0, 0
465 vmv.v.i v8, 0
466 vmv.v.i v16, 0
467 vmv.v.i v24, 0
468 /* note that the caller must clear SR_VS */
469.Lreset_regs_done_vector:
470#endif /* CONFIG_RISCV_ISA_V */
471 ret
472SYM_CODE_END(reset_regs)
473#endif /* CONFIG_RISCV_M_MODE */
474

source code of linux/arch/riscv/kernel/head.S