| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 2 | /* |
|---|
| 3 | * Copyright (C) 2012 Regents of the University of California |
|---|
| 4 | */ |
|---|
| 5 | |
|---|
| 6 | #include <asm/asm-offsets.h> |
|---|
| 7 | #include <asm/asm.h> |
|---|
| 8 | #include <linux/init.h> |
|---|
| 9 | #include <linux/linkage.h> |
|---|
| 10 | #include <asm/thread_info.h> |
|---|
| 11 | #include <asm/page.h> |
|---|
| 12 | #include <asm/pgtable.h> |
|---|
| 13 | #include <asm/csr.h> |
|---|
| 14 | #include <asm/hwcap.h> |
|---|
| 15 | #include <asm/image.h> |
|---|
| 16 | #include <asm/scs.h> |
|---|
| 17 | #include <asm/xip_fixup.h> |
|---|
| 18 | #include "efi-header.S" |
|---|
| 19 | |
|---|
| 20 | __HEAD |
|---|
| 21 | SYM_CODE_START(_start) |
|---|
| 22 | /* |
|---|
| 23 | * Image header expected by Linux boot-loaders. The image header data |
|---|
| 24 | * structure is described in asm/image.h. |
|---|
| 25 | * Do not modify it without modifying the structure and all bootloaders |
|---|
| 26 | * that expects this header format!! |
|---|
| 27 | */ |
|---|
| 28 | #ifdef CONFIG_EFI |
|---|
| 29 | /* |
|---|
| 30 | * This instruction decodes to "MZ" ASCII required by UEFI. |
|---|
| 31 | */ |
|---|
| 32 | c.li s4,-13 |
|---|
| 33 | j _start_kernel |
|---|
| 34 | #else |
|---|
| 35 | /* jump to start kernel */ |
|---|
| 36 | j _start_kernel |
|---|
| 37 | /* reserved */ |
|---|
| 38 | .word 0 |
|---|
| 39 | #endif |
|---|
| 40 | .balign 8 |
|---|
| 41 | #ifdef CONFIG_RISCV_M_MODE |
|---|
| 42 | /* Image load offset (0MB) from start of RAM for M-mode */ |
|---|
| 43 | .dword 0 |
|---|
| 44 | #else |
|---|
| 45 | #if __riscv_xlen == 64 |
|---|
| 46 | /* Image load offset(2MB) from start of RAM */ |
|---|
| 47 | .dword 0x200000 |
|---|
| 48 | #else |
|---|
| 49 | /* Image load offset(4MB) from start of RAM */ |
|---|
| 50 | .dword 0x400000 |
|---|
| 51 | #endif |
|---|
| 52 | #endif |
|---|
| 53 | /* Effective size of kernel image */ |
|---|
| 54 | .dword _end - _start |
|---|
| 55 | .dword __HEAD_FLAGS |
|---|
| 56 | .word RISCV_HEADER_VERSION |
|---|
| 57 | .word 0 |
|---|
| 58 | .dword 0 |
|---|
| 59 | .ascii RISCV_IMAGE_MAGIC |
|---|
| 60 | .balign 4 |
|---|
| 61 | .ascii RISCV_IMAGE_MAGIC2 |
|---|
| 62 | #ifdef CONFIG_EFI |
|---|
| 63 | .word pe_head_start - _start |
|---|
| 64 | pe_head_start: |
|---|
| 65 | |
|---|
| 66 | __EFI_PE_HEADER |
|---|
| 67 | #else |
|---|
| 68 | .word 0 |
|---|
| 69 | #endif |
|---|
| 70 | |
|---|
| 71 | .align 2 |
|---|
| 72 | #ifdef CONFIG_MMU |
|---|
| 73 | .global relocate_enable_mmu |
|---|
| 74 | relocate_enable_mmu: |
|---|
| 75 | /* Relocate return address */ |
|---|
| 76 | la a1, kernel_map |
|---|
| 77 | XIP_FIXUP_OFFSET a1 |
|---|
| 78 | REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) |
|---|
| 79 | la a2, _start |
|---|
| 80 | sub a1, a1, a2 |
|---|
| 81 | add ra, ra, a1 |
|---|
| 82 | |
|---|
| 83 | /* Point stvec to virtual address of intruction after satp write */ |
|---|
| 84 | la a2, 1f |
|---|
| 85 | add a2, a2, a1 |
|---|
| 86 | csrw CSR_TVEC, a2 |
|---|
| 87 | |
|---|
| 88 | /* Compute satp for kernel page tables, but don't load it yet */ |
|---|
| 89 | srl a2, a0, PAGE_SHIFT |
|---|
| 90 | la a1, satp_mode |
|---|
| 91 | XIP_FIXUP_OFFSET a1 |
|---|
| 92 | REG_L a1, 0(a1) |
|---|
| 93 | or a2, a2, a1 |
|---|
| 94 | |
|---|
| 95 | /* |
|---|
| 96 | * Load trampoline page directory, which will cause us to trap to |
|---|
| 97 | * stvec if VA != PA, or simply fall through if VA == PA. We need a |
|---|
| 98 | * full fence here because setup_vm() just wrote these PTEs and we need |
|---|
| 99 | * to ensure the new translations are in use. |
|---|
| 100 | */ |
|---|
| 101 | la a0, trampoline_pg_dir |
|---|
| 102 | XIP_FIXUP_OFFSET a0 |
|---|
| 103 | srl a0, a0, PAGE_SHIFT |
|---|
| 104 | or a0, a0, a1 |
|---|
| 105 | sfence.vma |
|---|
| 106 | csrw CSR_SATP, a0 |
|---|
| 107 | .align 2 |
|---|
| 108 | 1: |
|---|
| 109 | /* Set trap vector to spin forever to help debug */ |
|---|
| 110 | la a0, .Lsecondary_park |
|---|
| 111 | csrw CSR_TVEC, a0 |
|---|
| 112 | |
|---|
| 113 | /* Reload the global pointer */ |
|---|
| 114 | load_global_pointer |
|---|
| 115 | |
|---|
| 116 | /* |
|---|
| 117 | * Switch to kernel page tables. A full fence is necessary in order to |
|---|
| 118 | * avoid using the trampoline translations, which are only correct for |
|---|
| 119 | * the first superpage. Fetching the fence is guaranteed to work |
|---|
| 120 | * because that first superpage is translated the same way. |
|---|
| 121 | */ |
|---|
| 122 | csrw CSR_SATP, a2 |
|---|
| 123 | sfence.vma |
|---|
| 124 | |
|---|
| 125 | ret |
|---|
| 126 | #endif /* CONFIG_MMU */ |
|---|
| 127 | #ifdef CONFIG_SMP |
|---|
| 128 | .global secondary_start_sbi |
|---|
| 129 | secondary_start_sbi: |
|---|
| 130 | /* Mask all interrupts */ |
|---|
| 131 | csrw CSR_IE, zero |
|---|
| 132 | csrw CSR_IP, zero |
|---|
| 133 | |
|---|
| 134 | /* Load the global pointer */ |
|---|
| 135 | load_global_pointer |
|---|
| 136 | |
|---|
| 137 | /* |
|---|
| 138 | * Disable FPU & VECTOR to detect illegal usage of |
|---|
| 139 | * floating point or vector in kernel space |
|---|
| 140 | */ |
|---|
| 141 | li t0, SR_FS_VS |
|---|
| 142 | csrc CSR_STATUS, t0 |
|---|
| 143 | |
|---|
| 144 | /* Set trap vector to spin forever to help debug */ |
|---|
| 145 | la a3, .Lsecondary_park |
|---|
| 146 | csrw CSR_TVEC, a3 |
|---|
| 147 | |
|---|
| 148 | /* a0 contains the hartid & a1 contains boot data */ |
|---|
| 149 | li a2, SBI_HART_BOOT_TASK_PTR_OFFSET |
|---|
| 150 | XIP_FIXUP_OFFSET a2 |
|---|
| 151 | add a2, a2, a1 |
|---|
| 152 | REG_L tp, (a2) |
|---|
| 153 | li a3, SBI_HART_BOOT_STACK_PTR_OFFSET |
|---|
| 154 | XIP_FIXUP_OFFSET a3 |
|---|
| 155 | add a3, a3, a1 |
|---|
| 156 | REG_L sp, (a3) |
|---|
| 157 | |
|---|
| 158 | .Lsecondary_start_common: |
|---|
| 159 | |
|---|
| 160 | #ifdef CONFIG_MMU |
|---|
| 161 | /* Enable virtual memory and relocate to virtual address */ |
|---|
| 162 | la a0, swapper_pg_dir |
|---|
| 163 | XIP_FIXUP_OFFSET a0 |
|---|
| 164 | call relocate_enable_mmu |
|---|
| 165 | #endif |
|---|
| 166 | call .Lsetup_trap_vector |
|---|
| 167 | scs_load_current |
|---|
| 168 | tail smp_callin |
|---|
| 169 | #endif /* CONFIG_SMP */ |
|---|
| 170 | |
|---|
| 171 | .align 2 |
|---|
| 172 | .Lsetup_trap_vector: |
|---|
| 173 | /* Set trap vector to exception handler */ |
|---|
| 174 | la a0, handle_exception |
|---|
| 175 | csrw CSR_TVEC, a0 |
|---|
| 176 | |
|---|
| 177 | /* |
|---|
| 178 | * Set sup0 scratch register to 0, indicating to exception vector that |
|---|
| 179 | * we are presently executing in kernel. |
|---|
| 180 | */ |
|---|
| 181 | csrw CSR_SCRATCH, zero |
|---|
| 182 | ret |
|---|
| 183 | |
|---|
| 184 | .align 2 |
|---|
| 185 | .Lsecondary_park: |
|---|
| 186 | /* We lack SMP support or have too many harts, so park this hart */ |
|---|
| 187 | wfi |
|---|
| 188 | j .Lsecondary_park |
|---|
| 189 | |
|---|
| 190 | SYM_CODE_END(_start) |
|---|
| 191 | |
|---|
| 192 | SYM_CODE_START(_start_kernel) |
|---|
| 193 | /* Mask all interrupts */ |
|---|
| 194 | csrw CSR_IE, zero |
|---|
| 195 | csrw CSR_IP, zero |
|---|
| 196 | |
|---|
| 197 | #ifdef CONFIG_RISCV_M_MODE |
|---|
| 198 | /* flush the instruction cache */ |
|---|
| 199 | fence.i |
|---|
| 200 | |
|---|
| 201 | /* Reset all registers except ra, a0, a1 */ |
|---|
| 202 | call reset_regs |
|---|
| 203 | |
|---|
| 204 | /* |
|---|
| 205 | * Setup a PMP to permit access to all of memory. Some machines may |
|---|
| 206 | * not implement PMPs, so we set up a quick trap handler to just skip |
|---|
| 207 | * touching the PMPs on any trap. |
|---|
| 208 | */ |
|---|
| 209 | la a0, .Lpmp_done |
|---|
| 210 | csrw CSR_TVEC, a0 |
|---|
| 211 | |
|---|
| 212 | li a0, -1 |
|---|
| 213 | csrw CSR_PMPADDR0, a0 |
|---|
| 214 | li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) |
|---|
| 215 | csrw CSR_PMPCFG0, a0 |
|---|
| 216 | .align 2 |
|---|
| 217 | .Lpmp_done: |
|---|
| 218 | |
|---|
| 219 | /* |
|---|
| 220 | * The hartid in a0 is expected later on, and we have no firmware |
|---|
| 221 | * to hand it to us. |
|---|
| 222 | */ |
|---|
| 223 | csrr a0, CSR_MHARTID |
|---|
| 224 | #endif /* CONFIG_RISCV_M_MODE */ |
|---|
| 225 | |
|---|
| 226 | /* Load the global pointer */ |
|---|
| 227 | load_global_pointer |
|---|
| 228 | |
|---|
| 229 | /* |
|---|
| 230 | * Disable FPU & VECTOR to detect illegal usage of |
|---|
| 231 | * floating point or vector in kernel space |
|---|
| 232 | */ |
|---|
| 233 | li t0, SR_FS_VS |
|---|
| 234 | csrc CSR_STATUS, t0 |
|---|
| 235 | |
|---|
| 236 | #ifdef CONFIG_RISCV_BOOT_SPINWAIT |
|---|
| 237 | li t0, CONFIG_NR_CPUS |
|---|
| 238 | blt a0, t0, .Lgood_cores |
|---|
| 239 | tail .Lsecondary_park |
|---|
| 240 | .Lgood_cores: |
|---|
| 241 | |
|---|
| 242 | /* The lottery system is only required for spinwait booting method */ |
|---|
| 243 | #ifndef CONFIG_XIP_KERNEL |
|---|
| 244 | /* Pick one hart to run the main boot sequence */ |
|---|
| 245 | la a3, hart_lottery |
|---|
| 246 | li a2, 1 |
|---|
| 247 | amoadd.w a3, a2, (a3) |
|---|
| 248 | bnez a3, .Lsecondary_start |
|---|
| 249 | |
|---|
| 250 | #else |
|---|
| 251 | /* hart_lottery in flash contains a magic number */ |
|---|
| 252 | la a3, hart_lottery |
|---|
| 253 | mv a2, a3 |
|---|
| 254 | XIP_FIXUP_OFFSET a2 |
|---|
| 255 | XIP_FIXUP_FLASH_OFFSET a3 |
|---|
| 256 | lw t1, (a3) |
|---|
| 257 | amoswap.w t0, t1, (a2) |
|---|
| 258 | /* first time here if hart_lottery in RAM is not set */ |
|---|
| 259 | beq t0, t1, .Lsecondary_start |
|---|
| 260 | |
|---|
| 261 | #endif /* CONFIG_XIP */ |
|---|
| 262 | #endif /* CONFIG_RISCV_BOOT_SPINWAIT */ |
|---|
| 263 | |
|---|
| 264 | #ifdef CONFIG_XIP_KERNEL |
|---|
| 265 | la sp, _end + THREAD_SIZE |
|---|
| 266 | XIP_FIXUP_OFFSET sp |
|---|
| 267 | mv s0, a0 |
|---|
| 268 | mv s1, a1 |
|---|
| 269 | call __copy_data |
|---|
| 270 | |
|---|
| 271 | /* Restore a0 & a1 copy */ |
|---|
| 272 | mv a0, s0 |
|---|
| 273 | mv a1, s1 |
|---|
| 274 | #endif |
|---|
| 275 | |
|---|
| 276 | #ifndef CONFIG_XIP_KERNEL |
|---|
| 277 | /* Clear BSS for flat non-ELF images */ |
|---|
| 278 | la a3, __bss_start |
|---|
| 279 | la a4, __bss_stop |
|---|
| 280 | ble a4, a3, .Lclear_bss_done |
|---|
| 281 | .Lclear_bss: |
|---|
| 282 | REG_S zero, (a3) |
|---|
| 283 | add a3, a3, RISCV_SZPTR |
|---|
| 284 | blt a3, a4, .Lclear_bss |
|---|
| 285 | .Lclear_bss_done: |
|---|
| 286 | #endif |
|---|
| 287 | la a2, boot_cpu_hartid |
|---|
| 288 | XIP_FIXUP_OFFSET a2 |
|---|
| 289 | REG_S a0, (a2) |
|---|
| 290 | |
|---|
| 291 | /* Initialize page tables and relocate to virtual addresses */ |
|---|
| 292 | la tp, init_task |
|---|
| 293 | la sp, init_thread_union + THREAD_SIZE |
|---|
| 294 | XIP_FIXUP_OFFSET sp |
|---|
| 295 | addi sp, sp, -PT_SIZE_ON_STACK |
|---|
| 296 | scs_load_init_stack |
|---|
| 297 | #ifdef CONFIG_BUILTIN_DTB |
|---|
| 298 | la a0, __dtb_start |
|---|
| 299 | XIP_FIXUP_OFFSET a0 |
|---|
| 300 | #else |
|---|
| 301 | mv a0, a1 |
|---|
| 302 | #endif /* CONFIG_BUILTIN_DTB */ |
|---|
| 303 | call setup_vm |
|---|
| 304 | #ifdef CONFIG_MMU |
|---|
| 305 | la a0, early_pg_dir |
|---|
| 306 | XIP_FIXUP_OFFSET a0 |
|---|
| 307 | call relocate_enable_mmu |
|---|
| 308 | #endif /* CONFIG_MMU */ |
|---|
| 309 | |
|---|
| 310 | call .Lsetup_trap_vector |
|---|
| 311 | /* Restore C environment */ |
|---|
| 312 | la tp, init_task |
|---|
| 313 | la sp, init_thread_union + THREAD_SIZE |
|---|
| 314 | addi sp, sp, -PT_SIZE_ON_STACK |
|---|
| 315 | scs_load_current |
|---|
| 316 | |
|---|
| 317 | #ifdef CONFIG_KASAN |
|---|
| 318 | call kasan_early_init |
|---|
| 319 | #endif |
|---|
| 320 | /* Start the kernel */ |
|---|
| 321 | call soc_early_init |
|---|
| 322 | tail start_kernel |
|---|
| 323 | |
|---|
| 324 | #ifdef CONFIG_RISCV_BOOT_SPINWAIT |
|---|
| 325 | .Lsecondary_start: |
|---|
| 326 | /* Set trap vector to spin forever to help debug */ |
|---|
| 327 | la a3, .Lsecondary_park |
|---|
| 328 | csrw CSR_TVEC, a3 |
|---|
| 329 | |
|---|
| 330 | slli a3, a0, LGREG |
|---|
| 331 | la a1, __cpu_spinwait_stack_pointer |
|---|
| 332 | XIP_FIXUP_OFFSET a1 |
|---|
| 333 | la a2, __cpu_spinwait_task_pointer |
|---|
| 334 | XIP_FIXUP_OFFSET a2 |
|---|
| 335 | add a1, a3, a1 |
|---|
| 336 | add a2, a3, a2 |
|---|
| 337 | |
|---|
| 338 | /* |
|---|
| 339 | * This hart didn't win the lottery, so we wait for the winning hart to |
|---|
| 340 | * get far enough along the boot process that it should continue. |
|---|
| 341 | */ |
|---|
| 342 | .Lwait_for_cpu_up: |
|---|
| 343 | /* FIXME: We should WFI to save some energy here. */ |
|---|
| 344 | REG_L sp, (a1) |
|---|
| 345 | REG_L tp, (a2) |
|---|
| 346 | beqz sp, .Lwait_for_cpu_up |
|---|
| 347 | beqz tp, .Lwait_for_cpu_up |
|---|
| 348 | fence |
|---|
| 349 | |
|---|
| 350 | tail .Lsecondary_start_common |
|---|
| 351 | #endif /* CONFIG_RISCV_BOOT_SPINWAIT */ |
|---|
| 352 | |
|---|
| 353 | SYM_CODE_END(_start_kernel) |
|---|
| 354 | |
|---|
| 355 | #ifdef CONFIG_RISCV_M_MODE |
|---|
| 356 | SYM_CODE_START_LOCAL(reset_regs) |
|---|
| 357 | li sp, 0 |
|---|
| 358 | li gp, 0 |
|---|
| 359 | li tp, 0 |
|---|
| 360 | li t0, 0 |
|---|
| 361 | li t1, 0 |
|---|
| 362 | li t2, 0 |
|---|
| 363 | li s0, 0 |
|---|
| 364 | li s1, 0 |
|---|
| 365 | li a2, 0 |
|---|
| 366 | li a3, 0 |
|---|
| 367 | li a4, 0 |
|---|
| 368 | li a5, 0 |
|---|
| 369 | li a6, 0 |
|---|
| 370 | li a7, 0 |
|---|
| 371 | li s2, 0 |
|---|
| 372 | li s3, 0 |
|---|
| 373 | li s4, 0 |
|---|
| 374 | li s5, 0 |
|---|
| 375 | li s6, 0 |
|---|
| 376 | li s7, 0 |
|---|
| 377 | li s8, 0 |
|---|
| 378 | li s9, 0 |
|---|
| 379 | li s10, 0 |
|---|
| 380 | li s11, 0 |
|---|
| 381 | li t3, 0 |
|---|
| 382 | li t4, 0 |
|---|
| 383 | li t5, 0 |
|---|
| 384 | li t6, 0 |
|---|
| 385 | csrw CSR_SCRATCH, 0 |
|---|
| 386 | |
|---|
| 387 | #ifdef CONFIG_FPU |
|---|
| 388 | csrr t0, CSR_MISA |
|---|
| 389 | andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) |
|---|
| 390 | beqz t0, .Lreset_regs_done_fpu |
|---|
| 391 | |
|---|
| 392 | li t1, SR_FS |
|---|
| 393 | csrs CSR_STATUS, t1 |
|---|
| 394 | fmv.s.x f0, zero |
|---|
| 395 | fmv.s.x f1, zero |
|---|
| 396 | fmv.s.x f2, zero |
|---|
| 397 | fmv.s.x f3, zero |
|---|
| 398 | fmv.s.x f4, zero |
|---|
| 399 | fmv.s.x f5, zero |
|---|
| 400 | fmv.s.x f6, zero |
|---|
| 401 | fmv.s.x f7, zero |
|---|
| 402 | fmv.s.x f8, zero |
|---|
| 403 | fmv.s.x f9, zero |
|---|
| 404 | fmv.s.x f10, zero |
|---|
| 405 | fmv.s.x f11, zero |
|---|
| 406 | fmv.s.x f12, zero |
|---|
| 407 | fmv.s.x f13, zero |
|---|
| 408 | fmv.s.x f14, zero |
|---|
| 409 | fmv.s.x f15, zero |
|---|
| 410 | fmv.s.x f16, zero |
|---|
| 411 | fmv.s.x f17, zero |
|---|
| 412 | fmv.s.x f18, zero |
|---|
| 413 | fmv.s.x f19, zero |
|---|
| 414 | fmv.s.x f20, zero |
|---|
| 415 | fmv.s.x f21, zero |
|---|
| 416 | fmv.s.x f22, zero |
|---|
| 417 | fmv.s.x f23, zero |
|---|
| 418 | fmv.s.x f24, zero |
|---|
| 419 | fmv.s.x f25, zero |
|---|
| 420 | fmv.s.x f26, zero |
|---|
| 421 | fmv.s.x f27, zero |
|---|
| 422 | fmv.s.x f28, zero |
|---|
| 423 | fmv.s.x f29, zero |
|---|
| 424 | fmv.s.x f30, zero |
|---|
| 425 | fmv.s.x f31, zero |
|---|
| 426 | csrw fcsr, 0 |
|---|
| 427 | /* note that the caller must clear SR_FS */ |
|---|
| 428 | .Lreset_regs_done_fpu: |
|---|
| 429 | #endif /* CONFIG_FPU */ |
|---|
| 430 | |
|---|
| 431 | #ifdef CONFIG_RISCV_ISA_V |
|---|
| 432 | csrr t0, CSR_MISA |
|---|
| 433 | li t1, COMPAT_HWCAP_ISA_V |
|---|
| 434 | and t0, t0, t1 |
|---|
| 435 | beqz t0, .Lreset_regs_done_vector |
|---|
| 436 | |
|---|
| 437 | /* |
|---|
| 438 | * Clear vector registers and reset vcsr |
|---|
| 439 | * VLMAX has a defined value, VLEN is a constant, |
|---|
| 440 | * and this form of vsetvli is defined to set vl to VLMAX. |
|---|
| 441 | */ |
|---|
| 442 | li t1, SR_VS |
|---|
| 443 | csrs CSR_STATUS, t1 |
|---|
| 444 | csrs CSR_VCSR, x0 |
|---|
| 445 | vsetvli t1, x0, e8, m8, ta, ma |
|---|
| 446 | vmv.v.i v0, 0 |
|---|
| 447 | vmv.v.i v8, 0 |
|---|
| 448 | vmv.v.i v16, 0 |
|---|
| 449 | vmv.v.i v24, 0 |
|---|
| 450 | /* note that the caller must clear SR_VS */ |
|---|
| 451 | .Lreset_regs_done_vector: |
|---|
| 452 | #endif /* CONFIG_RISCV_ISA_V */ |
|---|
| 453 | ret |
|---|
| 454 | SYM_CODE_END(reset_regs) |
|---|
| 455 | #endif /* CONFIG_RISCV_M_MODE */ |
|---|
| 456 | |
|---|
