1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * alternative runtime patching
4 * inspired by the ARM64 and x86 version
5 *
6 * Copyright (C) 2021 Sifive.
7 */
8
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/cpu.h>
12#include <linux/uaccess.h>
13#include <asm/alternative.h>
14#include <asm/module.h>
15#include <asm/sections.h>
16#include <asm/vdso.h>
17#include <asm/vendorid_list.h>
18#include <asm/sbi.h>
19#include <asm/csr.h>
20#include <asm/insn.h>
21#include <asm/patch.h>
22
23struct cpu_manufacturer_info_t {
24 unsigned long vendor_id;
25 unsigned long arch_id;
26 unsigned long imp_id;
27 void (*patch_func)(struct alt_entry *begin, struct alt_entry *end,
28 unsigned long archid, unsigned long impid,
29 unsigned int stage);
30};
31
32static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info)
33{
34#ifdef CONFIG_RISCV_M_MODE
35 cpu_mfr_info->vendor_id = csr_read(CSR_MVENDORID);
36 cpu_mfr_info->arch_id = csr_read(CSR_MARCHID);
37 cpu_mfr_info->imp_id = csr_read(CSR_MIMPID);
38#else
39 cpu_mfr_info->vendor_id = sbi_get_mvendorid();
40 cpu_mfr_info->arch_id = sbi_get_marchid();
41 cpu_mfr_info->imp_id = sbi_get_mimpid();
42#endif
43
44 switch (cpu_mfr_info->vendor_id) {
45#ifdef CONFIG_ERRATA_ANDES
46 case ANDES_VENDOR_ID:
47 cpu_mfr_info->patch_func = andes_errata_patch_func;
48 break;
49#endif
50#ifdef CONFIG_ERRATA_SIFIVE
51 case SIFIVE_VENDOR_ID:
52 cpu_mfr_info->patch_func = sifive_errata_patch_func;
53 break;
54#endif
55#ifdef CONFIG_ERRATA_THEAD
56 case THEAD_VENDOR_ID:
57 cpu_mfr_info->patch_func = thead_errata_patch_func;
58 break;
59#endif
60 default:
61 cpu_mfr_info->patch_func = NULL;
62 }
63}
64
65static u32 riscv_instruction_at(void *p)
66{
67 u16 *parcel = p;
68
69 return (u32)parcel[0] | (u32)parcel[1] << 16;
70}
71
72static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,
73 u32 jalr_insn, int patch_offset)
74{
75 u32 call[2] = { auipc_insn, jalr_insn };
76 s32 imm;
77
78 /* get and adjust new target address */
79 imm = riscv_insn_extract_utype_itype_imm(auipc_insn, jalr_insn);
80 imm -= patch_offset;
81
82 /* update instructions */
83 riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm);
84
85 /* patch the call place again */
86 patch_text_nosync(ptr, call, sizeof(u32) * 2);
87}
88
89static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset)
90{
91 s32 imm;
92
93 /* get and adjust new target address */
94 imm = riscv_insn_extract_jtype_imm(jal_insn);
95 imm -= patch_offset;
96
97 /* update instruction */
98 riscv_insn_insert_jtype_imm(&jal_insn, imm);
99
100 /* patch the call place again */
101 patch_text_nosync(ptr, &jal_insn, sizeof(u32));
102}
103
104void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
105 int patch_offset)
106{
107 int num_insn = len / sizeof(u32);
108 int i;
109
110 for (i = 0; i < num_insn; i++) {
111 u32 insn = riscv_instruction_at(p: alt_ptr + i * sizeof(u32));
112
113 /*
114 * May be the start of an auipc + jalr pair
115 * Needs to check that at least one more instruction
116 * is in the list.
117 */
118 if (riscv_insn_is_auipc(insn) && i < num_insn - 1) {
119 u32 insn2 = riscv_instruction_at(p: alt_ptr + (i + 1) * sizeof(u32));
120
121 if (!riscv_insn_is_jalr(insn2))
122 continue;
123
124 /* if instruction pair is a call, it will use the ra register */
125 if (RV_EXTRACT_RD_REG(insn) != 1)
126 continue;
127
128 riscv_alternative_fix_auipc_jalr(ptr: alt_ptr + i * sizeof(u32),
129 auipc_insn: insn, jalr_insn: insn2, patch_offset);
130 i++;
131 }
132
133 if (riscv_insn_is_jal(insn)) {
134 s32 imm = riscv_insn_extract_jtype_imm(insn);
135
136 /* Don't modify jumps inside the alternative block */
137 if ((alt_ptr + i * sizeof(u32) + imm) >= alt_ptr &&
138 (alt_ptr + i * sizeof(u32) + imm) < (alt_ptr + len))
139 continue;
140
141 riscv_alternative_fix_jal(ptr: alt_ptr + i * sizeof(u32),
142 jal_insn: insn, patch_offset);
143 }
144 }
145}
146
147/*
148 * This is called very early in the boot process (directly after we run
149 * a feature detect on the boot CPU). No need to worry about other CPUs
150 * here.
151 */
152static void __init_or_module _apply_alternatives(struct alt_entry *begin,
153 struct alt_entry *end,
154 unsigned int stage)
155{
156 struct cpu_manufacturer_info_t cpu_mfr_info;
157
158 riscv_fill_cpu_mfr_info(cpu_mfr_info: &cpu_mfr_info);
159
160 riscv_cpufeature_patch_func(begin, end, stage);
161
162 if (!cpu_mfr_info.patch_func)
163 return;
164
165 cpu_mfr_info.patch_func(begin, end,
166 cpu_mfr_info.arch_id,
167 cpu_mfr_info.imp_id,
168 stage);
169}
170
171#ifdef CONFIG_MMU
172static void __init apply_vdso_alternatives(void)
173{
174 const Elf_Ehdr *hdr;
175 const Elf_Shdr *shdr;
176 const Elf_Shdr *alt;
177 struct alt_entry *begin, *end;
178
179 hdr = (Elf_Ehdr *)vdso_start;
180 shdr = (void *)hdr + hdr->e_shoff;
181 alt = find_section(hdr, shdr, ".alternative");
182 if (!alt)
183 return;
184
185 begin = (void *)hdr + alt->sh_offset,
186 end = (void *)hdr + alt->sh_offset + alt->sh_size,
187
188 _apply_alternatives(begin: (struct alt_entry *)begin,
189 end: (struct alt_entry *)end,
190 stage: RISCV_ALTERNATIVES_BOOT);
191}
192#else
193static void __init apply_vdso_alternatives(void) { }
194#endif
195
196void __init apply_boot_alternatives(void)
197{
198 /* If called on non-boot cpu things could go wrong */
199 WARN_ON(smp_processor_id() != 0);
200
201 _apply_alternatives((struct alt_entry *)__alt_start,
202 (struct alt_entry *)__alt_end,
203 RISCV_ALTERNATIVES_BOOT);
204
205 apply_vdso_alternatives();
206}
207
208/*
209 * apply_early_boot_alternatives() is called from setup_vm() with MMU-off.
210 *
211 * Following requirements should be honoured for it to work correctly:
212 * 1) It should use PC-relative addressing for accessing kernel symbols.
213 * To achieve this we always use GCC cmodel=medany.
214 * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
215 * so disable compiler instrumentation when FTRACE is enabled.
216 *
217 * Currently, the above requirements are honoured by using custom CFLAGS
218 * for alternative.o in kernel/Makefile.
219 */
220void __init apply_early_boot_alternatives(void)
221{
222#ifdef CONFIG_RISCV_ALTERNATIVE_EARLY
223 _apply_alternatives((struct alt_entry *)__alt_start,
224 (struct alt_entry *)__alt_end,
225 RISCV_ALTERNATIVES_EARLY_BOOT);
226#endif
227}
228
229#ifdef CONFIG_MODULES
230void apply_module_alternatives(void *start, size_t length)
231{
232 _apply_alternatives(begin: (struct alt_entry *)start,
233 end: (struct alt_entry *)(start + length),
234 stage: RISCV_ALTERNATIVES_MODULE);
235}
236#endif
237

source code of linux/arch/riscv/kernel/alternative.c