1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* Paravirtualization interfaces |
3 | Copyright (C) 2006 Rusty Russell IBM Corporation |
4 | |
5 | |
6 | 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc |
7 | */ |
8 | |
9 | #include <linux/errno.h> |
10 | #include <linux/init.h> |
11 | #include <linux/export.h> |
12 | #include <linux/efi.h> |
13 | #include <linux/bcd.h> |
14 | #include <linux/highmem.h> |
15 | #include <linux/kprobes.h> |
16 | #include <linux/pgtable.h> |
17 | #include <linux/static_call.h> |
18 | |
19 | #include <asm/bug.h> |
20 | #include <asm/paravirt.h> |
21 | #include <asm/debugreg.h> |
22 | #include <asm/desc.h> |
23 | #include <asm/setup.h> |
24 | #include <asm/time.h> |
25 | #include <asm/pgalloc.h> |
26 | #include <asm/irq.h> |
27 | #include <asm/delay.h> |
28 | #include <asm/fixmap.h> |
29 | #include <asm/apic.h> |
30 | #include <asm/tlbflush.h> |
31 | #include <asm/timer.h> |
32 | #include <asm/special_insns.h> |
33 | #include <asm/tlb.h> |
34 | #include <asm/io_bitmap.h> |
35 | #include <asm/gsseg.h> |
36 | |
37 | /* |
38 | * nop stub, which must not clobber anything *including the stack* to |
39 | * avoid confusing the entry prologues. |
40 | */ |
41 | DEFINE_PARAVIRT_ASM(_paravirt_nop, "" , .entry.text); |
42 | |
43 | /* stub always returning 0. */ |
44 | DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax" , .entry.text); |
45 | |
46 | void __init default_banner(void) |
47 | { |
48 | printk(KERN_INFO "Booting paravirtualized kernel on %s\n" , |
49 | pv_info.name); |
50 | } |
51 | |
52 | /* Undefined instruction for dealing with missing ops pointers. */ |
53 | noinstr void paravirt_BUG(void) |
54 | { |
55 | BUG(); |
56 | } |
57 | |
58 | static unsigned paravirt_patch_call(void *insn_buff, const void *target, |
59 | unsigned long addr, unsigned len) |
60 | { |
61 | __text_gen_insn(buf: insn_buff, CALL_INSN_OPCODE, |
62 | addr: (void *)addr, dest: target, CALL_INSN_SIZE); |
63 | return CALL_INSN_SIZE; |
64 | } |
65 | |
66 | #ifdef CONFIG_PARAVIRT_XXL |
67 | DEFINE_PARAVIRT_ASM(_paravirt_ident_64, "mov %rdi, %rax" , .text); |
68 | DEFINE_PARAVIRT_ASM(pv_native_save_fl, "pushf; pop %rax" , .noinstr.text); |
69 | DEFINE_PARAVIRT_ASM(pv_native_irq_disable, "cli" , .noinstr.text); |
70 | DEFINE_PARAVIRT_ASM(pv_native_irq_enable, "sti" , .noinstr.text); |
71 | DEFINE_PARAVIRT_ASM(pv_native_read_cr2, "mov %cr2, %rax" , .noinstr.text); |
72 | #endif |
73 | |
74 | DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key); |
75 | |
76 | void __init native_pv_lock_init(void) |
77 | { |
78 | if (IS_ENABLED(CONFIG_PARAVIRT_SPINLOCKS) && |
79 | !boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
80 | static_branch_disable(&virt_spin_lock_key); |
81 | } |
82 | |
83 | static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) |
84 | { |
85 | tlb_remove_page(tlb, page: table); |
86 | } |
87 | |
88 | unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, |
89 | unsigned int len) |
90 | { |
91 | /* |
92 | * Neat trick to map patch type back to the call within the |
93 | * corresponding structure. |
94 | */ |
95 | void *opfunc = *((void **)&pv_ops + type); |
96 | unsigned ret; |
97 | |
98 | if (opfunc == NULL) |
99 | /* If there's no function, patch it with paravirt_BUG() */ |
100 | ret = paravirt_patch_call(insn_buff, target: paravirt_BUG, addr, len); |
101 | else if (opfunc == _paravirt_nop) |
102 | ret = 0; |
103 | else |
104 | /* Otherwise call the function. */ |
105 | ret = paravirt_patch_call(insn_buff, target: opfunc, addr, len); |
106 | |
107 | return ret; |
108 | } |
109 | |
110 | struct static_key paravirt_steal_enabled; |
111 | struct static_key paravirt_steal_rq_enabled; |
112 | |
113 | static u64 native_steal_clock(int cpu) |
114 | { |
115 | return 0; |
116 | } |
117 | |
118 | DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); |
119 | DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); |
120 | |
121 | void paravirt_set_sched_clock(u64 (*func)(void)) |
122 | { |
123 | static_call_update(pv_sched_clock, func); |
124 | } |
125 | |
126 | /* These are in entry.S */ |
127 | static struct resource reserve_ioports = { |
128 | .start = 0, |
129 | .end = IO_SPACE_LIMIT, |
130 | .name = "paravirt-ioport" , |
131 | .flags = IORESOURCE_IO | IORESOURCE_BUSY, |
132 | }; |
133 | |
134 | /* |
135 | * Reserve the whole legacy IO space to prevent any legacy drivers |
136 | * from wasting time probing for their hardware. This is a fairly |
137 | * brute-force approach to disabling all non-virtual drivers. |
138 | * |
139 | * Note that this must be called very early to have any effect. |
140 | */ |
141 | int paravirt_disable_iospace(void) |
142 | { |
143 | return request_resource(root: &ioport_resource, new: &reserve_ioports); |
144 | } |
145 | |
146 | #ifdef CONFIG_PARAVIRT_XXL |
147 | static noinstr void pv_native_write_cr2(unsigned long val) |
148 | { |
149 | native_write_cr2(val); |
150 | } |
151 | |
152 | static noinstr unsigned long pv_native_get_debugreg(int regno) |
153 | { |
154 | return native_get_debugreg(regno); |
155 | } |
156 | |
157 | static noinstr void pv_native_set_debugreg(int regno, unsigned long val) |
158 | { |
159 | native_set_debugreg(regno, value: val); |
160 | } |
161 | |
162 | noinstr void pv_native_wbinvd(void) |
163 | { |
164 | native_wbinvd(); |
165 | } |
166 | |
167 | static noinstr void pv_native_safe_halt(void) |
168 | { |
169 | native_safe_halt(); |
170 | } |
171 | #endif |
172 | |
173 | struct pv_info pv_info = { |
174 | .name = "bare hardware" , |
175 | #ifdef CONFIG_PARAVIRT_XXL |
176 | .extra_user_64bit_cs = __USER_CS, |
177 | #endif |
178 | }; |
179 | |
180 | /* 64-bit pagetable entries */ |
181 | #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) |
182 | |
183 | struct paravirt_patch_template pv_ops = { |
184 | /* Cpu ops. */ |
185 | .cpu.io_delay = native_io_delay, |
186 | |
187 | #ifdef CONFIG_PARAVIRT_XXL |
188 | .cpu.cpuid = native_cpuid, |
189 | .cpu.get_debugreg = pv_native_get_debugreg, |
190 | .cpu.set_debugreg = pv_native_set_debugreg, |
191 | .cpu.read_cr0 = native_read_cr0, |
192 | .cpu.write_cr0 = native_write_cr0, |
193 | .cpu.write_cr4 = native_write_cr4, |
194 | .cpu.wbinvd = pv_native_wbinvd, |
195 | .cpu.read_msr = native_read_msr, |
196 | .cpu.write_msr = native_write_msr, |
197 | .cpu.read_msr_safe = native_read_msr_safe, |
198 | .cpu.write_msr_safe = native_write_msr_safe, |
199 | .cpu.read_pmc = native_read_pmc, |
200 | .cpu.load_tr_desc = native_load_tr_desc, |
201 | .cpu.set_ldt = native_set_ldt, |
202 | .cpu.load_gdt = native_load_gdt, |
203 | .cpu.load_idt = native_load_idt, |
204 | .cpu.store_tr = native_store_tr, |
205 | .cpu.load_tls = native_load_tls, |
206 | .cpu.load_gs_index = native_load_gs_index, |
207 | .cpu.write_ldt_entry = native_write_ldt_entry, |
208 | .cpu.write_gdt_entry = native_write_gdt_entry, |
209 | .cpu.write_idt_entry = native_write_idt_entry, |
210 | |
211 | .cpu.alloc_ldt = paravirt_nop, |
212 | .cpu.free_ldt = paravirt_nop, |
213 | |
214 | .cpu.load_sp0 = native_load_sp0, |
215 | |
216 | #ifdef CONFIG_X86_IOPL_IOPERM |
217 | .cpu.invalidate_io_bitmap = native_tss_invalidate_io_bitmap, |
218 | .cpu.update_io_bitmap = native_tss_update_io_bitmap, |
219 | #endif |
220 | |
221 | .cpu.start_context_switch = paravirt_nop, |
222 | .cpu.end_context_switch = paravirt_nop, |
223 | |
224 | /* Irq ops. */ |
225 | .irq.save_fl = __PV_IS_CALLEE_SAVE(pv_native_save_fl), |
226 | .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable), |
227 | .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable), |
228 | .irq.safe_halt = pv_native_safe_halt, |
229 | .irq.halt = native_halt, |
230 | #endif /* CONFIG_PARAVIRT_XXL */ |
231 | |
232 | /* Mmu ops. */ |
233 | .mmu.flush_tlb_user = native_flush_tlb_local, |
234 | .mmu.flush_tlb_kernel = native_flush_tlb_global, |
235 | .mmu.flush_tlb_one_user = native_flush_tlb_one_user, |
236 | .mmu.flush_tlb_multi = native_flush_tlb_multi, |
237 | .mmu.tlb_remove_table = native_tlb_remove_table, |
238 | |
239 | .mmu.exit_mmap = paravirt_nop, |
240 | .mmu.notify_page_enc_status_changed = paravirt_nop, |
241 | |
242 | #ifdef CONFIG_PARAVIRT_XXL |
243 | .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), |
244 | .mmu.write_cr2 = pv_native_write_cr2, |
245 | .mmu.read_cr3 = __native_read_cr3, |
246 | .mmu.write_cr3 = native_write_cr3, |
247 | |
248 | .mmu.pgd_alloc = __paravirt_pgd_alloc, |
249 | .mmu.pgd_free = paravirt_nop, |
250 | |
251 | .mmu.alloc_pte = paravirt_nop, |
252 | .mmu.alloc_pmd = paravirt_nop, |
253 | .mmu.alloc_pud = paravirt_nop, |
254 | .mmu.alloc_p4d = paravirt_nop, |
255 | .mmu.release_pte = paravirt_nop, |
256 | .mmu.release_pmd = paravirt_nop, |
257 | .mmu.release_pud = paravirt_nop, |
258 | .mmu.release_p4d = paravirt_nop, |
259 | |
260 | .mmu.set_pte = native_set_pte, |
261 | .mmu.set_pmd = native_set_pmd, |
262 | |
263 | .mmu.ptep_modify_prot_start = __ptep_modify_prot_start, |
264 | .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit, |
265 | |
266 | .mmu.set_pud = native_set_pud, |
267 | |
268 | .mmu.pmd_val = PTE_IDENT, |
269 | .mmu.make_pmd = PTE_IDENT, |
270 | |
271 | .mmu.pud_val = PTE_IDENT, |
272 | .mmu.make_pud = PTE_IDENT, |
273 | |
274 | .mmu.set_p4d = native_set_p4d, |
275 | |
276 | #if CONFIG_PGTABLE_LEVELS >= 5 |
277 | .mmu.p4d_val = PTE_IDENT, |
278 | .mmu.make_p4d = PTE_IDENT, |
279 | |
280 | .mmu.set_pgd = native_set_pgd, |
281 | #endif /* CONFIG_PGTABLE_LEVELS >= 5 */ |
282 | |
283 | .mmu.pte_val = PTE_IDENT, |
284 | .mmu.pgd_val = PTE_IDENT, |
285 | |
286 | .mmu.make_pte = PTE_IDENT, |
287 | .mmu.make_pgd = PTE_IDENT, |
288 | |
289 | .mmu.enter_mmap = paravirt_nop, |
290 | |
291 | .mmu.lazy_mode = { |
292 | .enter = paravirt_nop, |
293 | .leave = paravirt_nop, |
294 | .flush = paravirt_nop, |
295 | }, |
296 | |
297 | .mmu.set_fixmap = native_set_fixmap, |
298 | #endif /* CONFIG_PARAVIRT_XXL */ |
299 | |
300 | #if defined(CONFIG_PARAVIRT_SPINLOCKS) |
301 | /* Lock ops. */ |
302 | #ifdef CONFIG_SMP |
303 | .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, |
304 | .lock.queued_spin_unlock = |
305 | PV_CALLEE_SAVE(__native_queued_spin_unlock), |
306 | .lock.wait = paravirt_nop, |
307 | .lock.kick = paravirt_nop, |
308 | .lock.vcpu_is_preempted = |
309 | PV_CALLEE_SAVE(__native_vcpu_is_preempted), |
310 | #endif /* SMP */ |
311 | #endif |
312 | }; |
313 | |
314 | #ifdef CONFIG_PARAVIRT_XXL |
315 | NOKPROBE_SYMBOL(native_load_idt); |
316 | #endif |
317 | |
318 | EXPORT_SYMBOL(pv_ops); |
319 | EXPORT_SYMBOL_GPL(pv_info); |
320 | |