1	// SPDX-License-Identifier: GPL-2.0-only
2	/* cpu_feature_enabled() cannot be used this early */
3	#define USE_EARLY_PGTABLE_L5
4
5	#include <linux/memblock.h>
6	#include <linux/linkage.h>
7	#include <linux/bitops.h>
8	#include <linux/kernel.h>
9	#include <linux/export.h>
10	#include <linux/percpu.h>
11	#include <linux/string.h>
12	#include <linux/ctype.h>
13	#include <linux/delay.h>
14	#include <linux/sched/mm.h>
15	#include <linux/sched/clock.h>
16	#include <linux/sched/task.h>
17	#include <linux/sched/smt.h>
18	#include <linux/init.h>
19	#include <linux/kprobes.h>
20	#include <linux/kgdb.h>
21	#include <linux/mem_encrypt.h>
22	#include <linux/smp.h>
23	#include <linux/cpu.h>
24	#include <linux/io.h>
25	#include <linux/syscore_ops.h>
26	#include <linux/pgtable.h>
27	#include <linux/stackprotector.h>
28	#include <linux/utsname.h>
29
30	#include <asm/alternative.h>
31	#include <asm/cmdline.h>
32	#include <asm/cpuid/api.h>
33	#include <asm/perf_event.h>
34	#include <asm/mmu_context.h>
35	#include <asm/doublefault.h>
36	#include <asm/archrandom.h>
37	#include <asm/hypervisor.h>
38	#include <asm/processor.h>
39	#include <asm/tlbflush.h>
40	#include <asm/debugreg.h>
41	#include <asm/sections.h>
42	#include <asm/vsyscall.h>
43	#include <linux/topology.h>
44	#include <linux/cpumask.h>
45	#include <linux/atomic.h>
46	#include <asm/proto.h>
47	#include <asm/setup.h>
48	#include <asm/apic.h>
49	#include <asm/desc.h>
50	#include <asm/fpu/api.h>
51	#include <asm/mtrr.h>
52	#include <asm/hwcap2.h>
53	#include <linux/numa.h>
54	#include <asm/numa.h>
55	#include <asm/asm.h>
56	#include <asm/bugs.h>
57	#include <asm/cpu.h>
58	#include <asm/mce.h>
59	#include <asm/msr.h>
60	#include <asm/cacheinfo.h>
61	#include <asm/memtype.h>
62	#include <asm/microcode.h>
63	#include <asm/intel-family.h>
64	#include <asm/cpu_device_id.h>
65	#include <asm/fred.h>
66	#include <asm/uv/uv.h>
67	#include <asm/ia32.h>
68	#include <asm/set_memory.h>
69	#include <asm/traps.h>
70	#include <asm/sev.h>
71	#include <asm/tdx.h>
72	#include <asm/posted_intr.h>
73	#include <asm/runtime-const.h>
74
75	#include "cpu.h"
76
77	DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
78	EXPORT_PER_CPU_SYMBOL(cpu_info);
79
80	u32 elf_hwcap2 __read_mostly;
81
82	/* Number of siblings per CPU package */
83	unsigned int __max_threads_per_core __ro_after_init = 1;
84	EXPORT_SYMBOL(__max_threads_per_core);
85
86	unsigned int __max_dies_per_package __ro_after_init = 1;
87	EXPORT_SYMBOL(__max_dies_per_package);
88
89	unsigned int __max_logical_packages __ro_after_init = 1;
90	EXPORT_SYMBOL(__max_logical_packages);
91
92	unsigned int __num_cores_per_package __ro_after_init = 1;
93	EXPORT_SYMBOL(__num_cores_per_package);
94
95	unsigned int __num_threads_per_package __ro_after_init = 1;
96	EXPORT_SYMBOL(__num_threads_per_package);
97
98	static struct ppin_info {
99	int feature;
100	int msr_ppin_ctl;
101	int msr_ppin;
102	} ppin_info[] = {
103	[X86_VENDOR_INTEL] = {
104	.feature = X86_FEATURE_INTEL_PPIN,
105	.msr_ppin_ctl = MSR_PPIN_CTL,
106	.msr_ppin = MSR_PPIN
107	},
108	[X86_VENDOR_AMD] = {
109	.feature = X86_FEATURE_AMD_PPIN,
110	.msr_ppin_ctl = MSR_AMD_PPIN_CTL,
111	.msr_ppin = MSR_AMD_PPIN
112	},
113	};
114
115	static const struct x86_cpu_id ppin_cpuids[] = {
116	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),
117	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]),
118
119	/* Legacy models without CPUID enumeration */
120	X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),
121	X86_MATCH_VFM(INTEL_HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),
122	X86_MATCH_VFM(INTEL_BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),
123	X86_MATCH_VFM(INTEL_BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),
124	X86_MATCH_VFM(INTEL_SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),
125	X86_MATCH_VFM(INTEL_ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),
126	X86_MATCH_VFM(INTEL_ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),
127	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
128	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),
129	X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),
130	X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]),
131
132	{}
133	};
134
135	static void ppin_init(struct cpuinfo_x86 *c)
136	{
137	const struct x86_cpu_id *id;
138	unsigned long long val;
139	struct ppin_info *info;
140
141	id = x86_match_cpu(match: ppin_cpuids);
142	if (!id)
143	return;
144
145	/*
146	* Testing the presence of the MSR is not enough. Need to check
147	* that the PPIN_CTL allows reading of the PPIN.
148	*/
149	info = (struct ppin_info *)id->driver_data;
150
151	if (rdmsrq_safe(msr: info->msr_ppin_ctl, p: &val))
152	goto clear_ppin;
153
154	if ((val & 3UL) == 1UL) {
155	/* PPIN locked in disabled mode */
156	goto clear_ppin;
157	}
158
159	/* If PPIN is disabled, try to enable */
160	if (!(val & 2UL)) {
161	wrmsrq_safe(msr: info->msr_ppin_ctl, val: val | 2UL);
162	rdmsrq_safe(msr: info->msr_ppin_ctl, p: &val);
163	}
164
165	/* Is the enable bit set? */
166	if (val & 2UL) {
167	c->ppin = native_rdmsrq(msr: info->msr_ppin);
168	set_cpu_cap(c, info->feature);
169	return;
170	}
171
172	clear_ppin:
173	setup_clear_cpu_cap(info->feature);
174	}
175
176	static void default_init(struct cpuinfo_x86 *c)
177	{
178	#ifdef CONFIG_X86_64
179	cpu_detect_cache_sizes(c);
180	#else
181	/* Not much we can do here... */
182	/* Check if at least it has cpuid */
183	if (c->cpuid_level == -1) {
184	/* No cpuid. It must be an ancient CPU */
185	if (c->x86 == 4)
186	strcpy(c->x86_model_id, "486");
187	else if (c->x86 == 3)
188	strcpy(c->x86_model_id, "386");
189	}
190	#endif
191	}
192
193	static const struct cpu_dev default_cpu = {
194	.c_init = default_init,
195	.c_vendor = "Unknown",
196	.c_x86_vendor = X86_VENDOR_UNKNOWN,
197	};
198
199	static const struct cpu_dev *this_cpu = &default_cpu;
200
201	DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
202	#ifdef CONFIG_X86_64
203	/*
204	* We need valid kernel segments for data and code in long mode too
205	* IRET will check the segment types kkeil 2000/10/28
206	* Also sysret mandates a special GDT layout
207	*
208	* TLS descriptors are currently at a different place compared to i386.
209	* Hopefully nobody expects them at a fixed place (Wine?)
210	*/
211	[GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
212	[GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),
213	[GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),
214	[GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
215	[GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),
216	[GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),
217	#else
218	[GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),
219	[GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
220	[GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),
221	[GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),
222	/*
223	* Segments used for calling PnP BIOS have byte granularity.
224	* They code segments and data segments have fixed 64k limits,
225	* the transfer segment sizes are set at run time.
226	*/
227	[GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
228	[GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
229	[GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),
230	[GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
231	[GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(DESC_DATA16, 0, 0),
232	/*
233	* The APM segments have byte granularity and their bases
234	* are set at run time. All have 64k limits.
235	*/
236	[GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),
237	[GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),
238	[GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff),
239
240	[GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
241	[GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),
242	#endif
243	} };
244	EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
245	SYM_PIC_ALIAS(gdt_page);
246
247	#ifdef CONFIG_X86_64
248	static int __init x86_nopcid_setup(char *s)
249	{
250	/* nopcid doesn't accept parameters */
251	if (s)
252	return -EINVAL;
253
254	/* do not emit a message if the feature is not present */
255	if (!boot_cpu_has(X86_FEATURE_PCID))
256	return 0;
257
258	setup_clear_cpu_cap(X86_FEATURE_PCID);
259	pr_info("nopcid: PCID feature disabled\n");
260	return 0;
261	}
262	early_param("nopcid", x86_nopcid_setup);
263	#endif
264
265	static int __init x86_noinvpcid_setup(char *s)
266	{
267	/* noinvpcid doesn't accept parameters */
268	if (s)
269	return -EINVAL;
270
271	/* do not emit a message if the feature is not present */
272	if (!boot_cpu_has(X86_FEATURE_INVPCID))
273	return 0;
274
275	setup_clear_cpu_cap(X86_FEATURE_INVPCID);
276	pr_info("noinvpcid: INVPCID feature disabled\n");
277	return 0;
278	}
279	early_param("noinvpcid", x86_noinvpcid_setup);
280
281	/* Standard macro to see if a specific flag is changeable */
282	static inline bool flag_is_changeable_p(unsigned long flag)
283	{
284	unsigned long f1, f2;
285
286	if (!IS_ENABLED(CONFIG_X86_32))
287	return true;
288
289	/*
290	* Cyrix and IDT cpus allow disabling of CPUID
291	* so the code below may return different results
292	* when it is executed before and after enabling
293	* the CPUID. Add "volatile" to not allow gcc to
294	* optimize the subsequent calls to this function.
295	*/
296	asm volatile ("pushfl \n\t"
297	"pushfl \n\t"
298	"popl %0 \n\t"
299	"movl %0, %1 \n\t"
300	"xorl %2, %0 \n\t"
301	"pushl %0 \n\t"
302	"popfl \n\t"
303	"pushfl \n\t"
304	"popl %0 \n\t"
305	"popfl \n\t"
306
307	: "=&r" (f1), "=&r" (f2)
308	: "ir" (flag));
309
310	return (f1 ^ f2) & flag;
311	}
312
313	#ifdef CONFIG_X86_32
314	static int cachesize_override = -1;
315	static int disable_x86_serial_nr = 1;
316
317	static int __init cachesize_setup(char *str)
318	{
319	get_option(&str, &cachesize_override);
320	return 1;
321	}
322	__setup("cachesize=", cachesize_setup);
323
324	/* Probe for the CPUID instruction */
325	bool cpuid_feature(void)
326	{
327	return flag_is_changeable_p(X86_EFLAGS_ID);
328	}
329
330	static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
331	{
332	unsigned long lo, hi;
333
334	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
335	return;
336
337	/* Disable processor serial number: */
338
339	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
340	lo |= 0x200000;
341	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
342
343	pr_notice("CPU serial number disabled.\n");
344	clear_cpu_cap(c, X86_FEATURE_PN);
345
346	/* Disabling the serial number may affect the cpuid level */
347	c->cpuid_level = cpuid_eax(0);
348	}
349
350	static int __init x86_serial_nr_setup(char *s)
351	{
352	disable_x86_serial_nr = 0;
353	return 1;
354	}
355	__setup("serialnumber", x86_serial_nr_setup);
356	#else
357	static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
358	{
359	}
360	#endif
361
362	static __always_inline void setup_smep(struct cpuinfo_x86 *c)
363	{
364	if (cpu_has(c, X86_FEATURE_SMEP))
365	cr4_set_bits(X86_CR4_SMEP);
366	}
367
368	static __always_inline void setup_smap(struct cpuinfo_x86 *c)
369	{
370	unsigned long eflags = native_save_fl();
371
372	/* This should have been cleared long ago */
373	BUG_ON(eflags & X86_EFLAGS_AC);
374
375	if (cpu_has(c, X86_FEATURE_SMAP))
376	cr4_set_bits(X86_CR4_SMAP);
377	}
378
379	static __always_inline void setup_umip(struct cpuinfo_x86 *c)
380	{
381	/* Check the boot processor, plus build option for UMIP. */
382	if (!cpu_feature_enabled(X86_FEATURE_UMIP))
383	goto out;
384
385	/* Check the current processor's cpuid bits. */
386	if (!cpu_has(c, X86_FEATURE_UMIP))
387	goto out;
388
389	cr4_set_bits(X86_CR4_UMIP);
390
391	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
392
393	return;
394
395	out:
396	/*
397	* Make sure UMIP is disabled in case it was enabled in a
398	* previous boot (e.g., via kexec).
399	*/
400	cr4_clear_bits(X86_CR4_UMIP);
401	}
402
403	/* These bits should not change their value after CPU init is finished. */
404	static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |
405	X86_CR4_FSGSBASE | X86_CR4_CET | X86_CR4_FRED;
406	static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
407	static unsigned long cr4_pinned_bits __ro_after_init;
408
409	void native_write_cr0(unsigned long val)
410	{
411	unsigned long bits_missing = 0;
412
413	set_register:
414	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
415
416	if (static_branch_likely(&cr_pinning)) {
417	if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
418	bits_missing = X86_CR0_WP;
419	val |= bits_missing;
420	goto set_register;
421	}
422	/* Warn after we've set the missing bits. */
423	WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
424	}
425	}
426	EXPORT_SYMBOL(native_write_cr0);
427
428	void __no_profile native_write_cr4(unsigned long val)
429	{
430	unsigned long bits_changed = 0;
431
432	set_register:
433	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
434
435	if (static_branch_likely(&cr_pinning)) {
436	if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
437	bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
438	val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
439	goto set_register;
440	}
441	/* Warn after we've corrected the changed bits. */
442	WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
443	bits_changed);
444	}
445	}
446	#if IS_MODULE(CONFIG_LKDTM)
447	EXPORT_SYMBOL_GPL(native_write_cr4);
448	#endif
449
450	void cr4_update_irqsoff(unsigned long set, unsigned long clear)
451	{
452	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
453
454	lockdep_assert_irqs_disabled();
455
456	newval = (cr4 & ~clear) | set;
457	if (newval != cr4) {
458	this_cpu_write(cpu_tlbstate.cr4, newval);
459	__write_cr4(x: newval);
460	}
461	}
462	EXPORT_SYMBOL(cr4_update_irqsoff);
463
464	/* Read the CR4 shadow. */
465	unsigned long cr4_read_shadow(void)
466	{
467	return this_cpu_read(cpu_tlbstate.cr4);
468	}
469	EXPORT_SYMBOL_GPL(cr4_read_shadow);
470
471	void cr4_init(void)
472	{
473	unsigned long cr4 = __read_cr4();
474
475	if (boot_cpu_has(X86_FEATURE_PCID))
476	cr4 |= X86_CR4_PCIDE;
477	if (static_branch_likely(&cr_pinning))
478	cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
479
480	__write_cr4(x: cr4);
481
482	/* Initialize cr4 shadow for this CPU. */
483	this_cpu_write(cpu_tlbstate.cr4, cr4);
484	}
485
486	/*
487	* Once CPU feature detection is finished (and boot params have been
488	* parsed), record any of the sensitive CR bits that are set, and
489	* enable CR pinning.
490	*/
491	static void __init setup_cr_pinning(void)
492	{
493	cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
494	static_key_enable(key: &cr_pinning.key);
495	}
496
497	static __init int x86_nofsgsbase_setup(char *arg)
498	{
499	/* Require an exact match without trailing characters. */
500	if (strlen(arg))
501	return 0;
502
503	/* Do not emit a message if the feature is not present. */
504	if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
505	return 1;
506
507	setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
508	pr_info("FSGSBASE disabled via kernel command line\n");
509	return 1;
510	}
511	__setup("nofsgsbase", x86_nofsgsbase_setup);
512
513	/*
514	* Protection Keys are not available in 32-bit mode.
515	*/
516	static bool pku_disabled;
517
518	static __always_inline void setup_pku(struct cpuinfo_x86 *c)
519	{
520	if (c == &boot_cpu_data) {
521	if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
522	return;
523	/*
524	* Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
525	* bit to be set. Enforce it.
526	*/
527	setup_force_cpu_cap(X86_FEATURE_OSPKE);
528
529	} else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
530	return;
531	}
532
533	cr4_set_bits(X86_CR4_PKE);
534	/* Load the default PKRU value */
535	pkru_write_default();
536	}
537
538	#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
539	static __init int setup_disable_pku(char *arg)
540	{
541	/*
542	* Do not clear the X86_FEATURE_PKU bit. All of the
543	* runtime checks are against OSPKE so clearing the
544	* bit does nothing.
545	*
546	* This way, we will see "pku" in cpuinfo, but not
547	* "ospke", which is exactly what we want. It shows
548	* that the CPU has PKU, but the OS has not enabled it.
549	* This happens to be exactly how a system would look
550	* if we disabled the config option.
551	*/
552	pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
553	pku_disabled = true;
554	return 1;
555	}
556	__setup("nopku", setup_disable_pku);
557	#endif
558
559	#ifdef CONFIG_X86_KERNEL_IBT
560
561	__noendbr u64 ibt_save(bool disable)
562	{
563	u64 msr = 0;
564
565	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
566	rdmsrq(MSR_IA32_S_CET, msr);
567	if (disable)
568	wrmsrq(MSR_IA32_S_CET, val: msr & ~CET_ENDBR_EN);
569	}
570
571	return msr;
572	}
573
574	__noendbr void ibt_restore(u64 save)
575	{
576	u64 msr;
577
578	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
579	rdmsrq(MSR_IA32_S_CET, msr);
580	msr &= ~CET_ENDBR_EN;
581	msr |= (save & CET_ENDBR_EN);
582	wrmsrq(MSR_IA32_S_CET, val: msr);
583	}
584	}
585
586	#endif
587
588	static __always_inline void setup_cet(struct cpuinfo_x86 *c)
589	{
590	bool user_shstk, kernel_ibt;
591
592	if (!IS_ENABLED(CONFIG_X86_CET))
593	return;
594
595	kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
596	user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
597	IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
598
599	if (!kernel_ibt && !user_shstk)
600	return;
601
602	if (user_shstk)
603	set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
604
605	if (kernel_ibt)
606	wrmsrq(MSR_IA32_S_CET, CET_ENDBR_EN);
607	else
608	wrmsrq(MSR_IA32_S_CET, val: 0);
609
610	cr4_set_bits(X86_CR4_CET);
611
612	if (kernel_ibt && ibt_selftest()) {
613	pr_err("IBT selftest: Failed!\n");
614	wrmsrq(MSR_IA32_S_CET, val: 0);
615	setup_clear_cpu_cap(X86_FEATURE_IBT);
616	}
617	}
618
619	__noendbr void cet_disable(void)
620	{
621	if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
622	cpu_feature_enabled(X86_FEATURE_SHSTK)))
623	return;
624
625	wrmsrq(MSR_IA32_S_CET, val: 0);
626	wrmsrq(MSR_IA32_U_CET, val: 0);
627	}
628
629	/*
630	* Some CPU features depend on higher CPUID levels, which may not always
631	* be available due to CPUID level capping or broken virtualization
632	* software. Add those features to this table to auto-disable them.
633	*/
634	struct cpuid_dependent_feature {
635	u32 feature;
636	u32 level;
637	};
638
639	static const struct cpuid_dependent_feature
640	cpuid_dependent_features[] = {
641	{ X86_FEATURE_MWAIT, CPUID_LEAF_MWAIT },
642	{ X86_FEATURE_DCA, CPUID_LEAF_DCA },
643	{ X86_FEATURE_XSAVE, CPUID_LEAF_XSTATE },
644	{ 0, 0 }
645	};
646
647	static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
648	{
649	const struct cpuid_dependent_feature *df;
650
651	for (df = cpuid_dependent_features; df->feature; df++) {
652
653	if (!cpu_has(c, df->feature))
654	continue;
655	/*
656	* Note: cpuid_level is set to -1 if unavailable, but
657	* extended_extended_level is set to 0 if unavailable
658	* and the legitimate extended levels are all negative
659	* when signed; hence the weird messing around with
660	* signs here...
661	*/
662	if (!((s32)df->level < 0 ?
663	(u32)df->level > (u32)c->extended_cpuid_level :
664	(s32)df->level > (s32)c->cpuid_level))
665	continue;
666
667	clear_cpu_cap(c, bit: df->feature);
668	if (!warn)
669	continue;
670
671	pr_warn("CPU: CPU feature %s disabled, no CPUID level 0x%x\n",
672	x86_cap_flags[df->feature], df->level);
673	}
674	}
675
676	/*
677	* Naming convention should be: <Name> [(<Codename>)]
678	* This table only is used unless init_<vendor>() below doesn't set it;
679	* in particular, if CPUID levels 0x80000002..4 are supported, this
680	* isn't used
681	*/
682
683	/* Look up CPU names by table lookup. */
684	static const char *table_lookup_model(struct cpuinfo_x86 *c)
685	{
686	#ifdef CONFIG_X86_32
687	const struct legacy_cpu_model_info *info;
688
689	if (c->x86_model >= 16)
690	return NULL; /* Range check */
691
692	if (!this_cpu)
693	return NULL;
694
695	info = this_cpu->legacy_models;
696
697	while (info->family) {
698	if (info->family == c->x86)
699	return info->model_names[c->x86_model];
700	info++;
701	}
702	#endif
703	return NULL; /* Not found */
704	}
705
706	/* Aligned to unsigned long to avoid split lock in atomic bitmap ops */
707	__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
708	__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
709
710	#ifdef CONFIG_X86_32
711	/* The 32-bit entry code needs to find cpu_entry_area. */
712	DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
713	#endif
714
715	/* Load the original GDT from the per-cpu structure */
716	void load_direct_gdt(int cpu)
717	{
718	struct desc_ptr gdt_descr;
719
720	gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
721	gdt_descr.size = GDT_SIZE - 1;
722	load_gdt(dtr: &gdt_descr);
723	}
724	EXPORT_SYMBOL_GPL(load_direct_gdt);
725
726	/* Load a fixmap remapping of the per-cpu GDT */
727	void load_fixmap_gdt(int cpu)
728	{
729	struct desc_ptr gdt_descr;
730
731	gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
732	gdt_descr.size = GDT_SIZE - 1;
733	load_gdt(dtr: &gdt_descr);
734	}
735	EXPORT_SYMBOL_GPL(load_fixmap_gdt);
736
737	/**
738	* switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
739	* @cpu: The CPU number for which this is invoked
740	*
741	* Invoked during early boot to switch from early GDT and early per CPU to
742	* the direct GDT and the runtime per CPU area. On 32-bit the percpu base
743	* switch is implicit by loading the direct GDT. On 64bit this requires
744	* to update GSBASE.
745	*/
746	void __init switch_gdt_and_percpu_base(int cpu)
747	{
748	load_direct_gdt(cpu);
749
750	#ifdef CONFIG_X86_64
751	/*
752	* No need to load %gs. It is already correct.
753	*
754	* Writing %gs on 64bit would zero GSBASE which would make any per
755	* CPU operation up to the point of the wrmsrq() fault.
756	*
757	* Set GSBASE to the new offset. Until the wrmsrq() happens the
758	* early mapping is still valid. That means the GSBASE update will
759	* lose any prior per CPU data which was not copied over in
760	* setup_per_cpu_areas().
761	*
762	* This works even with stackprotector enabled because the
763	* per CPU stack canary is 0 in both per CPU areas.
764	*/
765	wrmsrq(MSR_GS_BASE, val: cpu_kernelmode_gs_base(cpu));
766	#else
767	/*
768	* %fs is already set to __KERNEL_PERCPU, but after switching GDT
769	* it is required to load FS again so that the 'hidden' part is
770	* updated from the new GDT. Up to this point the early per CPU
771	* translation is active. Any content of the early per CPU data
772	* which was not copied over in setup_per_cpu_areas() is lost.
773	*/
774	loadsegment(fs, __KERNEL_PERCPU);
775	#endif
776	}
777
778	static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
779
780	static void get_model_name(struct cpuinfo_x86 *c)
781	{
782	unsigned int *v;
783	char *p, *q, *s;
784
785	if (c->extended_cpuid_level < 0x80000004)
786	return;
787
788	v = (unsigned int *)c->x86_model_id;
789	cpuid(op: 0x80000002, eax: &v[0], ebx: &v[1], ecx: &v[2], edx: &v[3]);
790	cpuid(op: 0x80000003, eax: &v[4], ebx: &v[5], ecx: &v[6], edx: &v[7]);
791	cpuid(op: 0x80000004, eax: &v[8], ebx: &v[9], ecx: &v[10], edx: &v[11]);
792	c->x86_model_id[48] = 0;
793
794	/* Trim whitespace */
795	p = q = s = &c->x86_model_id[0];
796
797	while (*p == ' ')
798	p++;
799
800	while (*p) {
801	/* Note the last non-whitespace index */
802	if (!isspace(*p))
803	s = q;
804
805	*q++ = *p++;
806	}
807
808	*(s + 1) = '\0';
809	}
810
811	void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
812	{
813	unsigned int n, dummy, ebx, ecx, edx, l2size;
814
815	n = c->extended_cpuid_level;
816
817	if (n >= 0x80000005) {
818	cpuid(op: 0x80000005, eax: &dummy, ebx: &ebx, ecx: &ecx, edx: &edx);
819	c->x86_cache_size = (ecx>>24) + (edx>>24);
820	#ifdef CONFIG_X86_64
821	/* On K8 L1 TLB is inclusive, so don't count it */
822	c->x86_tlbsize = 0;
823	#endif
824	}
825
826	if (n < 0x80000006) /* Some chips just has a large L1. */
827	return;
828
829	cpuid(op: 0x80000006, eax: &dummy, ebx: &ebx, ecx: &ecx, edx: &edx);
830	l2size = ecx >> 16;
831
832	#ifdef CONFIG_X86_64
833	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
834	#else
835	/* do processor-specific cache resizing */
836	if (this_cpu->legacy_cache_size)
837	l2size = this_cpu->legacy_cache_size(c, l2size);
838
839	/* Allow user to override all this if necessary. */
840	if (cachesize_override != -1)
841	l2size = cachesize_override;
842
843	if (l2size == 0)
844	return; /* Again, no L2 cache is possible */
845	#endif
846
847	c->x86_cache_size = l2size;
848	}
849
850	u16 __read_mostly tlb_lli_4k;
851	u16 __read_mostly tlb_lli_2m;
852	u16 __read_mostly tlb_lli_4m;
853	u16 __read_mostly tlb_lld_4k;
854	u16 __read_mostly tlb_lld_2m;
855	u16 __read_mostly tlb_lld_4m;
856	u16 __read_mostly tlb_lld_1g;
857
858	static void cpu_detect_tlb(struct cpuinfo_x86 *c)
859	{
860	if (this_cpu->c_detect_tlb)
861	this_cpu->c_detect_tlb(c);
862
863	pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
864	tlb_lli_4k, tlb_lli_2m, tlb_lli_4m);
865
866	pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
867	tlb_lld_4k, tlb_lld_2m, tlb_lld_4m, tlb_lld_1g);
868	}
869
870	void get_cpu_vendor(struct cpuinfo_x86 *c)
871	{
872	char *v = c->x86_vendor_id;
873	int i;
874
875	for (i = 0; i < X86_VENDOR_NUM; i++) {
876	if (!cpu_devs[i])
877	break;
878
879	if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
880	(cpu_devs[i]->c_ident[1] &&
881	!strcmp(v, cpu_devs[i]->c_ident[1]))) {
882
883	this_cpu = cpu_devs[i];
884	c->x86_vendor = this_cpu->c_x86_vendor;
885	return;
886	}
887	}
888
889	pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
890	"CPU: Your system may be unstable.\n", v);
891
892	c->x86_vendor = X86_VENDOR_UNKNOWN;
893	this_cpu = &default_cpu;
894	}
895
896	void cpu_detect(struct cpuinfo_x86 *c)
897	{
898	/* Get vendor name */
899	cpuid(op: 0x00000000, eax: (unsigned int *)&c->cpuid_level,
900	ebx: (unsigned int *)&c->x86_vendor_id[0],
901	ecx: (unsigned int *)&c->x86_vendor_id[8],
902	edx: (unsigned int *)&c->x86_vendor_id[4]);
903
904	c->x86 = 4;
905	/* Intel-defined flags: level 0x00000001 */
906	if (c->cpuid_level >= 0x00000001) {
907	u32 junk, tfms, cap0, misc;
908
909	cpuid(op: 0x00000001, eax: &tfms, ebx: &misc, ecx: &junk, edx: &cap0);
910	c->x86 = x86_family(sig: tfms);
911	c->x86_model = x86_model(sig: tfms);
912	c->x86_stepping = x86_stepping(sig: tfms);
913
914	if (cap0 & (1<<19)) {
915	c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
916	c->x86_cache_alignment = c->x86_clflush_size;
917	}
918	}
919	}
920
921	static void apply_forced_caps(struct cpuinfo_x86 *c)
922	{
923	int i;
924
925	for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
926	c->x86_capability[i] &= ~cpu_caps_cleared[i];
927	c->x86_capability[i] |= cpu_caps_set[i];
928	}
929	}
930
931	static void init_speculation_control(struct cpuinfo_x86 *c)
932	{
933	/*
934	* The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
935	* and they also have a different bit for STIBP support. Also,
936	* a hypervisor might have set the individual AMD bits even on
937	* Intel CPUs, for finer-grained selection of what's available.
938	*/
939	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
940	set_cpu_cap(c, X86_FEATURE_IBRS);
941	set_cpu_cap(c, X86_FEATURE_IBPB);
942	set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
943	}
944
945	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
946	set_cpu_cap(c, X86_FEATURE_STIBP);
947
948	if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
949	cpu_has(c, X86_FEATURE_VIRT_SSBD))
950	set_cpu_cap(c, X86_FEATURE_SSBD);
951
952	if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
953	set_cpu_cap(c, X86_FEATURE_IBRS);
954	set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
955	}
956
957	if (cpu_has(c, X86_FEATURE_AMD_IBPB))
958	set_cpu_cap(c, X86_FEATURE_IBPB);
959
960	if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
961	set_cpu_cap(c, X86_FEATURE_STIBP);
962	set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
963	}
964
965	if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
966	set_cpu_cap(c, X86_FEATURE_SSBD);
967	set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
968	clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
969	}
970	}
971
972	void get_cpu_cap(struct cpuinfo_x86 *c)
973	{
974	u32 eax, ebx, ecx, edx;
975
976	/* Intel-defined flags: level 0x00000001 */
977	if (c->cpuid_level >= 0x00000001) {
978	cpuid(op: 0x00000001, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
979
980	c->x86_capability[CPUID_1_ECX] = ecx;
981	c->x86_capability[CPUID_1_EDX] = edx;
982	}
983
984	/* Thermal and Power Management Leaf: level 0x00000006 (eax) */
985	if (c->cpuid_level >= 0x00000006)
986	c->x86_capability[CPUID_6_EAX] = cpuid_eax(op: 0x00000006);
987
988	/* Additional Intel-defined flags: level 0x00000007 */
989	if (c->cpuid_level >= 0x00000007) {
990	cpuid_count(op: 0x00000007, count: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
991	c->x86_capability[CPUID_7_0_EBX] = ebx;
992	c->x86_capability[CPUID_7_ECX] = ecx;
993	c->x86_capability[CPUID_7_EDX] = edx;
994
995	/* Check valid sub-leaf index before accessing it */
996	if (eax >= 1) {
997	cpuid_count(op: 0x00000007, count: 1, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
998	c->x86_capability[CPUID_7_1_EAX] = eax;
999	}
1000	}
1001
1002	/* Extended state features: level 0x0000000d */
1003	if (c->cpuid_level >= 0x0000000d) {
1004	cpuid_count(op: 0x0000000d, count: 1, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
1005
1006	c->x86_capability[CPUID_D_1_EAX] = eax;
1007	}
1008
1009	/*
1010	* Check if extended CPUID leaves are implemented: Max extended
1011	* CPUID leaf must be in the 0x80000001-0x8000ffff range.
1012	*/
1013	eax = cpuid_eax(op: 0x80000000);
1014	c->extended_cpuid_level = ((eax & 0xffff0000) == 0x80000000) ? eax : 0;
1015
1016	if (c->extended_cpuid_level >= 0x80000001) {
1017	cpuid(op: 0x80000001, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
1018
1019	c->x86_capability[CPUID_8000_0001_ECX] = ecx;
1020	c->x86_capability[CPUID_8000_0001_EDX] = edx;
1021	}
1022
1023	if (c->extended_cpuid_level >= 0x80000007) {
1024	cpuid(op: 0x80000007, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
1025
1026	c->x86_capability[CPUID_8000_0007_EBX] = ebx;
1027	c->x86_power = edx;
1028	}
1029
1030	if (c->extended_cpuid_level >= 0x80000008) {
1031	cpuid(op: 0x80000008, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
1032	c->x86_capability[CPUID_8000_0008_EBX] = ebx;
1033	}
1034
1035	if (c->extended_cpuid_level >= 0x8000000a)
1036	c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(op: 0x8000000a);
1037
1038	if (c->extended_cpuid_level >= 0x8000001f)
1039	c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(op: 0x8000001f);
1040
1041	if (c->extended_cpuid_level >= 0x80000021)
1042	c->x86_capability[CPUID_8000_0021_EAX] = cpuid_eax(op: 0x80000021);
1043
1044	init_scattered_cpuid_features(c);
1045	init_speculation_control(c);
1046
1047	/*
1048	* Clear/Set all flags overridden by options, after probe.
1049	* This needs to happen each time we re-probe, which may happen
1050	* several times during CPU initialization.
1051	*/
1052	apply_forced_caps(c);
1053	}
1054
1055	void get_cpu_address_sizes(struct cpuinfo_x86 *c)
1056	{
1057	u32 eax, ebx, ecx, edx;
1058
1059	if (!cpu_has(c, X86_FEATURE_CPUID) ||
1060	(c->extended_cpuid_level < 0x80000008)) {
1061	if (IS_ENABLED(CONFIG_X86_64)) {
1062	c->x86_clflush_size = 64;
1063	c->x86_phys_bits = 36;
1064	c->x86_virt_bits = 48;
1065	} else {
1066	c->x86_clflush_size = 32;
1067	c->x86_virt_bits = 32;
1068	c->x86_phys_bits = 32;
1069
1070	if (cpu_has(c, X86_FEATURE_PAE) ||
1071	cpu_has(c, X86_FEATURE_PSE36))
1072	c->x86_phys_bits = 36;
1073	}
1074	} else {
1075	cpuid(op: 0x80000008, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
1076
1077	c->x86_virt_bits = (eax >> 8) & 0xff;
1078	c->x86_phys_bits = eax & 0xff;
1079
1080	/* Provide a sane default if not enumerated: */
1081	if (!c->x86_clflush_size)
1082	c->x86_clflush_size = 32;
1083	}
1084
1085	c->x86_cache_bits = c->x86_phys_bits;
1086	c->x86_cache_alignment = c->x86_clflush_size;
1087	}
1088
1089	static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
1090	{
1091	int i;
1092
1093	/*
1094	* First of all, decide if this is a 486 or higher
1095	* It's a 486 if we can modify the AC flag
1096	*/
1097	if (flag_is_changeable_p(X86_EFLAGS_AC))
1098	c->x86 = 4;
1099	else
1100	c->x86 = 3;
1101
1102	for (i = 0; i < X86_VENDOR_NUM; i++)
1103	if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1104	c->x86_vendor_id[0] = 0;
1105	cpu_devs[i]->c_identify(c);
1106	if (c->x86_vendor_id[0]) {
1107	get_cpu_vendor(c);
1108	break;
1109	}
1110	}
1111	}
1112
1113	#define NO_SPECULATION BIT(0)
1114	#define NO_MELTDOWN BIT(1)
1115	#define NO_SSB BIT(2)
1116	#define NO_L1TF BIT(3)
1117	#define NO_MDS BIT(4)
1118	#define MSBDS_ONLY BIT(5)
1119	#define NO_SWAPGS BIT(6)
1120	#define NO_ITLB_MULTIHIT BIT(7)
1121	#define NO_SPECTRE_V2 BIT(8)
1122	#define NO_MMIO BIT(9)
1123	#define NO_EIBRS_PBRSB BIT(10)
1124	#define NO_BHI BIT(11)
1125
1126	#define VULNWL(vendor, family, model, whitelist) \
1127	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1128
1129	#define VULNWL_INTEL(vfm, whitelist) \
1130	X86_MATCH_VFM(vfm, whitelist)
1131
1132	#define VULNWL_AMD(family, whitelist) \
1133	VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1134
1135	#define VULNWL_HYGON(family, whitelist) \
1136	VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1137
1138	static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1139	VULNWL(ANY, 4, X86_MODEL_ANY, NO_SPECULATION),
1140	VULNWL(CENTAUR, 5, X86_MODEL_ANY, NO_SPECULATION),
1141	VULNWL(INTEL, 5, X86_MODEL_ANY, NO_SPECULATION),
1142	VULNWL(NSC, 5, X86_MODEL_ANY, NO_SPECULATION),
1143	VULNWL(VORTEX, 5, X86_MODEL_ANY, NO_SPECULATION),
1144	VULNWL(VORTEX, 6, X86_MODEL_ANY, NO_SPECULATION),
1145
1146	/* Intel Family 6 */
1147	VULNWL_INTEL(INTEL_TIGERLAKE, NO_MMIO),
1148	VULNWL_INTEL(INTEL_TIGERLAKE_L, NO_MMIO),
1149	VULNWL_INTEL(INTEL_ALDERLAKE, NO_MMIO),
1150	VULNWL_INTEL(INTEL_ALDERLAKE_L, NO_MMIO),
1151
1152	VULNWL_INTEL(INTEL_ATOM_SALTWELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
1153	VULNWL_INTEL(INTEL_ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT),
1154	VULNWL_INTEL(INTEL_ATOM_SALTWELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
1155	VULNWL_INTEL(INTEL_ATOM_BONNELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
1156	VULNWL_INTEL(INTEL_ATOM_BONNELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
1157
1158	VULNWL_INTEL(INTEL_ATOM_SILVERMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1159	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_D, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1160	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1161	VULNWL_INTEL(INTEL_ATOM_AIRMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1162	VULNWL_INTEL(INTEL_XEON_PHI_KNL, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1163	VULNWL_INTEL(INTEL_XEON_PHI_KNM, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1164
1165	VULNWL_INTEL(INTEL_CORE_YONAH, NO_SSB),
1166
1167	VULNWL_INTEL(INTEL_ATOM_SILVERMONT_MID2,NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | MSBDS_ONLY),
1168	VULNWL_INTEL(INTEL_ATOM_AIRMONT_NP, NO_SSB | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1169
1170	VULNWL_INTEL(INTEL_ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1171	VULNWL_INTEL(INTEL_ATOM_GOLDMONT_D, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO),
1172	VULNWL_INTEL(INTEL_ATOM_GOLDMONT_PLUS, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB),
1173
1174	/*
1175	* Technically, swapgs isn't serializing on AMD (despite it previously
1176	* being documented as such in the APM). But according to AMD, %gs is
1177	* updated non-speculatively, and the issuing of %gs-relative memory
1178	* operands will be blocked until the %gs update completes, which is
1179	* good enough for our purposes.
1180	*/
1181
1182	VULNWL_INTEL(INTEL_ATOM_TREMONT, NO_EIBRS_PBRSB),
1183	VULNWL_INTEL(INTEL_ATOM_TREMONT_L, NO_EIBRS_PBRSB),
1184	VULNWL_INTEL(INTEL_ATOM_TREMONT_D, NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB),
1185
1186	/* AMD Family 0xf - 0x12 */
1187	VULNWL_AMD(0x0f, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1188	VULNWL_AMD(0x10, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1189	VULNWL_AMD(0x11, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1190	VULNWL_AMD(0x12, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_BHI),
1191
1192	/* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */
1193	VULNWL_AMD(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1194	VULNWL_HYGON(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB | NO_BHI),
1195
1196	/* Zhaoxin Family 7 */
1197	VULNWL(CENTAUR, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1198	VULNWL(ZHAOXIN, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO | NO_BHI),
1199	{}
1200	};
1201
1202	#define VULNBL(vendor, family, model, blacklist) \
1203	X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, blacklist)
1204
1205	#define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \
1206	X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues)
1207
1208	#define VULNBL_INTEL_TYPE(vfm, cpu_type, issues) \
1209	X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues)
1210
1211	#define VULNBL_AMD(family, blacklist) \
1212	VULNBL(AMD, family, X86_MODEL_ANY, blacklist)
1213
1214	#define VULNBL_HYGON(family, blacklist) \
1215	VULNBL(HYGON, family, X86_MODEL_ANY, blacklist)
1216
1217	#define SRBDS BIT(0)
1218	/* CPU is affected by X86_BUG_MMIO_STALE_DATA */
1219	#define MMIO BIT(1)
1220	/* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */
1221	#define MMIO_SBDS BIT(2)
1222	/* CPU is affected by RETbleed, speculating where you would not expect it */
1223	#define RETBLEED BIT(3)
1224	/* CPU is affected by SMT (cross-thread) return predictions */
1225	#define SMT_RSB BIT(4)
1226	/* CPU is affected by SRSO */
1227	#define SRSO BIT(5)
1228	/* CPU is affected by GDS */
1229	#define GDS BIT(6)
1230	/* CPU is affected by Register File Data Sampling */
1231	#define RFDS BIT(7)
1232	/* CPU is affected by Indirect Target Selection */
1233	#define ITS BIT(8)
1234	/* CPU is affected by Indirect Target Selection, but guest-host isolation is not affected */
1235	#define ITS_NATIVE_ONLY BIT(9)
1236
1237	static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1238	VULNBL_INTEL_STEPS(INTEL_IVYBRIDGE, X86_STEP_MAX, SRBDS),
1239	VULNBL_INTEL_STEPS(INTEL_HASWELL, X86_STEP_MAX, SRBDS),
1240	VULNBL_INTEL_STEPS(INTEL_HASWELL_L, X86_STEP_MAX, SRBDS),
1241	VULNBL_INTEL_STEPS(INTEL_HASWELL_G, X86_STEP_MAX, SRBDS),
1242	VULNBL_INTEL_STEPS(INTEL_HASWELL_X, X86_STEP_MAX, MMIO),
1243	VULNBL_INTEL_STEPS(INTEL_BROADWELL_D, X86_STEP_MAX, MMIO),
1244	VULNBL_INTEL_STEPS(INTEL_BROADWELL_G, X86_STEP_MAX, SRBDS),
1245	VULNBL_INTEL_STEPS(INTEL_BROADWELL_X, X86_STEP_MAX, MMIO),
1246	VULNBL_INTEL_STEPS(INTEL_BROADWELL, X86_STEP_MAX, SRBDS),
1247	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X, 0x5, MMIO | RETBLEED | GDS),
1248	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_X, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS),
1249	VULNBL_INTEL_STEPS(INTEL_SKYLAKE_L, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS),
1250	VULNBL_INTEL_STEPS(INTEL_SKYLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS),
1251	VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L, 0xb, MMIO | RETBLEED | GDS | SRBDS),
1252	VULNBL_INTEL_STEPS(INTEL_KABYLAKE_L, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS | ITS),
1253	VULNBL_INTEL_STEPS(INTEL_KABYLAKE, 0xc, MMIO | RETBLEED | GDS | SRBDS),
1254	VULNBL_INTEL_STEPS(INTEL_KABYLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | SRBDS | ITS),
1255	VULNBL_INTEL_STEPS(INTEL_CANNONLAKE_L, X86_STEP_MAX, RETBLEED),
1256	VULNBL_INTEL_STEPS(INTEL_ICELAKE_L, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
1257	VULNBL_INTEL_STEPS(INTEL_ICELAKE_D, X86_STEP_MAX, MMIO | GDS | ITS | ITS_NATIVE_ONLY),
1258	VULNBL_INTEL_STEPS(INTEL_ICELAKE_X, X86_STEP_MAX, MMIO | GDS | ITS | ITS_NATIVE_ONLY),
1259	VULNBL_INTEL_STEPS(INTEL_COMETLAKE, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS),
1260	VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L, 0x0, MMIO | RETBLEED | ITS),
1261	VULNBL_INTEL_STEPS(INTEL_COMETLAKE_L, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED | GDS | ITS),
1262	VULNBL_INTEL_STEPS(INTEL_TIGERLAKE_L, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY),
1263	VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY),
1264	VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED),
1265	VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY),
1266	VULNBL_INTEL_TYPE(INTEL_ALDERLAKE, ATOM, RFDS),
1267	VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS),
1268	VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE, ATOM, RFDS),
1269	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS),
1270	VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS),
1271	VULNBL_INTEL_STEPS(INTEL_ATOM_GRACEMONT, X86_STEP_MAX, RFDS),
1272	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT, X86_STEP_MAX, MMIO | MMIO_SBDS | RFDS),
1273	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_D, X86_STEP_MAX, MMIO | RFDS),
1274	VULNBL_INTEL_STEPS(INTEL_ATOM_TREMONT_L, X86_STEP_MAX, MMIO | MMIO_SBDS | RFDS),
1275	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT, X86_STEP_MAX, RFDS),
1276	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_D, X86_STEP_MAX, RFDS),
1277	VULNBL_INTEL_STEPS(INTEL_ATOM_GOLDMONT_PLUS, X86_STEP_MAX, RFDS),
1278
1279	VULNBL_AMD(0x15, RETBLEED),
1280	VULNBL_AMD(0x16, RETBLEED),
1281	VULNBL_AMD(0x17, RETBLEED | SMT_RSB | SRSO),
1282	VULNBL_HYGON(0x18, RETBLEED | SMT_RSB | SRSO),
1283	VULNBL_AMD(0x19, SRSO),
1284	VULNBL_AMD(0x1a, SRSO),
1285	{}
1286	};
1287
1288	static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1289	{
1290	const struct x86_cpu_id *m = x86_match_cpu(match: table);
1291
1292	return m && !!(m->driver_data & which);
1293	}
1294
1295	u64 x86_read_arch_cap_msr(void)
1296	{
1297	u64 x86_arch_cap_msr = 0;
1298
1299	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1300	rdmsrq(MSR_IA32_ARCH_CAPABILITIES, x86_arch_cap_msr);
1301
1302	return x86_arch_cap_msr;
1303	}
1304
1305	static bool arch_cap_mmio_immune(u64 x86_arch_cap_msr)
1306	{
1307	return (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO &&
1308	x86_arch_cap_msr & ARCH_CAP_PSDP_NO &&
1309	x86_arch_cap_msr & ARCH_CAP_SBDR_SSDP_NO);
1310	}
1311
1312	static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr)
1313	{
1314	/* The "immunity" bit trumps everything else: */
1315	if (x86_arch_cap_msr & ARCH_CAP_RFDS_NO)
1316	return false;
1317
1318	/*
1319	* VMMs set ARCH_CAP_RFDS_CLEAR for processors not in the blacklist to
1320	* indicate that mitigation is needed because guest is running on a
1321	* vulnerable hardware or may migrate to such hardware:
1322	*/
1323	if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
1324	return true;
1325
1326	/* Only consult the blacklist when there is no enumeration: */
1327	return cpu_matches(table: cpu_vuln_blacklist, RFDS);
1328	}
1329
1330	static bool __init vulnerable_to_its(u64 x86_arch_cap_msr)
1331	{
1332	/* The "immunity" bit trumps everything else: */
1333	if (x86_arch_cap_msr & ARCH_CAP_ITS_NO)
1334	return false;
1335	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
1336	return false;
1337
1338	/* None of the affected CPUs have BHI_CTRL */
1339	if (boot_cpu_has(X86_FEATURE_BHI_CTRL))
1340	return false;
1341
1342	/*
1343	* If a VMM did not expose ITS_NO, assume that a guest could
1344	* be running on a vulnerable hardware or may migrate to such
1345	* hardware.
1346	*/
1347	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1348	return true;
1349
1350	if (cpu_matches(table: cpu_vuln_blacklist, ITS))
1351	return true;
1352
1353	return false;
1354	}
1355
1356	static struct x86_cpu_id cpu_latest_microcode[] = {
1357	#include "microcode/intel-ucode-defs.h"
1358	{}
1359	};
1360
1361	static bool __init cpu_has_old_microcode(void)
1362	{
1363	const struct x86_cpu_id *m = x86_match_cpu(match: cpu_latest_microcode);
1364
1365	/* Give unknown CPUs a pass: */
1366	if (!m) {
1367	/* Intel CPUs should be in the list. Warn if not: */
1368	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1369	pr_info("x86/CPU: Model not found in latest microcode list\n");
1370	return false;
1371	}
1372
1373	/*
1374	* Hosts usually lie to guests with a super high microcode
1375	* version. Just ignore what hosts tell guests:
1376	*/
1377	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1378	return false;
1379
1380	/* Consider all debug microcode to be old: */
1381	if (boot_cpu_data.microcode & BIT(31))
1382	return true;
1383
1384	/* Give new microcode a pass: */
1385	if (boot_cpu_data.microcode >= m->driver_data)
1386	return false;
1387
1388	/* Uh oh, too old: */
1389	return true;
1390	}
1391
1392	static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1393	{
1394	u64 x86_arch_cap_msr = x86_read_arch_cap_msr();
1395
1396	if (cpu_has_old_microcode()) {
1397	pr_warn("x86/CPU: Running old microcode\n");
1398	setup_force_cpu_bug(X86_BUG_OLD_MICROCODE);
1399	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1400	}
1401
1402	/* Set ITLB_MULTIHIT bug if cpu is not in the whitelist and not mitigated */
1403	if (!cpu_matches(table: cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1404	!(x86_arch_cap_msr & ARCH_CAP_PSCHANGE_MC_NO))
1405	setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1406
1407	if (cpu_matches(table: cpu_vuln_whitelist, NO_SPECULATION))
1408	return;
1409
1410	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1411
1412	if (!cpu_matches(table: cpu_vuln_whitelist, NO_SPECTRE_V2)) {
1413	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1414	setup_force_cpu_bug(X86_BUG_SPECTRE_V2_USER);
1415	}
1416
1417	if (!cpu_matches(table: cpu_vuln_whitelist, NO_SSB) &&
1418	!(x86_arch_cap_msr & ARCH_CAP_SSB_NO) &&
1419	!cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1420	setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1421
1422	/*
1423	* AMD's AutoIBRS is equivalent to Intel's eIBRS - use the Intel feature
1424	* flag and protect from vendor-specific bugs via the whitelist.
1425	*
1426	* Don't use AutoIBRS when SNP is enabled because it degrades host
1427	* userspace indirect branch performance.
1428	*/
1429	if ((x86_arch_cap_msr & ARCH_CAP_IBRS_ALL) ||
1430	(cpu_has(c, X86_FEATURE_AUTOIBRS) &&
1431	!cpu_feature_enabled(X86_FEATURE_SEV_SNP))) {
1432	setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1433	if (!cpu_matches(table: cpu_vuln_whitelist, NO_EIBRS_PBRSB) &&
1434	!(x86_arch_cap_msr & ARCH_CAP_PBRSB_NO))
1435	setup_force_cpu_bug(X86_BUG_EIBRS_PBRSB);
1436	}
1437
1438	if (!cpu_matches(table: cpu_vuln_whitelist, NO_MDS) &&
1439	!(x86_arch_cap_msr & ARCH_CAP_MDS_NO)) {
1440	setup_force_cpu_bug(X86_BUG_MDS);
1441	if (cpu_matches(table: cpu_vuln_whitelist, MSBDS_ONLY))
1442	setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1443	}
1444
1445	if (!cpu_matches(table: cpu_vuln_whitelist, NO_SWAPGS))
1446	setup_force_cpu_bug(X86_BUG_SWAPGS);
1447
1448	/*
1449	* When the CPU is not mitigated for TAA (TAA_NO=0) set TAA bug when:
1450	* - TSX is supported or
1451	* - TSX_CTRL is present
1452	*
1453	* TSX_CTRL check is needed for cases when TSX could be disabled before
1454	* the kernel boot e.g. kexec.
1455	* TSX_CTRL check alone is not sufficient for cases when the microcode
1456	* update is not present or running as guest that don't get TSX_CTRL.
1457	*/
1458	if (!(x86_arch_cap_msr & ARCH_CAP_TAA_NO) &&
1459	(cpu_has(c, X86_FEATURE_RTM) ||
1460	(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR)))
1461	setup_force_cpu_bug(X86_BUG_TAA);
1462
1463	/*
1464	* SRBDS affects CPUs which support RDRAND or RDSEED and are listed
1465	* in the vulnerability blacklist.
1466	*
1467	* Some of the implications and mitigation of Shared Buffers Data
1468	* Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as
1469	* SRBDS.
1470	*/
1471	if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1472	cpu_has(c, X86_FEATURE_RDSEED)) &&
1473	cpu_matches(table: cpu_vuln_blacklist, SRBDS | MMIO_SBDS))
1474	setup_force_cpu_bug(X86_BUG_SRBDS);
1475
1476	/*
1477	* Processor MMIO Stale Data bug enumeration
1478	*
1479	* Affected CPU list is generally enough to enumerate the vulnerability,
1480	* but for virtualization case check for ARCH_CAP MSR bits also, VMM may
1481	* not want the guest to enumerate the bug.
1482	*/
1483	if (!arch_cap_mmio_immune(x86_arch_cap_msr)) {
1484	if (cpu_matches(table: cpu_vuln_blacklist, MMIO))
1485	setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA);
1486	}
1487
1488	if (!cpu_has(c, X86_FEATURE_BTC_NO)) {
1489	if (cpu_matches(table: cpu_vuln_blacklist, RETBLEED) || (x86_arch_cap_msr & ARCH_CAP_RSBA))
1490	setup_force_cpu_bug(X86_BUG_RETBLEED);
1491	}
1492
1493	if (cpu_matches(table: cpu_vuln_blacklist, SMT_RSB))
1494	setup_force_cpu_bug(X86_BUG_SMT_RSB);
1495
1496	if (!cpu_has(c, X86_FEATURE_SRSO_NO)) {
1497	if (cpu_matches(table: cpu_vuln_blacklist, SRSO))
1498	setup_force_cpu_bug(X86_BUG_SRSO);
1499	}
1500
1501	/*
1502	* Check if CPU is vulnerable to GDS. If running in a virtual machine on
1503	* an affected processor, the VMM may have disabled the use of GATHER by
1504	* disabling AVX2. The only way to do this in HW is to clear XCR0[2],
1505	* which means that AVX will be disabled.
1506	*/
1507	if (cpu_matches(table: cpu_vuln_blacklist, GDS) && !(x86_arch_cap_msr & ARCH_CAP_GDS_NO) &&
1508	boot_cpu_has(X86_FEATURE_AVX))
1509	setup_force_cpu_bug(X86_BUG_GDS);
1510
1511	if (vulnerable_to_rfds(x86_arch_cap_msr))
1512	setup_force_cpu_bug(X86_BUG_RFDS);
1513
1514	/*
1515	* Intel parts with eIBRS are vulnerable to BHI attacks. Parts with
1516	* BHI_NO still need to use the BHI mitigation to prevent Intra-mode
1517	* attacks. When virtualized, eIBRS could be hidden, assume vulnerable.
1518	*/
1519	if (!cpu_matches(table: cpu_vuln_whitelist, NO_BHI) &&
1520	(boot_cpu_has(X86_FEATURE_IBRS_ENHANCED) ||
1521	boot_cpu_has(X86_FEATURE_HYPERVISOR)))
1522	setup_force_cpu_bug(X86_BUG_BHI);
1523
1524	if (cpu_has(c, X86_FEATURE_AMD_IBPB) && !cpu_has(c, X86_FEATURE_AMD_IBPB_RET))
1525	setup_force_cpu_bug(X86_BUG_IBPB_NO_RET);
1526
1527	if (vulnerable_to_its(x86_arch_cap_msr)) {
1528	setup_force_cpu_bug(X86_BUG_ITS);
1529	if (cpu_matches(table: cpu_vuln_blacklist, ITS_NATIVE_ONLY))
1530	setup_force_cpu_bug(X86_BUG_ITS_NATIVE_ONLY);
1531	}
1532
1533	if (cpu_matches(table: cpu_vuln_whitelist, NO_MELTDOWN))
1534	return;
1535
1536	/* Rogue Data Cache Load? No! */
1537	if (x86_arch_cap_msr & ARCH_CAP_RDCL_NO)
1538	return;
1539
1540	setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1541
1542	if (cpu_matches(table: cpu_vuln_whitelist, NO_L1TF))
1543	return;
1544
1545	setup_force_cpu_bug(X86_BUG_L1TF);
1546	}
1547
1548	/*
1549	* The NOPL instruction is supposed to exist on all CPUs of family >= 6;
1550	* unfortunately, that's not true in practice because of early VIA
1551	* chips and (more importantly) broken virtualizers that are not easy
1552	* to detect. In the latter case it doesn't even *fail* reliably, so
1553	* probing for it doesn't even work. Disable it completely on 32-bit
1554	* unless we can find a reliable way to detect all the broken cases.
1555	* Enable it explicitly on 64-bit for non-constant inputs of cpu_has().
1556	*/
1557	static void detect_nopl(void)
1558	{
1559	#ifdef CONFIG_X86_32
1560	setup_clear_cpu_cap(X86_FEATURE_NOPL);
1561	#else
1562	setup_force_cpu_cap(X86_FEATURE_NOPL);
1563	#endif
1564	}
1565
1566	static inline bool parse_set_clear_cpuid(char *arg, bool set)
1567	{
1568	char *opt;
1569	int taint = 0;
1570
1571	while (arg) {
1572	bool found __maybe_unused = false;
1573	unsigned int bit;
1574
1575	opt = strsep(&arg, ",");
1576
1577	/*
1578	* Handle naked numbers first for feature flags which don't
1579	* have names. It doesn't make sense for a bug not to have a
1580	* name so don't handle bug flags here.
1581	*/
1582	if (!kstrtouint(s: opt, base: 10, res: &bit)) {
1583	if (bit < NCAPINTS * 32) {
1584
1585	if (set) {
1586	pr_warn("setcpuid: force-enabling CPU feature flag:");
1587	setup_force_cpu_cap(bit);
1588	} else {
1589	pr_warn("clearcpuid: force-disabling CPU feature flag:");
1590	setup_clear_cpu_cap(bit);
1591	}
1592	/* empty-string, i.e., ""-defined feature flags */
1593	if (!x86_cap_flags[bit])
1594	pr_cont(" %d:%d\n", bit >> 5, bit & 31);
1595	else
1596	pr_cont(" %s\n", x86_cap_flags[bit]);
1597
1598	taint++;
1599	}
1600	/*
1601	* The assumption is that there are no feature names with only
1602	* numbers in the name thus go to the next argument.
1603	*/
1604	continue;
1605	}
1606
1607	for (bit = 0; bit < 32 * (NCAPINTS + NBUGINTS); bit++) {
1608	const char *flag;
1609	const char *kind;
1610
1611	if (bit < 32 * NCAPINTS) {
1612	flag = x86_cap_flags[bit];
1613	kind = "feature";
1614	} else {
1615	kind = "bug";
1616	flag = x86_bug_flags[bit - (32 * NCAPINTS)];
1617	}
1618
1619	if (!flag)
1620	continue;
1621
1622	if (strcmp(flag, opt))
1623	continue;
1624
1625	if (set) {
1626	pr_warn("setcpuid: force-enabling CPU %s flag: %s\n",
1627	kind, flag);
1628	setup_force_cpu_cap(bit);
1629	} else {
1630	pr_warn("clearcpuid: force-disabling CPU %s flag: %s\n",
1631	kind, flag);
1632	setup_clear_cpu_cap(bit);
1633	}
1634	taint++;
1635	found = true;
1636	break;
1637	}
1638
1639	if (!found)
1640	pr_warn("%s: unknown CPU flag: %s", set ? "setcpuid" : "clearcpuid", opt);
1641	}
1642
1643	return taint;
1644	}
1645
1646
1647	/*
1648	* We parse cpu parameters early because fpu__init_system() is executed
1649	* before parse_early_param().
1650	*/
1651	static void __init cpu_parse_early_param(void)
1652	{
1653	bool cpuid_taint = false;
1654	char arg[128];
1655	int arglen;
1656
1657	#ifdef CONFIG_X86_32
1658	if (cmdline_find_option_bool(boot_command_line, "no387"))
1659	#ifdef CONFIG_MATH_EMULATION
1660	setup_clear_cpu_cap(X86_FEATURE_FPU);
1661	#else
1662	pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1663	#endif
1664
1665	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1666	setup_clear_cpu_cap(X86_FEATURE_FXSR);
1667	#endif
1668
1669	if (cmdline_find_option_bool(cmdline_ptr: boot_command_line, option: "noxsave"))
1670	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1671
1672	if (cmdline_find_option_bool(cmdline_ptr: boot_command_line, option: "noxsaveopt"))
1673	setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1674
1675	if (cmdline_find_option_bool(cmdline_ptr: boot_command_line, option: "noxsaves"))
1676	setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1677
1678	if (cmdline_find_option_bool(cmdline_ptr: boot_command_line, option: "nousershstk"))
1679	setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1680
1681	/* Minimize the gap between FRED is available and available but disabled. */
1682	arglen = cmdline_find_option(cmdline_ptr: boot_command_line, option: "fred", buffer: arg, bufsize: sizeof(arg));
1683	if (arglen != 2 || strncmp(arg, "on", 2))
1684	setup_clear_cpu_cap(X86_FEATURE_FRED);
1685
1686	arglen = cmdline_find_option(cmdline_ptr: boot_command_line, option: "clearcpuid", buffer: arg, bufsize: sizeof(arg));
1687	if (arglen > 0)
1688	cpuid_taint |= parse_set_clear_cpuid(arg, set: false);
1689
1690	arglen = cmdline_find_option(cmdline_ptr: boot_command_line, option: "setcpuid", buffer: arg, bufsize: sizeof(arg));
1691	if (arglen > 0)
1692	cpuid_taint |= parse_set_clear_cpuid(arg, set: true);
1693
1694	if (cpuid_taint) {
1695	pr_warn("!!! setcpuid=/clearcpuid= in use, this is for TESTING ONLY, may break things horribly. Tainting kernel.\n");
1696	add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1697	}
1698	}
1699
1700	/*
1701	* Do minimum CPU detection early.
1702	* Fields really needed: vendor, cpuid_level, family, model, mask,
1703	* cache alignment.
1704	* The others are not touched to avoid unwanted side effects.
1705	*
1706	* WARNING: this function is only called on the boot CPU. Don't add code
1707	* here that is supposed to run on all CPUs.
1708	*/
1709	static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1710	{
1711	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1712	c->extended_cpuid_level = 0;
1713
1714	if (!cpuid_feature())
1715	identify_cpu_without_cpuid(c);
1716
1717	/* cyrix could have cpuid enabled via c_identify()*/
1718	if (cpuid_feature()) {
1719	cpu_detect(c);
1720	get_cpu_vendor(c);
1721	intel_unlock_cpuid_leafs(c);
1722	get_cpu_cap(c);
1723	setup_force_cpu_cap(X86_FEATURE_CPUID);
1724	get_cpu_address_sizes(c);
1725	cpu_parse_early_param();
1726
1727	cpu_init_topology(c);
1728
1729	if (this_cpu->c_early_init)
1730	this_cpu->c_early_init(c);
1731
1732	c->cpu_index = 0;
1733	filter_cpuid_features(c, warn: false);
1734	check_cpufeature_deps(c);
1735
1736	if (this_cpu->c_bsp_init)
1737	this_cpu->c_bsp_init(c);
1738	} else {
1739	setup_clear_cpu_cap(X86_FEATURE_CPUID);
1740	get_cpu_address_sizes(c);
1741	cpu_init_topology(c);
1742	}
1743
1744	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1745
1746	cpu_set_bug_bits(c);
1747
1748	sld_setup(c);
1749
1750	#ifdef CONFIG_X86_32
1751	/*
1752	* Regardless of whether PCID is enumerated, the SDM says
1753	* that it can't be enabled in 32-bit mode.
1754	*/
1755	setup_clear_cpu_cap(X86_FEATURE_PCID);
1756	#endif
1757
1758	/*
1759	* Later in the boot process pgtable_l5_enabled() relies on
1760	* cpu_feature_enabled(X86_FEATURE_LA57). If 5-level paging is not
1761	* enabled by this point we need to clear the feature bit to avoid
1762	* false-positives at the later stage.
1763	*
1764	* pgtable_l5_enabled() can be false here for several reasons:
1765	* - 5-level paging is disabled compile-time;
1766	* - it's 32-bit kernel;
1767	* - machine doesn't support 5-level paging;
1768	* - user specified 'no5lvl' in kernel command line.
1769	*/
1770	if (!pgtable_l5_enabled())
1771	setup_clear_cpu_cap(X86_FEATURE_LA57);
1772
1773	detect_nopl();
1774	}
1775
1776	void __init init_cpu_devs(void)
1777	{
1778	const struct cpu_dev *const *cdev;
1779	int count = 0;
1780
1781	for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1782	const struct cpu_dev *cpudev = *cdev;
1783
1784	if (count >= X86_VENDOR_NUM)
1785	break;
1786	cpu_devs[count] = cpudev;
1787	count++;
1788	}
1789	}
1790
1791	void __init early_cpu_init(void)
1792	{
1793	#ifdef CONFIG_PROCESSOR_SELECT
1794	unsigned int i, j;
1795
1796	pr_info("KERNEL supported cpus:\n");
1797	#endif
1798
1799	init_cpu_devs();
1800
1801	#ifdef CONFIG_PROCESSOR_SELECT
1802	for (i = 0; i < X86_VENDOR_NUM && cpu_devs[i]; i++) {
1803	for (j = 0; j < 2; j++) {
1804	if (!cpu_devs[i]->c_ident[j])
1805	continue;
1806	pr_info(" %s %s\n", cpu_devs[i]->c_vendor,
1807	cpu_devs[i]->c_ident[j]);
1808	}
1809	}
1810	#endif
1811
1812	early_identify_cpu(c: &boot_cpu_data);
1813	}
1814
1815	static bool detect_null_seg_behavior(void)
1816	{
1817	/*
1818	* Empirically, writing zero to a segment selector on AMD does
1819	* not clear the base, whereas writing zero to a segment
1820	* selector on Intel does clear the base. Intel's behavior
1821	* allows slightly faster context switches in the common case
1822	* where GS is unused by the prev and next threads.
1823	*
1824	* Since neither vendor documents this anywhere that I can see,
1825	* detect it directly instead of hard-coding the choice by
1826	* vendor.
1827	*
1828	* I've designated AMD's behavior as the "bug" because it's
1829	* counterintuitive and less friendly.
1830	*/
1831
1832	unsigned long old_base, tmp;
1833	rdmsrq(MSR_FS_BASE, old_base);
1834	wrmsrq(MSR_FS_BASE, val: 1);
1835	loadsegment(fs, 0);
1836	rdmsrq(MSR_FS_BASE, tmp);
1837	wrmsrq(MSR_FS_BASE, val: old_base);
1838	return tmp == 0;
1839	}
1840
1841	void check_null_seg_clears_base(struct cpuinfo_x86 *c)
1842	{
1843	/* BUG_NULL_SEG is only relevant with 64bit userspace */
1844	if (!IS_ENABLED(CONFIG_X86_64))
1845	return;
1846
1847	if (cpu_has(c, X86_FEATURE_NULL_SEL_CLR_BASE))
1848	return;
1849
1850	/*
1851	* CPUID bit above wasn't set. If this kernel is still running
1852	* as a HV guest, then the HV has decided not to advertize
1853	* that CPUID bit for whatever reason. For example, one
1854	* member of the migration pool might be vulnerable. Which
1855	* means, the bug is present: set the BUG flag and return.
1856	*/
1857	if (cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1858	set_cpu_bug(c, X86_BUG_NULL_SEG);
1859	return;
1860	}
1861
1862	/*
1863	* Zen2 CPUs also have this behaviour, but no CPUID bit.
1864	* 0x18 is the respective family for Hygon.
1865	*/
1866	if ((c->x86 == 0x17 || c->x86 == 0x18) &&
1867	detect_null_seg_behavior())
1868	return;
1869
1870	/* All the remaining ones are affected */
1871	set_cpu_bug(c, X86_BUG_NULL_SEG);
1872	}
1873
1874	static void generic_identify(struct cpuinfo_x86 *c)
1875	{
1876	c->extended_cpuid_level = 0;
1877
1878	if (!cpuid_feature())
1879	identify_cpu_without_cpuid(c);
1880
1881	/* cyrix could have cpuid enabled via c_identify()*/
1882	if (!cpuid_feature())
1883	return;
1884
1885	cpu_detect(c);
1886
1887	get_cpu_vendor(c);
1888	intel_unlock_cpuid_leafs(c);
1889	get_cpu_cap(c);
1890
1891	get_cpu_address_sizes(c);
1892
1893	get_model_name(c); /* Default name */
1894
1895	/*
1896	* ESPFIX is a strange bug. All real CPUs have it. Paravirt
1897	* systems that run Linux at CPL > 0 may or may not have the
1898	* issue, but, even if they have the issue, there's absolutely
1899	* nothing we can do about it because we can't use the real IRET
1900	* instruction.
1901	*
1902	* NB: For the time being, only 32-bit kernels support
1903	* X86_BUG_ESPFIX as such. 64-bit kernels directly choose
1904	* whether to apply espfix using paravirt hooks. If any
1905	* non-paravirt system ever shows up that does *not* have the
1906	* ESPFIX issue, we can change this.
1907	*/
1908	#ifdef CONFIG_X86_32
1909	set_cpu_bug(c, X86_BUG_ESPFIX);
1910	#endif
1911	}
1912
1913	/*
1914	* This does the hard work of actually picking apart the CPU stuff...
1915	*/
1916	static void identify_cpu(struct cpuinfo_x86 *c)
1917	{
1918	int i;
1919
1920	c->loops_per_jiffy = loops_per_jiffy;
1921	c->x86_cache_size = 0;
1922	c->x86_vendor = X86_VENDOR_UNKNOWN;
1923	c->x86_model = c->x86_stepping = 0; /* So far unknown... */
1924	c->x86_vendor_id[0] = '\0'; /* Unset */
1925	c->x86_model_id[0] = '\0'; /* Unset */
1926	#ifdef CONFIG_X86_64
1927	c->x86_clflush_size = 64;
1928	c->x86_phys_bits = 36;
1929	c->x86_virt_bits = 48;
1930	#else
1931	c->cpuid_level = -1; /* CPUID not detected */
1932	c->x86_clflush_size = 32;
1933	c->x86_phys_bits = 32;
1934	c->x86_virt_bits = 32;
1935	#endif
1936	c->x86_cache_alignment = c->x86_clflush_size;
1937	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1938	#ifdef CONFIG_X86_VMX_FEATURE_NAMES
1939	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1940	#endif
1941
1942	generic_identify(c);
1943
1944	cpu_parse_topology(c);
1945
1946	if (this_cpu->c_identify)
1947	this_cpu->c_identify(c);
1948
1949	/* Clear/Set all flags overridden by options, after probe */
1950	apply_forced_caps(c);
1951
1952	/*
1953	* Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
1954	* Hygon will clear it in ->c_init() below.
1955	*/
1956	set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1957
1958	/*
1959	* Vendor-specific initialization. In this section we
1960	* canonicalize the feature flags, meaning if there are
1961	* features a certain CPU supports which CPUID doesn't
1962	* tell us, CPUID claiming incorrect flags, or other bugs,
1963	* we handle them here.
1964	*
1965	* At the end of this section, c->x86_capability better
1966	* indicate the features this CPU genuinely supports!
1967	*/
1968	if (this_cpu->c_init)
1969	this_cpu->c_init(c);
1970
1971	bus_lock_init();
1972
1973	/* Disable the PN if appropriate */
1974	squash_the_stupid_serial_number(c);
1975
1976	/* Set up SMEP/SMAP/UMIP */
1977	setup_smep(c);
1978	setup_smap(c);
1979	setup_umip(c);
1980
1981	/* Enable FSGSBASE instructions if available. */
1982	if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1983	cr4_set_bits(X86_CR4_FSGSBASE);
1984	elf_hwcap2 |= HWCAP2_FSGSBASE;
1985	}
1986
1987	/*
1988	* The vendor-specific functions might have changed features.
1989	* Now we do "generic changes."
1990	*/
1991
1992	/* Filter out anything that depends on CPUID levels we don't have */
1993	filter_cpuid_features(c, warn: true);
1994
1995	/* Check for unmet dependencies based on the CPUID dependency table */
1996	check_cpufeature_deps(c);
1997
1998	/* If the model name is still unset, do table lookup. */
1999	if (!c->x86_model_id[0]) {
2000	const char *p;
2001	p = table_lookup_model(c);
2002	if (p)
2003	strcpy(p: c->x86_model_id, q: p);
2004	else
2005	/* Last resort... */
2006	sprintf(buf: c->x86_model_id, fmt: "%02x/%02x",
2007	c->x86, c->x86_model);
2008	}
2009
2010	x86_init_rdrand(c);
2011	setup_pku(c);
2012	setup_cet(c);
2013
2014	/*
2015	* Clear/Set all flags overridden by options, need do it
2016	* before following smp all cpus cap AND.
2017	*/
2018	apply_forced_caps(c);
2019
2020	/*
2021	* On SMP, boot_cpu_data holds the common feature set between
2022	* all CPUs; so make sure that we indicate which features are
2023	* common between the CPUs. The first time this routine gets
2024	* executed, c == &boot_cpu_data.
2025	*/
2026	if (c != &boot_cpu_data) {
2027	/* AND the already accumulated flags with these */
2028	for (i = 0; i < NCAPINTS; i++)
2029	boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
2030
2031	/* OR, i.e. replicate the bug flags */
2032	for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
2033	c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
2034	}
2035
2036	ppin_init(c);
2037
2038	/* Init Machine Check Exception if available. */
2039	mcheck_cpu_init(c);
2040
2041	numa_add_cpu(smp_processor_id());
2042	}
2043
2044	/*
2045	* Set up the CPU state needed to execute SYSENTER/SYSEXIT instructions
2046	* on 32-bit kernels:
2047	*/
2048	#ifdef CONFIG_X86_32
2049	void enable_sep_cpu(void)
2050	{
2051	struct tss_struct *tss;
2052	int cpu;
2053
2054	if (!boot_cpu_has(X86_FEATURE_SEP))
2055	return;
2056
2057	cpu = get_cpu();
2058	tss = &per_cpu(cpu_tss_rw, cpu);
2059
2060	/*
2061	* We cache MSR_IA32_SYSENTER_CS's value in the TSS's ss1 field --
2062	* see the big comment in struct x86_hw_tss's definition.
2063	*/
2064
2065	tss->x86_tss.ss1 = __KERNEL_CS;
2066	wrmsrq(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1);
2067	wrmsrq(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1));
2068	wrmsrq(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32);
2069
2070	put_cpu();
2071	}
2072	#endif
2073
2074	static __init void identify_boot_cpu(void)
2075	{
2076	identify_cpu(c: &boot_cpu_data);
2077	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
2078	pr_info("CET detected: Indirect Branch Tracking enabled\n");
2079	#ifdef CONFIG_X86_32
2080	enable_sep_cpu();
2081	#endif
2082	cpu_detect_tlb(c: &boot_cpu_data);
2083	setup_cr_pinning();
2084
2085	tsx_init();
2086	tdx_init();
2087	lkgs_init();
2088	}
2089
2090	void identify_secondary_cpu(unsigned int cpu)
2091	{
2092	struct cpuinfo_x86 *c = &cpu_data(cpu);
2093
2094	/* Copy boot_cpu_data only on the first bringup */
2095	if (!c->initialized)
2096	*c = boot_cpu_data;
2097	c->cpu_index = cpu;
2098
2099	identify_cpu(c);
2100	#ifdef CONFIG_X86_32
2101	enable_sep_cpu();
2102	#endif
2103	x86_spec_ctrl_setup_ap();
2104	update_srbds_msr();
2105	if (boot_cpu_has_bug(X86_BUG_GDS))
2106	update_gds_msr();
2107
2108	tsx_ap_init();
2109	c->initialized = true;
2110	}
2111
2112	void print_cpu_info(struct cpuinfo_x86 *c)
2113	{
2114	const char *vendor = NULL;
2115
2116	if (c->x86_vendor < X86_VENDOR_NUM) {
2117	vendor = this_cpu->c_vendor;
2118	} else {
2119	if (c->cpuid_level >= 0)
2120	vendor = c->x86_vendor_id;
2121	}
2122
2123	if (vendor && !strstr(c->x86_model_id, vendor))
2124	pr_cont("%s ", vendor);
2125
2126	if (c->x86_model_id[0])
2127	pr_cont("%s", c->x86_model_id);
2128	else
2129	pr_cont("%d86", c->x86);
2130
2131	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
2132
2133	if (c->x86_stepping || c->cpuid_level >= 0)
2134	pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
2135	else
2136	pr_cont(")\n");
2137	}
2138
2139	/*
2140	* clearcpuid= and setcpuid= were already parsed in cpu_parse_early_param().
2141	* These dummy functions prevent them from becoming an environment variable for
2142	* init.
2143	*/
2144
2145	static __init int setup_clearcpuid(char *arg)
2146	{
2147	return 1;
2148	}
2149	__setup("clearcpuid=", setup_clearcpuid);
2150
2151	static __init int setup_setcpuid(char *arg)
2152	{
2153	return 1;
2154	}
2155	__setup("setcpuid=", setup_setcpuid);
2156
2157	DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
2158	EXPORT_PER_CPU_SYMBOL(current_task);
2159	EXPORT_PER_CPU_SYMBOL(const_current_task);
2160
2161	DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
2162	EXPORT_PER_CPU_SYMBOL(__preempt_count);
2163
2164	DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2165
2166	#ifdef CONFIG_X86_64
2167	/*
2168	* Note: Do not make this dependant on CONFIG_MITIGATION_CALL_DEPTH_TRACKING
2169	* so that this space is reserved in the hot cache section even when the
2170	* mitigation is disabled.
2171	*/
2172	DEFINE_PER_CPU_CACHE_HOT(u64, __x86_call_depth);
2173	EXPORT_PER_CPU_SYMBOL(__x86_call_depth);
2174
2175	static void wrmsrq_cstar(unsigned long val)
2176	{
2177	/*
2178	* Intel CPUs do not support 32-bit SYSCALL. Writing to MSR_CSTAR
2179	* is so far ignored by the CPU, but raises a #VE trap in a TDX
2180	* guest. Avoid the pointless write on all Intel CPUs.
2181	*/
2182	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
2183	wrmsrq(MSR_CSTAR, val);
2184	}
2185
2186	static inline void idt_syscall_init(void)
2187	{
2188	wrmsrq(MSR_LSTAR, val: (unsigned long)entry_SYSCALL_64);
2189
2190	if (ia32_enabled()) {
2191	wrmsrq_cstar(val: (unsigned long)entry_SYSCALL_compat);
2192	/*
2193	* This only works on Intel CPUs.
2194	* On AMD CPUs these MSRs are 32-bit, CPU truncates MSR_IA32_SYSENTER_EIP.
2195	* This does not cause SYSENTER to jump to the wrong location, because
2196	* AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
2197	*/
2198	wrmsrq_safe(MSR_IA32_SYSENTER_CS, val: (u64)__KERNEL_CS);
2199	wrmsrq_safe(MSR_IA32_SYSENTER_ESP,
2200	val: (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
2201	wrmsrq_safe(MSR_IA32_SYSENTER_EIP, val: (u64)entry_SYSENTER_compat);
2202	} else {
2203	wrmsrq_cstar(val: (unsigned long)entry_SYSCALL32_ignore);
2204	wrmsrq_safe(MSR_IA32_SYSENTER_CS, val: (u64)GDT_ENTRY_INVALID_SEG);
2205	wrmsrq_safe(MSR_IA32_SYSENTER_ESP, val: 0ULL);
2206	wrmsrq_safe(MSR_IA32_SYSENTER_EIP, val: 0ULL);
2207	}
2208
2209	/*
2210	* Flags to clear on syscall; clear as much as possible
2211	* to minimize user space-kernel interference.
2212	*/
2213	wrmsrq(MSR_SYSCALL_MASK,
2214	X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
2215	X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
2216	X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
2217	X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
2218	X86_EFLAGS_AC|X86_EFLAGS_ID);
2219	}
2220
2221	/* May not be marked __init: used by software suspend */
2222	void syscall_init(void)
2223	{
2224	/* The default user and kernel segments */
2225	wrmsr(MSR_STAR, low: 0, high: (__USER32_CS << 16) | __KERNEL_CS);
2226
2227	/*
2228	* Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
2229	* SYSENTER MSRs for FRED, because FRED uses the ring 3 FRED
2230	* entrypoint for SYSCALL and SYSENTER, and ERETU is the only legit
2231	* instruction to return to ring 3 (both sysexit and sysret cause
2232	* #UD when FRED is enabled).
2233	*/
2234	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2235	idt_syscall_init();
2236	}
2237	#endif /* CONFIG_X86_64 */
2238
2239	#ifdef CONFIG_STACKPROTECTOR
2240	DEFINE_PER_CPU_CACHE_HOT(unsigned long, __stack_chk_guard);
2241	#ifndef CONFIG_SMP
2242	EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
2243	#endif
2244	#endif
2245
2246	/*
2247	* Clear all 6 debug registers:
2248	*/
2249	static void clear_all_debug_regs(void)
2250	{
2251	int i;
2252
2253	for (i = 0; i < 8; i++) {
2254	/* Ignore db4, db5 */
2255	if ((i == 4) || (i == 5))
2256	continue;
2257
2258	set_debugreg(val: 0, reg: i);
2259	}
2260	}
2261
2262	#ifdef CONFIG_KGDB
2263	/*
2264	* Restore debug regs if using kgdbwait and you have a kernel debugger
2265	* connection established.
2266	*/
2267	static void dbg_restore_debug_regs(void)
2268	{
2269	if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
2270	arch_kgdb_ops.correct_hw_break();
2271	}
2272	#else /* ! CONFIG_KGDB */
2273	#define dbg_restore_debug_regs()
2274	#endif /* ! CONFIG_KGDB */
2275
2276	static inline void setup_getcpu(int cpu)
2277	{
2278	unsigned long cpudata = vdso_encode_cpunode(cpu, node: early_cpu_to_node(cpu));
2279	struct desc_struct d = { };
2280
2281	if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
2282	wrmsrq(MSR_TSC_AUX, val: cpudata);
2283
2284	/* Store CPU and node number in limit. */
2285	d.limit0 = cpudata;
2286	d.limit1 = cpudata >> 16;
2287
2288	d.type = 5; /* RO data, expand down, accessed */
2289	d.dpl = 3; /* Visible to user code */
2290	d.s = 1; /* Not a system segment */
2291	d.p = 1; /* Present */
2292	d.d = 1; /* 32-bit */
2293
2294	write_gdt_entry(dt: get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, desc: &d, type: DESCTYPE_S);
2295	}
2296
2297	#ifdef CONFIG_X86_64
2298	static inline void tss_setup_ist(struct tss_struct *tss)
2299	{
2300	/* Set up the per-CPU TSS IST stacks */
2301	tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
2302	tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
2303	tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
2304	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
2305	/* Only mapped when SEV-ES is active */
2306	tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
2307	}
2308	#else /* CONFIG_X86_64 */
2309	static inline void tss_setup_ist(struct tss_struct *tss) { }
2310	#endif /* !CONFIG_X86_64 */
2311
2312	static inline void tss_setup_io_bitmap(struct tss_struct *tss)
2313	{
2314	tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
2315
2316	#ifdef CONFIG_X86_IOPL_IOPERM
2317	tss->io_bitmap.prev_max = 0;
2318	tss->io_bitmap.prev_sequence = 0;
2319	memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
2320	/*
2321	* Invalidate the extra array entry past the end of the all
2322	* permission bitmap as required by the hardware.
2323	*/
2324	tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
2325	#endif
2326	}
2327
2328	/*
2329	* Setup everything needed to handle exceptions from the IDT, including the IST
2330	* exceptions which use paranoid_entry().
2331	*/
2332	void cpu_init_exception_handling(bool boot_cpu)
2333	{
2334	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
2335	int cpu = raw_smp_processor_id();
2336
2337	/* paranoid_entry() gets the CPU number from the GDT */
2338	setup_getcpu(cpu);
2339
2340	/* For IDT mode, IST vectors need to be set in TSS. */
2341	if (!cpu_feature_enabled(X86_FEATURE_FRED))
2342	tss_setup_ist(tss);
2343	tss_setup_io_bitmap(tss);
2344	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
2345
2346	load_TR_desc();
2347
2348	/* GHCB needs to be setup to handle #VC. */
2349	setup_ghcb();
2350
2351	if (cpu_feature_enabled(X86_FEATURE_FRED)) {
2352	/* The boot CPU has enabled FRED during early boot */
2353	if (!boot_cpu)
2354	cpu_init_fred_exceptions();
2355
2356	cpu_init_fred_rsps();
2357	} else {
2358	load_current_idt();
2359	}
2360	}
2361
2362	void __init cpu_init_replace_early_idt(void)
2363	{
2364	if (cpu_feature_enabled(X86_FEATURE_FRED))
2365	cpu_init_fred_exceptions();
2366	else
2367	idt_setup_early_pf();
2368	}
2369
2370	/*
2371	* cpu_init() initializes state that is per-CPU. Some data is already
2372	* initialized (naturally) in the bootstrap process, such as the GDT. We
2373	* reload it nevertheless, this function acts as a 'CPU state barrier',
2374	* nothing should get across.
2375	*/
2376	void cpu_init(void)
2377	{
2378	struct task_struct *cur = current;
2379	int cpu = raw_smp_processor_id();
2380
2381	#ifdef CONFIG_NUMA
2382	if (this_cpu_read(numa_node) == 0 &&
2383	early_cpu_to_node(cpu) != NUMA_NO_NODE)
2384	set_numa_node(early_cpu_to_node(cpu));
2385	#endif
2386	pr_debug("Initializing CPU#%d\n", cpu);
2387
2388	if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
2389	boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
2390	cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
2391
2392	if (IS_ENABLED(CONFIG_X86_64)) {
2393	loadsegment(fs, 0);
2394	memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
2395	syscall_init();
2396
2397	wrmsrq(MSR_FS_BASE, val: 0);
2398	wrmsrq(MSR_KERNEL_GS_BASE, val: 0);
2399	barrier();
2400
2401	x2apic_setup();
2402
2403	intel_posted_msi_init();
2404	}
2405
2406	mmgrab(mm: &init_mm);
2407	cur->active_mm = &init_mm;
2408	BUG_ON(cur->mm);
2409	initialize_tlbstate_and_flush();
2410	enter_lazy_tlb(mm: &init_mm, tsk: cur);
2411
2412	/*
2413	* sp0 points to the entry trampoline stack regardless of what task
2414	* is running.
2415	*/
2416	load_sp0(sp0: (unsigned long)(cpu_entry_stack(cpu) + 1));
2417
2418	load_mm_ldt(mm: &init_mm);
2419
2420	clear_all_debug_regs();
2421	dbg_restore_debug_regs();
2422
2423	doublefault_init_cpu_tss();
2424
2425	if (is_uv_system())
2426	uv_cpu_init();
2427
2428	load_fixmap_gdt(cpu);
2429	}
2430
2431	#ifdef CONFIG_MICROCODE_LATE_LOADING
2432	/**
2433	* store_cpu_caps() - Store a snapshot of CPU capabilities
2434	* @curr_info: Pointer where to store it
2435	*
2436	* Returns: None
2437	*/
2438	void store_cpu_caps(struct cpuinfo_x86 *curr_info)
2439	{
2440	/* Reload CPUID max function as it might've changed. */
2441	curr_info->cpuid_level = cpuid_eax(op: 0);
2442
2443	/* Copy all capability leafs and pick up the synthetic ones. */
2444	memcpy(&curr_info->x86_capability, &boot_cpu_data.x86_capability,
2445	sizeof(curr_info->x86_capability));
2446
2447	/* Get the hardware CPUID leafs */
2448	get_cpu_cap(c: curr_info);
2449	}
2450
2451	/**
2452	* microcode_check() - Check if any CPU capabilities changed after an update.
2453	* @prev_info: CPU capabilities stored before an update.
2454	*
2455	* The microcode loader calls this upon late microcode load to recheck features,
2456	* only when microcode has been updated. Caller holds and CPU hotplug lock.
2457	*
2458	* Return: None
2459	*/
2460	void microcode_check(struct cpuinfo_x86 *prev_info)
2461	{
2462	struct cpuinfo_x86 curr_info;
2463
2464	perf_check_microcode();
2465
2466	amd_check_microcode();
2467
2468	store_cpu_caps(curr_info: &curr_info);
2469
2470	if (!memcmp(p: &prev_info->x86_capability, q: &curr_info.x86_capability,
2471	size: sizeof(prev_info->x86_capability)))
2472	return;
2473
2474	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2475	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2476	}
2477	#endif
2478
2479	/*
2480	* Invoked from core CPU hotplug code after hotplug operations
2481	*/
2482	void arch_smt_update(void)
2483	{
2484	/* Handle the speculative execution misfeatures */
2485	cpu_bugs_smt_update();
2486	/* Check whether IPI broadcasting can be enabled */
2487	apic_smt_update();
2488	}
2489
2490	void __init arch_cpu_finalize_init(void)
2491	{
2492	struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info);
2493
2494	identify_boot_cpu();
2495
2496	select_idle_routine();
2497
2498	/*
2499	* identify_boot_cpu() initialized SMT support information, let the
2500	* core code know.
2501	*/
2502	cpu_smt_set_num_threads(num_threads: __max_threads_per_core, max_threads: __max_threads_per_core);
2503
2504	if (!IS_ENABLED(CONFIG_SMP)) {
2505	pr_info("CPU: ");
2506	print_cpu_info(c: &boot_cpu_data);
2507	}
2508
2509	cpu_select_mitigations();
2510
2511	arch_smt_update();
2512
2513	if (IS_ENABLED(CONFIG_X86_32)) {
2514	/*
2515	* Check whether this is a real i386 which is not longer
2516	* supported and fixup the utsname.
2517	*/
2518	if (boot_cpu_data.x86 < 4)
2519	panic(fmt: "Kernel requires i486+ for 'invlpg' and other features");
2520
2521	init_utsname()->machine[1] =
2522	'0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
2523	}
2524
2525	/*
2526	* Must be before alternatives because it might set or clear
2527	* feature bits.
2528	*/
2529	fpu__init_system();
2530	fpu__init_cpu();
2531
2532	/*
2533	* Ensure that access to the per CPU representation has the initial
2534	* boot CPU configuration.
2535	*/
2536	*c = boot_cpu_data;
2537	c->initialized = true;
2538
2539	alternative_instructions();
2540
2541	if (IS_ENABLED(CONFIG_X86_64)) {
2542	unsigned long USER_PTR_MAX = TASK_SIZE_MAX;
2543
2544	/*
2545	* Enable this when LAM is gated on LASS support
2546	if (cpu_feature_enabled(X86_FEATURE_LAM))
2547	USER_PTR_MAX = (1ul << 63) - PAGE_SIZE;
2548	*/
2549	runtime_const_init(ptr, USER_PTR_MAX);
2550
2551	/*
2552	* Make sure the first 2MB area is not mapped by huge pages
2553	* There are typically fixed size MTRRs in there and overlapping
2554	* MTRRs into large pages causes slow downs.
2555	*
2556	* Right now we don't do that with gbpages because there seems
2557	* very little benefit for that case.
2558	*/
2559	if (!direct_gbpages)
2560	set_memory_4k(addr: (unsigned long)__va(0), numpages: 1);
2561	} else {
2562	fpu__init_check_bugs();
2563	}
2564
2565	/*
2566	* This needs to be called before any devices perform DMA
2567	* operations that might use the SWIOTLB bounce buffers. It will
2568	* mark the bounce buffers as decrypted so that their usage will
2569	* not cause "plain-text" data to be decrypted when accessed. It
2570	* must be called after late_time_init() so that Hyper-V x86/x64
2571	* hypercalls work when the SWIOTLB bounce buffers are decrypted.
2572	*/
2573	mem_encrypt_init();
2574	}
2575