1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Local APIC handling, local APIC timers
4	*
5	* (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6	*
7	* Fixes
8	* Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9	* thanks to Eric Gilmore
10	* and Rolf G. Tews
11	* for testing these extensively.
12	* Maciej W. Rozycki : Various updates and fixes.
13	* Mikael Pettersson : Power Management for UP-APIC.
14	* Pavel Machek and
15	* Mikael Pettersson : PM converted to driver model.
16	*/
17
18	#include <linux/perf_event.h>
19	#include <linux/kernel_stat.h>
20	#include <linux/mc146818rtc.h>
21	#include <linux/acpi_pmtmr.h>
22	#include <linux/bitmap.h>
23	#include <linux/clockchips.h>
24	#include <linux/interrupt.h>
25	#include <linux/memblock.h>
26	#include <linux/ftrace.h>
27	#include <linux/ioport.h>
28	#include <linux/export.h>
29	#include <linux/syscore_ops.h>
30	#include <linux/delay.h>
31	#include <linux/timex.h>
32	#include <linux/i8253.h>
33	#include <linux/dmar.h>
34	#include <linux/init.h>
35	#include <linux/cpu.h>
36	#include <linux/dmi.h>
37	#include <linux/smp.h>
38	#include <linux/mm.h>
39
40	#include <xen/xen.h>
41
42	#include <asm/trace/irq_vectors.h>
43	#include <asm/irq_remapping.h>
44	#include <asm/pc-conf-reg.h>
45	#include <asm/perf_event.h>
46	#include <asm/x86_init.h>
47	#include <linux/atomic.h>
48	#include <asm/barrier.h>
49	#include <asm/mpspec.h>
50	#include <asm/i8259.h>
51	#include <asm/proto.h>
52	#include <asm/traps.h>
53	#include <asm/apic.h>
54	#include <asm/acpi.h>
55	#include <asm/io_apic.h>
56	#include <asm/desc.h>
57	#include <asm/hpet.h>
58	#include <asm/mtrr.h>
59	#include <asm/time.h>
60	#include <asm/smp.h>
61	#include <asm/mce.h>
62	#include <asm/msr.h>
63	#include <asm/tsc.h>
64	#include <asm/hypervisor.h>
65	#include <asm/cpu_device_id.h>
66	#include <asm/intel-family.h>
67	#include <asm/irq_regs.h>
68	#include <asm/cpu.h>
69
70	#include "local.h"
71
72	/* Processor that is doing the boot up */
73	u32 boot_cpu_physical_apicid __ro_after_init = BAD_APICID;
74	EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
75
76	u8 boot_cpu_apic_version __ro_after_init;
77
78	/*
79	* This variable controls which CPUs receive external NMIs. By default,
80	* external NMIs are delivered only to the BSP.
81	*/
82	static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
83
84	/*
85	* Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
86	*/
87	static bool virt_ext_dest_id __ro_after_init;
88
89	/* For parallel bootup. */
90	unsigned long apic_mmio_base __ro_after_init;
91
92	static inline bool apic_accessible(void)
93	{
94	return x2apic_mode || apic_mmio_base;
95	}
96
97	#ifdef CONFIG_X86_32
98	/* Local APIC was disabled by the BIOS and enabled by the kernel */
99	static int enabled_via_apicbase __ro_after_init;
100
101	/*
102	* Handle interrupt mode configuration register (IMCR).
103	* This register controls whether the interrupt signals
104	* that reach the BSP come from the master PIC or from the
105	* local APIC. Before entering Symmetric I/O Mode, either
106	* the BIOS or the operating system must switch out of
107	* PIC Mode by changing the IMCR.
108	*/
109	static inline void imcr_pic_to_apic(void)
110	{
111	/* NMI and 8259 INTR go through APIC */
112	pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
113	}
114
115	static inline void imcr_apic_to_pic(void)
116	{
117	/* NMI and 8259 INTR go directly to BSP */
118	pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
119	}
120	#endif
121
122	/*
123	* Knob to control our willingness to enable the local APIC.
124	*
125	* +1=force-enable
126	*/
127	static int force_enable_local_apic __initdata;
128
129	/*
130	* APIC command line parameters
131	*/
132	static int __init parse_lapic(char *arg)
133	{
134	if (IS_ENABLED(CONFIG_X86_32) && !arg)
135	force_enable_local_apic = 1;
136	else if (arg && !strncmp(arg, "notscdeadline", 13))
137	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
138	return 0;
139	}
140	early_param("lapic", parse_lapic);
141
142	#ifdef CONFIG_X86_64
143	static int apic_calibrate_pmtmr __initdata;
144	static __init int setup_apicpmtimer(char *s)
145	{
146	apic_calibrate_pmtmr = 1;
147	notsc_setup(NULL);
148	return 1;
149	}
150	__setup("apicpmtimer", setup_apicpmtimer);
151	#endif
152
153	static unsigned long mp_lapic_addr __ro_after_init;
154	bool apic_is_disabled __ro_after_init;
155	/* Disable local APIC timer from the kernel commandline or via dmi quirk */
156	static int disable_apic_timer __initdata;
157	/* Local APIC timer works in C2 */
158	int local_apic_timer_c2_ok __ro_after_init;
159	EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
160
161	/*
162	* Debug level, exported for io_apic.c
163	*/
164	int apic_verbosity __ro_after_init;
165
166	int pic_mode __ro_after_init;
167
168	/* Have we found an MP table */
169	int smp_found_config __ro_after_init;
170
171	static struct resource lapic_resource = {
172	.name = "Local APIC",
173	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
174	};
175
176	unsigned int lapic_timer_period = 0;
177
178	static void apic_pm_activate(void);
179
180	/*
181	* Get the LAPIC version
182	*/
183	static inline int lapic_get_version(void)
184	{
185	return GET_APIC_VERSION(apic_read(APIC_LVR));
186	}
187
188	/*
189	* Check, if the APIC is integrated or a separate chip
190	*/
191	static inline int lapic_is_integrated(void)
192	{
193	return APIC_INTEGRATED(lapic_get_version());
194	}
195
196	/*
197	* Check, whether this is a modern or a first generation APIC
198	*/
199	static int modern_apic(void)
200	{
201	/* AMD systems use old APIC versions, so check the CPU */
202	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
203	boot_cpu_data.x86 >= 0xf)
204	return 1;
205
206	/* Hygon systems use modern APIC */
207	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
208	return 1;
209
210	return lapic_get_version() >= 0x14;
211	}
212
213	/*
214	* right after this call apic become NOOP driven
215	* so apic->write/read doesn't do anything
216	*/
217	static void __init apic_disable(void)
218	{
219	apic_install_driver(driver: &apic_noop);
220	}
221
222	void native_apic_icr_write(u32 low, u32 id)
223	{
224	unsigned long flags;
225
226	local_irq_save(flags);
227	apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id));
228	apic_write(APIC_ICR, val: low);
229	local_irq_restore(flags);
230	}
231
232	u64 native_apic_icr_read(void)
233	{
234	u32 icr1, icr2;
235
236	icr2 = apic_read(APIC_ICR2);
237	icr1 = apic_read(APIC_ICR);
238
239	return icr1 | ((u64)icr2 << 32);
240	}
241
242	/**
243	* lapic_get_maxlvt - get the maximum number of local vector table entries
244	*/
245	int lapic_get_maxlvt(void)
246	{
247	/*
248	* - we always have APIC integrated on 64bit mode
249	* - 82489DXs do not report # of LVT entries
250	*/
251	return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
252	}
253
254	/*
255	* Local APIC timer
256	*/
257
258	/* Clock divisor */
259	#define APIC_DIVISOR 16
260	#define TSC_DIVISOR 8
261
262	/* i82489DX specific */
263	#define I82489DX_BASE_DIVIDER (((0x2) << 18))
264
265	/*
266	* This function sets up the local APIC timer, with a timeout of
267	* 'clocks' APIC bus clock. During calibration we actually call
268	* this function twice on the boot CPU, once with a bogus timeout
269	* value, second time for real. The other (noncalibrating) CPUs
270	* call this function only once, with the real, calibrated value.
271	*
272	* We do reads before writes even if unnecessary, to get around the
273	* P5 APIC double write bug.
274	*/
275	static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
276	{
277	unsigned int lvtt_value, tmp_value;
278
279	lvtt_value = LOCAL_TIMER_VECTOR;
280	if (!oneshot)
281	lvtt_value |= APIC_LVT_TIMER_PERIODIC;
282	else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
283	lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
284
285	/*
286	* The i82489DX APIC uses bit 18 and 19 for the base divider. This
287	* overlaps with bit 18 on integrated APICs, but is not documented
288	* in the SDM. No problem though. i82489DX equipped systems do not
289	* have TSC deadline timer.
290	*/
291	if (!lapic_is_integrated())
292	lvtt_value |= I82489DX_BASE_DIVIDER;
293
294	if (!irqen)
295	lvtt_value |= APIC_LVT_MASKED;
296
297	apic_write(APIC_LVTT, val: lvtt_value);
298
299	if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
300	/*
301	* See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
302	* writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
303	* According to Intel, MFENCE can do the serialization here.
304	*/
305	asm volatile("mfence" : : : "memory");
306	return;
307	}
308
309	/*
310	* Divide PICLK by 16
311	*/
312	tmp_value = apic_read(APIC_TDCR);
313	apic_write(APIC_TDCR,
314	val: (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
315	APIC_TDR_DIV_16);
316
317	if (!oneshot)
318	apic_write(APIC_TMICT, val: clocks / APIC_DIVISOR);
319	}
320
321	/*
322	* Setup extended LVT, AMD specific
323	*
324	* Software should use the LVT offsets the BIOS provides. The offsets
325	* are determined by the subsystems using it like those for MCE
326	* threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
327	* are supported. Beginning with family 10h at least 4 offsets are
328	* available.
329	*
330	* Since the offsets must be consistent for all cores, we keep track
331	* of the LVT offsets in software and reserve the offset for the same
332	* vector also to be used on other cores. An offset is freed by
333	* setting the entry to APIC_EILVT_MASKED.
334	*
335	* If the BIOS is right, there should be no conflicts. Otherwise a
336	* "[Firmware Bug]: ..." error message is generated. However, if
337	* software does not properly determines the offsets, it is not
338	* necessarily a BIOS bug.
339	*/
340
341	static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
342
343	static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
344	{
345	return (old & APIC_EILVT_MASKED)
346	|| (new == APIC_EILVT_MASKED)
347	|| ((new & ~APIC_EILVT_MASKED) == old);
348	}
349
350	static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
351	{
352	unsigned int rsvd, vector;
353
354	if (offset >= APIC_EILVT_NR_MAX)
355	return ~0;
356
357	rsvd = atomic_read(v: &eilvt_offsets[offset]);
358	do {
359	vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
360	if (vector && !eilvt_entry_is_changeable(old: vector, new))
361	/* may not change if vectors are different */
362	return rsvd;
363	} while (!atomic_try_cmpxchg(v: &eilvt_offsets[offset], old: &rsvd, new));
364
365	rsvd = new & ~APIC_EILVT_MASKED;
366	if (rsvd && rsvd != vector)
367	pr_info("LVT offset %d assigned for vector 0x%02x\n",
368	offset, rsvd);
369
370	return new;
371	}
372
373	/*
374	* If mask=1, the LVT entry does not generate interrupts while mask=0
375	* enables the vector. See also the BKDGs. Must be called with
376	* preemption disabled.
377	*/
378
379	int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
380	{
381	unsigned long reg = APIC_EILVTn(offset);
382	unsigned int new, old, reserved;
383
384	new = (mask << 16) | (msg_type << 8) | vector;
385	old = apic_read(reg);
386	reserved = reserve_eilvt_offset(offset, new);
387
388	if (reserved != new) {
389	pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
390	"vector 0x%x, but the register is already in use for "
391	"vector 0x%x on another cpu\n",
392	smp_processor_id(), reg, offset, new, reserved);
393	return -EINVAL;
394	}
395
396	if (!eilvt_entry_is_changeable(old, new)) {
397	pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
398	"vector 0x%x, but the register is already in use for "
399	"vector 0x%x on this cpu\n",
400	smp_processor_id(), reg, offset, new, old);
401	return -EBUSY;
402	}
403
404	apic_write(reg, val: new);
405
406	return 0;
407	}
408	EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
409
410	/*
411	* Program the next event, relative to now
412	*/
413	static int lapic_next_event(unsigned long delta,
414	struct clock_event_device *evt)
415	{
416	apic_write(APIC_TMICT, val: delta);
417	return 0;
418	}
419
420	static int lapic_next_deadline(unsigned long delta,
421	struct clock_event_device *evt)
422	{
423	u64 tsc;
424
425	/* This MSR is special and need a special fence: */
426	weak_wrmsr_fence();
427
428	tsc = rdtsc();
429	wrmsrq(MSR_IA32_TSC_DEADLINE, val: tsc + (((u64) delta) * TSC_DIVISOR));
430	return 0;
431	}
432
433	static int lapic_timer_shutdown(struct clock_event_device *evt)
434	{
435	unsigned int v;
436
437	/* Lapic used as dummy for broadcast ? */
438	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
439	return 0;
440
441	v = apic_read(APIC_LVTT);
442	v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
443	apic_write(APIC_LVTT, val: v);
444
445	/*
446	* Setting APIC_LVT_MASKED (above) should be enough to tell
447	* the hardware that this timer will never fire. But AMD
448	* erratum 411 and some Intel CPU behavior circa 2024 say
449	* otherwise. Time for belt and suspenders programming: mask
450	* the timer _and_ zero the counter registers:
451	*/
452	if (v & APIC_LVT_TIMER_TSCDEADLINE)
453	wrmsrq(MSR_IA32_TSC_DEADLINE, val: 0);
454	else
455	apic_write(APIC_TMICT, val: 0);
456
457	return 0;
458	}
459
460	static inline int
461	lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
462	{
463	/* Lapic used as dummy for broadcast ? */
464	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
465	return 0;
466
467	__setup_APIC_LVTT(clocks: lapic_timer_period, oneshot, irqen: 1);
468	return 0;
469	}
470
471	static int lapic_timer_set_periodic(struct clock_event_device *evt)
472	{
473	return lapic_timer_set_periodic_oneshot(evt, oneshot: false);
474	}
475
476	static int lapic_timer_set_oneshot(struct clock_event_device *evt)
477	{
478	return lapic_timer_set_periodic_oneshot(evt, oneshot: true);
479	}
480
481	/*
482	* Local APIC timer broadcast function
483	*/
484	static void lapic_timer_broadcast(const struct cpumask *mask)
485	{
486	#ifdef CONFIG_SMP
487	__apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
488	#endif
489	}
490
491
492	/*
493	* The local apic timer can be used for any function which is CPU local.
494	*/
495	static struct clock_event_device lapic_clockevent = {
496	.name = "lapic",
497	.features = CLOCK_EVT_FEAT_PERIODIC |
498	CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
499	| CLOCK_EVT_FEAT_DUMMY,
500	.shift = 32,
501	.set_state_shutdown = lapic_timer_shutdown,
502	.set_state_periodic = lapic_timer_set_periodic,
503	.set_state_oneshot = lapic_timer_set_oneshot,
504	.set_state_oneshot_stopped = lapic_timer_shutdown,
505	.set_next_event = lapic_next_event,
506	.broadcast = lapic_timer_broadcast,
507	.rating = 100,
508	.irq = -1,
509	};
510	static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
511
512	static const struct x86_cpu_id deadline_match[] __initconst = {
513	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X, 0x2, 0x2, 0x3a), /* EP */
514	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X, 0x4, 0x4, 0x0f), /* EX */
515
516	X86_MATCH_VFM(INTEL_BROADWELL_X, 0x0b000020),
517
518	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x2, 0x2, 0x00000011),
519	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x3, 0x3, 0x0700000e),
520	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x4, 0x4, 0x0f00000c),
521	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x5, 0x5, 0x0e000003),
522
523	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x3, 0x3, 0x01000136),
524	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x4, 0x4, 0x02000014),
525	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x5, 0xf, 0),
526
527	X86_MATCH_VFM(INTEL_HASWELL, 0x22),
528	X86_MATCH_VFM(INTEL_HASWELL_L, 0x20),
529	X86_MATCH_VFM(INTEL_HASWELL_G, 0x17),
530
531	X86_MATCH_VFM(INTEL_BROADWELL, 0x25),
532	X86_MATCH_VFM(INTEL_BROADWELL_G, 0x17),
533
534	X86_MATCH_VFM(INTEL_SKYLAKE_L, 0xb2),
535	X86_MATCH_VFM(INTEL_SKYLAKE, 0xb2),
536
537	X86_MATCH_VFM(INTEL_KABYLAKE_L, 0x52),
538	X86_MATCH_VFM(INTEL_KABYLAKE, 0x52),
539
540	{},
541	};
542
543	static __init bool apic_validate_deadline_timer(void)
544	{
545	const struct x86_cpu_id *m;
546	u32 rev;
547
548	if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
549	return false;
550	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
551	return true;
552
553	m = x86_match_cpu(match: deadline_match);
554	if (!m)
555	return true;
556
557	rev = (u32)m->driver_data;
558
559	if (boot_cpu_data.microcode >= rev)
560	return true;
561
562	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
563	pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
564	"please update microcode to version: 0x%x (or later)\n", rev);
565	return false;
566	}
567
568	/*
569	* Setup the local APIC timer for this CPU. Copy the initialized values
570	* of the boot CPU and register the clock event in the framework.
571	*/
572	static void setup_APIC_timer(void)
573	{
574	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
575
576	if (this_cpu_has(X86_FEATURE_ARAT)) {
577	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
578	/* Make LAPIC timer preferable over percpu HPET */
579	lapic_clockevent.rating = 150;
580	}
581
582	memcpy(levt, &lapic_clockevent, sizeof(*levt));
583	levt->cpumask = cpumask_of(smp_processor_id());
584
585	if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
586	levt->name = "lapic-deadline";
587	levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
588	CLOCK_EVT_FEAT_DUMMY);
589	levt->set_next_event = lapic_next_deadline;
590	clockevents_config_and_register(dev: levt,
591	freq: tsc_khz * (1000 / TSC_DIVISOR),
592	min_delta: 0xF, max_delta: ~0UL);
593	} else
594	clockevents_register_device(dev: levt);
595	}
596
597	/*
598	* Install the updated TSC frequency from recalibration at the TSC
599	* deadline clockevent devices.
600	*/
601	static void __lapic_update_tsc_freq(void *info)
602	{
603	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
604
605	if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
606	return;
607
608	clockevents_update_freq(ce: levt, freq: tsc_khz * (1000 / TSC_DIVISOR));
609	}
610
611	void lapic_update_tsc_freq(void)
612	{
613	/*
614	* The clockevent device's ->mult and ->shift can both be
615	* changed. In order to avoid races, schedule the frequency
616	* update code on each CPU.
617	*/
618	on_each_cpu(func: __lapic_update_tsc_freq, NULL, wait: 0);
619	}
620
621	/*
622	* In this functions we calibrate APIC bus clocks to the external timer.
623	*
624	* We want to do the calibration only once since we want to have local timer
625	* irqs synchronous. CPUs connected by the same APIC bus have the very same bus
626	* frequency.
627	*
628	* This was previously done by reading the PIT/HPET and waiting for a wrap
629	* around to find out, that a tick has elapsed. I have a box, where the PIT
630	* readout is broken, so it never gets out of the wait loop again. This was
631	* also reported by others.
632	*
633	* Monitoring the jiffies value is inaccurate and the clockevents
634	* infrastructure allows us to do a simple substitution of the interrupt
635	* handler.
636	*
637	* The calibration routine also uses the pm_timer when possible, as the PIT
638	* happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
639	* back to normal later in the boot process).
640	*/
641
642	#define LAPIC_CAL_LOOPS (HZ/10)
643
644	static __initdata int lapic_cal_loops = -1;
645	static __initdata long lapic_cal_t1, lapic_cal_t2;
646	static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
647	static __initdata u32 lapic_cal_pm1, lapic_cal_pm2;
648	static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
649
650	/*
651	* Temporary interrupt handler and polled calibration function.
652	*/
653	static void __init lapic_cal_handler(struct clock_event_device *dev)
654	{
655	unsigned long long tsc = 0;
656	long tapic = apic_read(APIC_TMCCT);
657	u32 pm = acpi_pm_read_early();
658
659	if (boot_cpu_has(X86_FEATURE_TSC))
660	tsc = rdtsc();
661
662	switch (lapic_cal_loops++) {
663	case 0:
664	lapic_cal_t1 = tapic;
665	lapic_cal_tsc1 = tsc;
666	lapic_cal_pm1 = pm;
667	lapic_cal_j1 = jiffies;
668	break;
669
670	case LAPIC_CAL_LOOPS:
671	lapic_cal_t2 = tapic;
672	lapic_cal_tsc2 = tsc;
673	if (pm < lapic_cal_pm1)
674	pm += ACPI_PM_OVRRUN;
675	lapic_cal_pm2 = pm;
676	lapic_cal_j2 = jiffies;
677	break;
678	}
679	}
680
681	static int __init
682	calibrate_by_pmtimer(u32 deltapm, long *delta, long *deltatsc)
683	{
684	const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
685	const long pm_thresh = pm_100ms / 100;
686	unsigned long mult;
687	u64 res;
688
689	#ifndef CONFIG_X86_PM_TIMER
690	return -1;
691	#endif
692
693	apic_pr_verbose("... PM-Timer delta = %u\n", deltapm);
694
695	/* Check, if the PM timer is available */
696	if (!deltapm)
697	return -1;
698
699	mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, shift_constant: 22);
700
701	if (deltapm > (pm_100ms - pm_thresh) &&
702	deltapm < (pm_100ms + pm_thresh)) {
703	apic_pr_verbose("... PM-Timer result ok\n");
704	return 0;
705	}
706
707	res = (((u64)deltapm) * mult) >> 22;
708	do_div(res, 1000000);
709	pr_warn("APIC calibration not consistent with PM-Timer: %ldms instead of 100ms\n",
710	(long)res);
711
712	/* Correct the lapic counter value */
713	res = (((u64)(*delta)) * pm_100ms);
714	do_div(res, deltapm);
715	pr_info("APIC delta adjusted to PM-Timer: "
716	"%lu (%ld)\n", (unsigned long)res, *delta);
717	*delta = (long)res;
718
719	/* Correct the tsc counter value */
720	if (boot_cpu_has(X86_FEATURE_TSC)) {
721	res = (((u64)(*deltatsc)) * pm_100ms);
722	do_div(res, deltapm);
723	apic_pr_verbose("TSC delta adjusted to PM-Timer: %lu (%ld)\n",
724	(unsigned long)res, *deltatsc);
725	*deltatsc = (long)res;
726	}
727
728	return 0;
729	}
730
731	static int __init lapic_init_clockevent(void)
732	{
733	if (!lapic_timer_period)
734	return -1;
735
736	/* Calculate the scaled math multiplication factor */
737	lapic_clockevent.mult = div_sc(ticks: lapic_timer_period/APIC_DIVISOR,
738	TICK_NSEC, shift: lapic_clockevent.shift);
739	lapic_clockevent.max_delta_ns =
740	clockevent_delta2ns(latch: 0x7FFFFFFF, evt: &lapic_clockevent);
741	lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
742	lapic_clockevent.min_delta_ns =
743	clockevent_delta2ns(latch: 0xF, evt: &lapic_clockevent);
744	lapic_clockevent.min_delta_ticks = 0xF;
745
746	return 0;
747	}
748
749	bool __init apic_needs_pit(void)
750	{
751	/*
752	* If the frequencies are not known, PIT is required for both TSC
753	* and apic timer calibration.
754	*/
755	if (!tsc_khz || !cpu_khz)
756	return true;
757
758	/* Is there an APIC at all or is it disabled? */
759	if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
760	return true;
761
762	/*
763	* If interrupt delivery mode is legacy PIC or virtual wire without
764	* configuration, the local APIC timer won't be set up. Make sure
765	* that the PIT is initialized.
766	*/
767	if (apic_intr_mode == APIC_PIC ||
768	apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
769	return true;
770
771	/* Virt guests may lack ARAT, but still have DEADLINE */
772	if (!boot_cpu_has(X86_FEATURE_ARAT))
773	return true;
774
775	/* Deadline timer is based on TSC so no further PIT action required */
776	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
777	return false;
778
779	/* APIC timer disabled? */
780	if (disable_apic_timer)
781	return true;
782	/*
783	* The APIC timer frequency is known already, no PIT calibration
784	* required. If unknown, let the PIT be initialized.
785	*/
786	return lapic_timer_period == 0;
787	}
788
789	static int __init calibrate_APIC_clock(void)
790	{
791	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
792	u64 tsc_perj = 0, tsc_start = 0;
793	unsigned long jif_start;
794	unsigned long deltaj;
795	long delta, deltatsc;
796	int pm_referenced = 0;
797
798	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
799	return 0;
800
801	/*
802	* Check if lapic timer has already been calibrated by platform
803	* specific routine, such as tsc calibration code. If so just fill
804	* in the clockevent structure and return.
805	*/
806	if (!lapic_init_clockevent()) {
807	apic_pr_verbose("lapic timer already calibrated %d\n", lapic_timer_period);
808	/*
809	* Direct calibration methods must have an always running
810	* local APIC timer, no need for broadcast timer.
811	*/
812	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
813	return 0;
814	}
815
816	apic_pr_verbose("Using local APIC timer interrupts. Calibrating APIC timer ...\n");
817
818	/*
819	* There are platforms w/o global clockevent devices. Instead of
820	* making the calibration conditional on that, use a polling based
821	* approach everywhere.
822	*/
823	local_irq_disable();
824
825	/*
826	* Setup the APIC counter to maximum. There is no way the lapic
827	* can underflow in the 100ms detection time frame
828	*/
829	__setup_APIC_LVTT(clocks: 0xffffffff, oneshot: 0, irqen: 0);
830
831	/*
832	* Methods to terminate the calibration loop:
833	* 1) Global clockevent if available (jiffies)
834	* 2) TSC if available and frequency is known
835	*/
836	jif_start = READ_ONCE(jiffies);
837
838	if (tsc_khz) {
839	tsc_start = rdtsc();
840	tsc_perj = div_u64(dividend: (u64)tsc_khz * 1000, HZ);
841	}
842
843	/*
844	* Enable interrupts so the tick can fire, if a global
845	* clockevent device is available
846	*/
847	local_irq_enable();
848
849	while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
850	/* Wait for a tick to elapse */
851	while (1) {
852	if (tsc_khz) {
853	u64 tsc_now = rdtsc();
854	if ((tsc_now - tsc_start) >= tsc_perj) {
855	tsc_start += tsc_perj;
856	break;
857	}
858	} else {
859	unsigned long jif_now = READ_ONCE(jiffies);
860
861	if (time_after(jif_now, jif_start)) {
862	jif_start = jif_now;
863	break;
864	}
865	}
866	cpu_relax();
867	}
868
869	/* Invoke the calibration routine */
870	local_irq_disable();
871	lapic_cal_handler(NULL);
872	local_irq_enable();
873	}
874
875	local_irq_disable();
876
877	/* Build delta t1-t2 as apic timer counts down */
878	delta = lapic_cal_t1 - lapic_cal_t2;
879	apic_pr_verbose("... lapic delta = %ld\n", delta);
880
881	deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
882
883	/* we trust the PM based calibration if possible */
884	pm_referenced = !calibrate_by_pmtimer(deltapm: lapic_cal_pm2 - lapic_cal_pm1,
885	delta: &delta, deltatsc: &deltatsc);
886
887	lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
888	lapic_init_clockevent();
889
890	apic_pr_verbose("..... delta %ld\n", delta);
891	apic_pr_verbose("..... mult: %u\n", lapic_clockevent.mult);
892	apic_pr_verbose("..... calibration result: %u\n", lapic_timer_period);
893
894	if (boot_cpu_has(X86_FEATURE_TSC)) {
895	apic_pr_verbose("..... CPU clock speed is %ld.%04ld MHz.\n",
896	(deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
897	(deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
898	}
899
900	apic_pr_verbose("..... host bus clock speed is %u.%04u MHz.\n",
901	lapic_timer_period / (1000000 / HZ),
902	lapic_timer_period % (1000000 / HZ));
903
904	/*
905	* Do a sanity check on the APIC calibration result
906	*/
907	if (lapic_timer_period < (1000000 / HZ)) {
908	local_irq_enable();
909	pr_warn("APIC frequency too slow, disabling apic timer\n");
910	return -1;
911	}
912
913	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
914
915	/*
916	* PM timer calibration failed or not turned on so lets try APIC
917	* timer based calibration, if a global clockevent device is
918	* available.
919	*/
920	if (!pm_referenced && global_clock_event) {
921	apic_pr_verbose("... verify APIC timer\n");
922
923	/*
924	* Setup the apic timer manually
925	*/
926	levt->event_handler = lapic_cal_handler;
927	lapic_timer_set_periodic(evt: levt);
928	lapic_cal_loops = -1;
929
930	/* Let the interrupts run */
931	local_irq_enable();
932
933	while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
934	cpu_relax();
935
936	/* Stop the lapic timer */
937	local_irq_disable();
938	lapic_timer_shutdown(evt: levt);
939
940	/* Jiffies delta */
941	deltaj = lapic_cal_j2 - lapic_cal_j1;
942	apic_pr_verbose("... jiffies delta = %lu\n", deltaj);
943
944	/* Check, if the jiffies result is consistent */
945	if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
946	apic_pr_verbose("... jiffies result ok\n");
947	else
948	levt->features |= CLOCK_EVT_FEAT_DUMMY;
949	}
950	local_irq_enable();
951
952	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
953	pr_warn("APIC timer disabled due to verification failure\n");
954	return -1;
955	}
956
957	return 0;
958	}
959
960	/*
961	* Setup the boot APIC
962	*
963	* Calibrate and verify the result.
964	*/
965	void __init setup_boot_APIC_clock(void)
966	{
967	/*
968	* The local apic timer can be disabled via the kernel
969	* commandline or from the CPU detection code. Register the lapic
970	* timer as a dummy clock event source on SMP systems, so the
971	* broadcast mechanism is used. On UP systems simply ignore it.
972	*/
973	if (disable_apic_timer) {
974	pr_info("Disabling APIC timer\n");
975	/* No broadcast on UP ! */
976	if (num_possible_cpus() > 1) {
977	lapic_clockevent.mult = 1;
978	setup_APIC_timer();
979	}
980	return;
981	}
982
983	if (calibrate_APIC_clock()) {
984	/* No broadcast on UP ! */
985	if (num_possible_cpus() > 1)
986	setup_APIC_timer();
987	return;
988	}
989
990	/*
991	* If nmi_watchdog is set to IO_APIC, we need the
992	* PIT/HPET going. Otherwise register lapic as a dummy
993	* device.
994	*/
995	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
996
997	/* Setup the lapic or request the broadcast */
998	setup_APIC_timer();
999	amd_e400_c1e_apic_setup();
1000	}
1001
1002	void setup_secondary_APIC_clock(void)
1003	{
1004	setup_APIC_timer();
1005	amd_e400_c1e_apic_setup();
1006	}
1007
1008	/*
1009	* The guts of the apic timer interrupt
1010	*/
1011	static void local_apic_timer_interrupt(void)
1012	{
1013	struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1014
1015	/*
1016	* Normally we should not be here till LAPIC has been initialized but
1017	* in some cases like kdump, its possible that there is a pending LAPIC
1018	* timer interrupt from previous kernel's context and is delivered in
1019	* new kernel the moment interrupts are enabled.
1020	*
1021	* Interrupts are enabled early and LAPIC is setup much later, hence
1022	* its possible that when we get here evt->event_handler is NULL.
1023	* Check for event_handler being NULL and discard the interrupt as
1024	* spurious.
1025	*/
1026	if (!evt->event_handler) {
1027	pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1028	smp_processor_id());
1029	/* Switch it off */
1030	lapic_timer_shutdown(evt);
1031	return;
1032	}
1033
1034	/*
1035	* the NMI deadlock-detector uses this.
1036	*/
1037	inc_irq_stat(apic_timer_irqs);
1038
1039	evt->event_handler(evt);
1040	}
1041
1042	/*
1043	* Local APIC timer interrupt. This is the most natural way for doing
1044	* local interrupts, but local timer interrupts can be emulated by
1045	* broadcast interrupts too. [in case the hw doesn't support APIC timers]
1046	*
1047	* [ if a single-CPU system runs an SMP kernel then we call the local
1048	* interrupt as well. Thus we cannot inline the local irq ... ]
1049	*/
1050	DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1051	{
1052	struct pt_regs *old_regs = set_irq_regs(regs);
1053
1054	apic_eoi();
1055	trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1056	local_apic_timer_interrupt();
1057	trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1058
1059	set_irq_regs(old_regs);
1060	}
1061
1062	/*
1063	* Local APIC start and shutdown
1064	*/
1065
1066	/**
1067	* clear_local_APIC - shutdown the local APIC
1068	*
1069	* This is called, when a CPU is disabled and before rebooting, so the state of
1070	* the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1071	* leftovers during boot.
1072	*/
1073	void clear_local_APIC(void)
1074	{
1075	int maxlvt;
1076	u32 v;
1077
1078	if (!apic_accessible())
1079	return;
1080
1081	maxlvt = lapic_get_maxlvt();
1082	/*
1083	* Masking an LVT entry can trigger a local APIC error
1084	* if the vector is zero. Mask LVTERR first to prevent this.
1085	*/
1086	if (maxlvt >= 3) {
1087	v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1088	apic_write(APIC_LVTERR, val: v | APIC_LVT_MASKED);
1089	}
1090	/*
1091	* Careful: we have to set masks only first to deassert
1092	* any level-triggered sources.
1093	*/
1094	v = apic_read(APIC_LVTT);
1095	apic_write(APIC_LVTT, val: v | APIC_LVT_MASKED);
1096	v = apic_read(APIC_LVT0);
1097	apic_write(APIC_LVT0, val: v | APIC_LVT_MASKED);
1098	v = apic_read(APIC_LVT1);
1099	apic_write(APIC_LVT1, val: v | APIC_LVT_MASKED);
1100	if (maxlvt >= 4) {
1101	v = apic_read(APIC_LVTPC);
1102	apic_write(APIC_LVTPC, val: v | APIC_LVT_MASKED);
1103	}
1104
1105	/* lets not touch this if we didn't frob it */
1106	#ifdef CONFIG_X86_THERMAL_VECTOR
1107	if (maxlvt >= 5) {
1108	v = apic_read(APIC_LVTTHMR);
1109	apic_write(APIC_LVTTHMR, val: v | APIC_LVT_MASKED);
1110	}
1111	#endif
1112	#ifdef CONFIG_X86_MCE_INTEL
1113	if (maxlvt >= 6) {
1114	v = apic_read(APIC_LVTCMCI);
1115	if (!(v & APIC_LVT_MASKED))
1116	apic_write(APIC_LVTCMCI, val: v | APIC_LVT_MASKED);
1117	}
1118	#endif
1119
1120	/*
1121	* Clean APIC state for other OSs:
1122	*/
1123	apic_write(APIC_LVTT, APIC_LVT_MASKED);
1124	apic_write(APIC_LVT0, APIC_LVT_MASKED);
1125	apic_write(APIC_LVT1, APIC_LVT_MASKED);
1126	if (maxlvt >= 3)
1127	apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1128	if (maxlvt >= 4)
1129	apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1130
1131	/* Integrated APIC (!82489DX) ? */
1132	if (lapic_is_integrated()) {
1133	if (maxlvt > 3)
1134	/* Clear ESR due to Pentium errata 3AP and 11AP */
1135	apic_write(APIC_ESR, val: 0);
1136	apic_read(APIC_ESR);
1137	}
1138	}
1139
1140	/**
1141	* apic_soft_disable - Clears and software disables the local APIC on hotplug
1142	*
1143	* Contrary to disable_local_APIC() this does not touch the enable bit in
1144	* MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1145	* bus would require a hardware reset as the APIC would lose track of bus
1146	* arbitration. On systems with FSB delivery APICBASE could be disabled,
1147	* but it has to be guaranteed that no interrupt is sent to the APIC while
1148	* in that state and it's not clear from the SDM whether it still responds
1149	* to INIT/SIPI messages. Stay on the safe side and use software disable.
1150	*/
1151	void apic_soft_disable(void)
1152	{
1153	u32 value;
1154
1155	clear_local_APIC();
1156
1157	/* Soft disable APIC (implies clearing of registers for 82489DX!). */
1158	value = apic_read(APIC_SPIV);
1159	value &= ~APIC_SPIV_APIC_ENABLED;
1160	apic_write(APIC_SPIV, val: value);
1161	}
1162
1163	/**
1164	* disable_local_APIC - clear and disable the local APIC
1165	*/
1166	void disable_local_APIC(void)
1167	{
1168	if (!apic_accessible())
1169	return;
1170
1171	apic_soft_disable();
1172
1173	#ifdef CONFIG_X86_32
1174	/*
1175	* When LAPIC was disabled by the BIOS and enabled by the kernel,
1176	* restore the disabled state.
1177	*/
1178	if (enabled_via_apicbase) {
1179	unsigned int l, h;
1180
1181	rdmsr(MSR_IA32_APICBASE, l, h);
1182	l &= ~MSR_IA32_APICBASE_ENABLE;
1183	wrmsr(MSR_IA32_APICBASE, l, h);
1184	}
1185	#endif
1186	}
1187
1188	/*
1189	* If Linux enabled the LAPIC against the BIOS default disable it down before
1190	* re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1191	* not power-off. Additionally clear all LVT entries before disable_local_APIC
1192	* for the case where Linux didn't enable the LAPIC.
1193	*/
1194	void lapic_shutdown(void)
1195	{
1196	unsigned long flags;
1197
1198	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1199	return;
1200
1201	local_irq_save(flags);
1202
1203	#ifdef CONFIG_X86_32
1204	if (!enabled_via_apicbase)
1205	clear_local_APIC();
1206	else
1207	#endif
1208	disable_local_APIC();
1209
1210
1211	local_irq_restore(flags);
1212	}
1213
1214	/**
1215	* sync_Arb_IDs - synchronize APIC bus arbitration IDs
1216	*/
1217	void __init sync_Arb_IDs(void)
1218	{
1219	/*
1220	* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1221	* needed on AMD.
1222	*/
1223	if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1224	return;
1225
1226	/*
1227	* Wait for idle.
1228	*/
1229	apic_wait_icr_idle();
1230
1231	apic_pr_debug("Synchronizing Arb IDs.\n");
1232	apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG | APIC_DM_INIT);
1233	}
1234
1235	enum apic_intr_mode_id apic_intr_mode __ro_after_init;
1236
1237	static int __init __apic_intr_mode_select(void)
1238	{
1239	/* Check kernel option */
1240	if (apic_is_disabled) {
1241	pr_info("APIC disabled via kernel command line\n");
1242	return APIC_PIC;
1243	}
1244
1245	/* Check BIOS */
1246	#ifdef CONFIG_X86_64
1247	/* On 64-bit, the APIC must be integrated, Check local APIC only */
1248	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1249	apic_is_disabled = true;
1250	pr_info("APIC disabled by BIOS\n");
1251	return APIC_PIC;
1252	}
1253	#else
1254	/* On 32-bit, the APIC may be integrated APIC or 82489DX */
1255
1256	/* Neither 82489DX nor integrated APIC ? */
1257	if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
1258	apic_is_disabled = true;
1259	return APIC_PIC;
1260	}
1261
1262	/* If the BIOS pretends there is an integrated APIC ? */
1263	if (!boot_cpu_has(X86_FEATURE_APIC) &&
1264	APIC_INTEGRATED(boot_cpu_apic_version)) {
1265	apic_is_disabled = true;
1266	pr_err(FW_BUG "Local APIC not detected, force emulation\n");
1267	return APIC_PIC;
1268	}
1269	#endif
1270
1271	/* Check MP table or ACPI MADT configuration */
1272	if (!smp_found_config) {
1273	disable_ioapic_support();
1274	if (!acpi_lapic) {
1275	pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1276	return APIC_VIRTUAL_WIRE_NO_CONFIG;
1277	}
1278	return APIC_VIRTUAL_WIRE;
1279	}
1280
1281	#ifdef CONFIG_SMP
1282	/* If SMP should be disabled, then really disable it! */
1283	if (!setup_max_cpus) {
1284	pr_info("APIC: SMP mode deactivated\n");
1285	return APIC_SYMMETRIC_IO_NO_ROUTING;
1286	}
1287	#endif
1288
1289	return APIC_SYMMETRIC_IO;
1290	}
1291
1292	/* Select the interrupt delivery mode for the BSP */
1293	void __init apic_intr_mode_select(void)
1294	{
1295	apic_intr_mode = __apic_intr_mode_select();
1296	}
1297
1298	/*
1299	* An initial setup of the virtual wire mode.
1300	*/
1301	void __init init_bsp_APIC(void)
1302	{
1303	unsigned int value;
1304
1305	/*
1306	* Don't do the setup now if we have a SMP BIOS as the
1307	* through-I/O-APIC virtual wire mode might be active.
1308	*/
1309	if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1310	return;
1311
1312	/*
1313	* Do not trust the local APIC being empty at bootup.
1314	*/
1315	clear_local_APIC();
1316
1317	/*
1318	* Enable APIC.
1319	*/
1320	value = apic_read(APIC_SPIV);
1321	value &= ~APIC_VECTOR_MASK;
1322	value |= APIC_SPIV_APIC_ENABLED;
1323
1324	#ifdef CONFIG_X86_32
1325	/* This bit is reserved on P4/Xeon and should be cleared */
1326	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1327	(boot_cpu_data.x86 == 15))
1328	value &= ~APIC_SPIV_FOCUS_DISABLED;
1329	else
1330	#endif
1331	value |= APIC_SPIV_FOCUS_DISABLED;
1332	value |= SPURIOUS_APIC_VECTOR;
1333	apic_write(APIC_SPIV, val: value);
1334
1335	/*
1336	* Set up the virtual wire mode.
1337	*/
1338	apic_write(APIC_LVT0, APIC_DM_EXTINT);
1339	value = APIC_DM_NMI;
1340	if (!lapic_is_integrated()) /* 82489DX */
1341	value |= APIC_LVT_LEVEL_TRIGGER;
1342	if (apic_extnmi == APIC_EXTNMI_NONE)
1343	value |= APIC_LVT_MASKED;
1344	apic_write(APIC_LVT1, val: value);
1345	}
1346
1347	static void __init apic_bsp_setup(bool upmode);
1348
1349	/* Init the interrupt delivery mode for the BSP */
1350	void __init apic_intr_mode_init(void)
1351	{
1352	bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
1353
1354	switch (apic_intr_mode) {
1355	case APIC_PIC:
1356	pr_info("APIC: Keep in PIC mode(8259)\n");
1357	return;
1358	case APIC_VIRTUAL_WIRE:
1359	pr_info("APIC: Switch to virtual wire mode setup\n");
1360	break;
1361	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1362	pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1363	upmode = true;
1364	break;
1365	case APIC_SYMMETRIC_IO:
1366	pr_info("APIC: Switch to symmetric I/O mode setup\n");
1367	break;
1368	case APIC_SYMMETRIC_IO_NO_ROUTING:
1369	pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1370	break;
1371	}
1372
1373	x86_64_probe_apic();
1374
1375	if (x86_platform.apic_post_init)
1376	x86_platform.apic_post_init();
1377
1378	apic_bsp_setup(upmode);
1379	}
1380
1381	static void lapic_setup_esr(void)
1382	{
1383	unsigned int oldvalue, value, maxlvt;
1384
1385	if (!lapic_is_integrated()) {
1386	pr_info("No ESR for 82489DX.\n");
1387	return;
1388	}
1389
1390	if (apic->disable_esr) {
1391	/*
1392	* Something untraceable is creating bad interrupts on
1393	* secondary quads ... for the moment, just leave the
1394	* ESR disabled - we can't do anything useful with the
1395	* errors anyway - mbligh
1396	*/
1397	pr_info("Leaving ESR disabled.\n");
1398	return;
1399	}
1400
1401	maxlvt = lapic_get_maxlvt();
1402	if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1403	apic_write(APIC_ESR, val: 0);
1404	oldvalue = apic_read(APIC_ESR);
1405
1406	/* enables sending errors */
1407	value = ERROR_APIC_VECTOR;
1408	apic_write(APIC_LVTERR, val: value);
1409
1410	/*
1411	* spec says clear errors after enabling vector.
1412	*/
1413	if (maxlvt > 3)
1414	apic_write(APIC_ESR, val: 0);
1415	value = apic_read(APIC_ESR);
1416	if (value != oldvalue) {
1417	apic_pr_verbose("ESR value before enabling vector: 0x%08x after: 0x%08x\n",
1418	oldvalue, value);
1419	}
1420	}
1421
1422	#define APIC_IR_REGS APIC_ISR_NR
1423	#define APIC_IR_BITS (APIC_IR_REGS * 32)
1424	#define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
1425
1426	union apic_ir {
1427	unsigned long map[APIC_IR_MAPSIZE];
1428	u32 regs[APIC_IR_REGS];
1429	};
1430
1431	static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
1432	{
1433	int i, bit;
1434
1435	/* Read the IRRs */
1436	for (i = 0; i < APIC_IR_REGS; i++)
1437	irr->regs[i] = apic_read(APIC_IRR + i * 0x10);
1438
1439	/* Read the ISRs */
1440	for (i = 0; i < APIC_IR_REGS; i++)
1441	isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1442
1443	/*
1444	* If the ISR map is not empty. ACK the APIC and run another round
1445	* to verify whether a pending IRR has been unblocked and turned
1446	* into a ISR.
1447	*/
1448	if (!bitmap_empty(src: isr->map, APIC_IR_BITS)) {
1449	/*
1450	* There can be multiple ISR bits set when a high priority
1451	* interrupt preempted a lower priority one. Issue an ACK
1452	* per set bit.
1453	*/
1454	for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1455	apic_eoi();
1456	return true;
1457	}
1458
1459	return !bitmap_empty(src: irr->map, APIC_IR_BITS);
1460	}
1461
1462	/*
1463	* After a crash, we no longer service the interrupts and a pending
1464	* interrupt from previous kernel might still have ISR bit set.
1465	*
1466	* Most probably by now the CPU has serviced that pending interrupt and it
1467	* might not have done the apic_eoi() because it thought, interrupt
1468	* came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
1469	* the ISR bit and cpu thinks it has already serviced the interrupt. Hence
1470	* a vector might get locked. It was noticed for timer irq (vector
1471	* 0x31). Issue an extra EOI to clear ISR.
1472	*
1473	* If there are pending IRR bits they turn into ISR bits after a higher
1474	* priority ISR bit has been acked.
1475	*/
1476	static void apic_pending_intr_clear(void)
1477	{
1478	union apic_ir irr, isr;
1479	unsigned int i;
1480
1481	/* 512 loops are way oversized and give the APIC a chance to obey. */
1482	for (i = 0; i < 512; i++) {
1483	if (!apic_check_and_ack(irr: &irr, isr: &isr))
1484	return;
1485	}
1486	/* Dump the IRR/ISR content if that failed */
1487	pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map);
1488	}
1489
1490	/**
1491	* setup_local_APIC - setup the local APIC
1492	*
1493	* Used to setup local APIC while initializing BSP or bringing up APs.
1494	* Always called with preemption disabled.
1495	*/
1496	static void setup_local_APIC(void)
1497	{
1498	int cpu = smp_processor_id();
1499	unsigned int value;
1500
1501	if (apic_is_disabled) {
1502	disable_ioapic_support();
1503	return;
1504	}
1505
1506	/*
1507	* If this comes from kexec/kcrash the APIC might be enabled in
1508	* SPIV. Soft disable it before doing further initialization.
1509	*/
1510	value = apic_read(APIC_SPIV);
1511	value &= ~APIC_SPIV_APIC_ENABLED;
1512	apic_write(APIC_SPIV, val: value);
1513
1514	#ifdef CONFIG_X86_32
1515	/* Pound the ESR really hard over the head with a big hammer - mbligh */
1516	if (lapic_is_integrated() && apic->disable_esr) {
1517	apic_write(APIC_ESR, 0);
1518	apic_write(APIC_ESR, 0);
1519	apic_write(APIC_ESR, 0);
1520	apic_write(APIC_ESR, 0);
1521	}
1522	#endif
1523	/*
1524	* Intel recommends to set DFR, LDR and TPR before enabling
1525	* an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1526	* document number 292116).
1527	*
1528	* Except for APICs which operate in physical destination mode.
1529	*/
1530	if (apic->init_apic_ldr)
1531	apic->init_apic_ldr();
1532
1533	/*
1534	* Set Task Priority to 'accept all except vectors 0-31'. An APIC
1535	* vector in the 16-31 range could be delivered if TPR == 0, but we
1536	* would think it's an exception and terrible things will happen. We
1537	* never change this later on.
1538	*/
1539	value = apic_read(APIC_TASKPRI);
1540	value &= ~APIC_TPRI_MASK;
1541	value |= 0x10;
1542	apic_write(APIC_TASKPRI, val: value);
1543
1544	/* Clear eventually stale ISR/IRR bits */
1545	apic_pending_intr_clear();
1546
1547	/*
1548	* Now that we are all set up, enable the APIC
1549	*/
1550	value = apic_read(APIC_SPIV);
1551	value &= ~APIC_VECTOR_MASK;
1552	/*
1553	* Enable APIC
1554	*/
1555	value |= APIC_SPIV_APIC_ENABLED;
1556
1557	#ifdef CONFIG_X86_32
1558	/*
1559	* Some unknown Intel IO/APIC (or APIC) errata is biting us with
1560	* certain networking cards. If high frequency interrupts are
1561	* happening on a particular IOAPIC pin, plus the IOAPIC routing
1562	* entry is masked/unmasked at a high rate as well then sooner or
1563	* later IOAPIC line gets 'stuck', no more interrupts are received
1564	* from the device. If focus CPU is disabled then the hang goes
1565	* away, oh well :-(
1566	*
1567	* [ This bug can be reproduced easily with a level-triggered
1568	* PCI Ne2000 networking cards and PII/PIII processors, dual
1569	* BX chipset. ]
1570	*/
1571	/*
1572	* Actually disabling the focus CPU check just makes the hang less
1573	* frequent as it makes the interrupt distribution model be more
1574	* like LRU than MRU (the short-term load is more even across CPUs).
1575	*/
1576
1577	/*
1578	* - enable focus processor (bit==0)
1579	* - 64bit mode always use processor focus
1580	* so no need to set it
1581	*/
1582	value &= ~APIC_SPIV_FOCUS_DISABLED;
1583	#endif
1584
1585	/*
1586	* Set spurious IRQ vector
1587	*/
1588	value |= SPURIOUS_APIC_VECTOR;
1589	apic_write(APIC_SPIV, val: value);
1590
1591	perf_events_lapic_init();
1592
1593	/*
1594	* Set up LVT0, LVT1:
1595	*
1596	* set up through-local-APIC on the boot CPU's LINT0. This is not
1597	* strictly necessary in pure symmetric-IO mode, but sometimes
1598	* we delegate interrupts to the 8259A.
1599	*/
1600	/*
1601	* TODO: set up through-local-APIC from through-I/O-APIC? --macro
1602	*/
1603	value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1604	if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
1605	value = APIC_DM_EXTINT;
1606	apic_pr_verbose("Enabled ExtINT on CPU#%d\n", cpu);
1607	} else {
1608	value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1609	apic_pr_verbose("Masked ExtINT on CPU#%d\n", cpu);
1610	}
1611	apic_write(APIC_LVT0, val: value);
1612
1613	/*
1614	* Only the BSP sees the LINT1 NMI signal by default. This can be
1615	* modified by apic_extnmi= boot option.
1616	*/
1617	if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1618	apic_extnmi == APIC_EXTNMI_ALL)
1619	value = APIC_DM_NMI;
1620	else
1621	value = APIC_DM_NMI | APIC_LVT_MASKED;
1622
1623	/* Is 82489DX ? */
1624	if (!lapic_is_integrated())
1625	value |= APIC_LVT_LEVEL_TRIGGER;
1626	apic_write(APIC_LVT1, val: value);
1627
1628	#ifdef CONFIG_X86_MCE_INTEL
1629	/* Recheck CMCI information after local APIC is up on CPU #0 */
1630	if (!cpu)
1631	cmci_recheck();
1632	#endif
1633	}
1634
1635	static void end_local_APIC_setup(void)
1636	{
1637	lapic_setup_esr();
1638
1639	#ifdef CONFIG_X86_32
1640	{
1641	unsigned int value;
1642	/* Disable the local apic timer */
1643	value = apic_read(APIC_LVTT);
1644	value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1645	apic_write(APIC_LVTT, value);
1646	}
1647	#endif
1648
1649	apic_pm_activate();
1650	}
1651
1652	/*
1653	* APIC setup function for application processors. Called from smpboot.c
1654	*/
1655	void apic_ap_setup(void)
1656	{
1657	setup_local_APIC();
1658	end_local_APIC_setup();
1659	}
1660
1661	static __init void apic_read_boot_cpu_id(bool x2apic)
1662	{
1663	/*
1664	* This can be invoked from check_x2apic() before the APIC has been
1665	* selected. But that code knows for sure that the BIOS enabled
1666	* X2APIC.
1667	*/
1668	if (x2apic) {
1669	boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
1670	boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
1671	} else {
1672	boot_cpu_physical_apicid = read_apic_id();
1673	boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1674	}
1675	topology_register_boot_apic(apic_id: boot_cpu_physical_apicid);
1676	}
1677
1678	#ifdef CONFIG_X86_X2APIC
1679	int x2apic_mode;
1680	EXPORT_SYMBOL_GPL(x2apic_mode);
1681
1682	enum {
1683	X2APIC_OFF,
1684	X2APIC_DISABLED,
1685	/* All states below here have X2APIC enabled */
1686	X2APIC_ON,
1687	X2APIC_ON_LOCKED
1688	};
1689	static int x2apic_state;
1690
1691	static bool x2apic_hw_locked(void)
1692	{
1693	u64 x86_arch_cap_msr;
1694	u64 msr;
1695
1696	x86_arch_cap_msr = x86_read_arch_cap_msr();
1697	if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) {
1698	rdmsrq(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
1699	return (msr & LEGACY_XAPIC_DISABLED);
1700	}
1701	return false;
1702	}
1703
1704	static void __x2apic_disable(void)
1705	{
1706	u64 msr;
1707
1708	if (!boot_cpu_has(X86_FEATURE_APIC))
1709	return;
1710
1711	rdmsrq(MSR_IA32_APICBASE, msr);
1712	if (!(msr & X2APIC_ENABLE))
1713	return;
1714	/* Disable xapic and x2apic first and then reenable xapic mode */
1715	wrmsrq(MSR_IA32_APICBASE, val: msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1716	wrmsrq(MSR_IA32_APICBASE, val: msr & ~X2APIC_ENABLE);
1717	printk_once(KERN_INFO "x2apic disabled\n");
1718	}
1719
1720	static void __x2apic_enable(void)
1721	{
1722	u64 msr;
1723
1724	rdmsrq(MSR_IA32_APICBASE, msr);
1725	if (msr & X2APIC_ENABLE)
1726	return;
1727	wrmsrq(MSR_IA32_APICBASE, val: msr | X2APIC_ENABLE);
1728	printk_once(KERN_INFO "x2apic enabled\n");
1729	}
1730
1731	static int __init setup_nox2apic(char *str)
1732	{
1733	if (x2apic_enabled()) {
1734	u32 apicid = native_apic_msr_read(APIC_ID);
1735
1736	if (apicid >= 255) {
1737	pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1738	apicid);
1739	return 0;
1740	}
1741	if (x2apic_hw_locked()) {
1742	pr_warn("APIC locked in x2apic mode, can't disable\n");
1743	return 0;
1744	}
1745	pr_warn("x2apic already enabled.\n");
1746	__x2apic_disable();
1747	}
1748	setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1749	x2apic_state = X2APIC_DISABLED;
1750	x2apic_mode = 0;
1751	return 0;
1752	}
1753	early_param("nox2apic", setup_nox2apic);
1754
1755	/* Called from cpu_init() to enable x2apic on (secondary) cpus */
1756	void x2apic_setup(void)
1757	{
1758	/*
1759	* Try to make the AP's APIC state match that of the BSP, but if the
1760	* BSP is unlocked and the AP is locked then there is a state mismatch.
1761	* Warn about the mismatch in case a GP fault occurs due to a locked AP
1762	* trying to be turned off.
1763	*/
1764	if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
1765	pr_warn("x2apic lock mismatch between BSP and AP.\n");
1766	/*
1767	* If x2apic is not in ON or LOCKED state, disable it if already enabled
1768	* from BIOS.
1769	*/
1770	if (x2apic_state < X2APIC_ON) {
1771	__x2apic_disable();
1772	return;
1773	}
1774	__x2apic_enable();
1775	}
1776
1777	static __init void apic_set_fixmap(bool read_apic);
1778
1779	static __init void x2apic_disable(void)
1780	{
1781	u32 x2apic_id;
1782
1783	if (x2apic_state < X2APIC_ON)
1784	return;
1785
1786	x2apic_id = read_apic_id();
1787	if (x2apic_id >= 255)
1788	panic(fmt: "Cannot disable x2apic, id: %08x\n", x2apic_id);
1789
1790	if (x2apic_hw_locked()) {
1791	pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
1792	return;
1793	}
1794
1795	__x2apic_disable();
1796
1797	x2apic_mode = 0;
1798	x2apic_state = X2APIC_DISABLED;
1799
1800	/*
1801	* Don't reread the APIC ID as it was already done from
1802	* check_x2apic() and the APIC driver still is a x2APIC variant,
1803	* which fails to do the read after x2APIC was disabled.
1804	*/
1805	apic_set_fixmap(read_apic: false);
1806	}
1807
1808	static __init void x2apic_enable(void)
1809	{
1810	if (x2apic_state != X2APIC_OFF)
1811	return;
1812
1813	x2apic_mode = 1;
1814	x2apic_state = X2APIC_ON;
1815	__x2apic_enable();
1816	}
1817
1818	static __init void try_to_enable_x2apic(int remap_mode)
1819	{
1820	if (x2apic_state == X2APIC_DISABLED)
1821	return;
1822
1823	if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1824	u32 apic_limit = 255;
1825
1826	/*
1827	* Using X2APIC without IR is not architecturally supported
1828	* on bare metal but may be supported in guests.
1829	*/
1830	if (!x86_init.hyper.x2apic_available()) {
1831	pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1832	x2apic_disable();
1833	return;
1834	}
1835
1836	/*
1837	* If the hypervisor supports extended destination ID in
1838	* MSI, that increases the maximum APIC ID that can be
1839	* used for non-remapped IRQ domains.
1840	*/
1841	if (x86_init.hyper.msi_ext_dest_id()) {
1842	virt_ext_dest_id = 1;
1843	apic_limit = 32767;
1844	}
1845
1846	/*
1847	* Without IR, all CPUs can be addressed by IOAPIC/MSI only
1848	* in physical mode, and CPUs with an APIC ID that cannot
1849	* be addressed must not be brought online.
1850	*/
1851	x2apic_set_max_apicid(apicid: apic_limit);
1852	x2apic_phys = 1;
1853	}
1854	x2apic_enable();
1855	}
1856
1857	void __init check_x2apic(void)
1858	{
1859	if (x2apic_enabled()) {
1860	pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1861	x2apic_mode = 1;
1862	if (x2apic_hw_locked())
1863	x2apic_state = X2APIC_ON_LOCKED;
1864	else
1865	x2apic_state = X2APIC_ON;
1866	apic_read_boot_cpu_id(x2apic: true);
1867	} else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1868	x2apic_state = X2APIC_DISABLED;
1869	}
1870	}
1871	#else /* CONFIG_X86_X2APIC */
1872	void __init check_x2apic(void)
1873	{
1874	if (!apic_is_x2apic_enabled())
1875	return;
1876	/*
1877	* Checkme: Can we simply turn off x2APIC here instead of disabling the APIC?
1878	*/
1879	pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
1880	pr_err("Disabling APIC, expect reduced performance and functionality.\n");
1881
1882	apic_is_disabled = true;
1883	setup_clear_cpu_cap(X86_FEATURE_APIC);
1884	}
1885
1886	static inline void try_to_enable_x2apic(int remap_mode) { }
1887	static inline void __x2apic_enable(void) { }
1888	#endif /* !CONFIG_X86_X2APIC */
1889
1890	void __init enable_IR_x2apic(void)
1891	{
1892	unsigned long flags;
1893	int ret, ir_stat;
1894
1895	if (ioapic_is_disabled) {
1896	pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1897	return;
1898	}
1899
1900	ir_stat = irq_remapping_prepare();
1901	if (ir_stat < 0 && !x2apic_supported())
1902	return;
1903
1904	ret = save_ioapic_entries();
1905	if (ret) {
1906	pr_info("Saving IO-APIC state failed: %d\n", ret);
1907	return;
1908	}
1909
1910	local_irq_save(flags);
1911	legacy_pic->mask_all();
1912	mask_ioapic_entries();
1913
1914	/* If irq_remapping_prepare() succeeded, try to enable it */
1915	if (ir_stat >= 0)
1916	ir_stat = irq_remapping_enable();
1917	/* ir_stat contains the remap mode or an error code */
1918	try_to_enable_x2apic(remap_mode: ir_stat);
1919
1920	if (ir_stat < 0)
1921	restore_ioapic_entries();
1922	legacy_pic->restore_mask();
1923	local_irq_restore(flags);
1924	}
1925
1926	#ifdef CONFIG_X86_64
1927	/*
1928	* Detect and enable local APICs on non-SMP boards.
1929	* Original code written by Keir Fraser.
1930	* On AMD64 we trust the BIOS - if it says no APIC it is likely
1931	* not correctly set up (usually the APIC timer won't work etc.)
1932	*/
1933	static bool __init detect_init_APIC(void)
1934	{
1935	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1936	pr_info("No local APIC present\n");
1937	return false;
1938	}
1939
1940	register_lapic_address(APIC_DEFAULT_PHYS_BASE);
1941	return true;
1942	}
1943	#else
1944
1945	static bool __init apic_verify(unsigned long addr)
1946	{
1947	u32 features, h, l;
1948
1949	/*
1950	* The APIC feature bit should now be enabled
1951	* in `cpuid'
1952	*/
1953	features = cpuid_edx(1);
1954	if (!(features & (1 << X86_FEATURE_APIC))) {
1955	pr_warn("Could not enable APIC!\n");
1956	return false;
1957	}
1958	set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1959
1960	/* The BIOS may have set up the APIC at some other address */
1961	if (boot_cpu_data.x86 >= 6) {
1962	rdmsr(MSR_IA32_APICBASE, l, h);
1963	if (l & MSR_IA32_APICBASE_ENABLE)
1964	addr = l & MSR_IA32_APICBASE_BASE;
1965	}
1966
1967	register_lapic_address(addr);
1968	pr_info("Found and enabled local APIC!\n");
1969	return true;
1970	}
1971
1972	bool __init apic_force_enable(unsigned long addr)
1973	{
1974	u32 h, l;
1975
1976	if (apic_is_disabled)
1977	return false;
1978
1979	/*
1980	* Some BIOSes disable the local APIC in the APIC_BASE
1981	* MSR. This can only be done in software for Intel P6 or later
1982	* and AMD K7 (Model > 1) or later.
1983	*/
1984	if (boot_cpu_data.x86 >= 6) {
1985	rdmsr(MSR_IA32_APICBASE, l, h);
1986	if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1987	pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1988	l &= ~MSR_IA32_APICBASE_BASE;
1989	l |= MSR_IA32_APICBASE_ENABLE | addr;
1990	wrmsr(MSR_IA32_APICBASE, l, h);
1991	enabled_via_apicbase = 1;
1992	}
1993	}
1994	return apic_verify(addr);
1995	}
1996
1997	/*
1998	* Detect and initialize APIC
1999	*/
2000	static bool __init detect_init_APIC(void)
2001	{
2002	/* Disabled by kernel option? */
2003	if (apic_is_disabled)
2004	return false;
2005
2006	switch (boot_cpu_data.x86_vendor) {
2007	case X86_VENDOR_AMD:
2008	if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2009	(boot_cpu_data.x86 >= 15))
2010	break;
2011	goto no_apic;
2012	case X86_VENDOR_HYGON:
2013	break;
2014	case X86_VENDOR_INTEL:
2015	if ((boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)) ||
2016	boot_cpu_data.x86_vfm >= INTEL_PENTIUM_PRO)
2017	break;
2018	goto no_apic;
2019	default:
2020	goto no_apic;
2021	}
2022
2023	if (!boot_cpu_has(X86_FEATURE_APIC)) {
2024	/*
2025	* Over-ride BIOS and try to enable the local APIC only if
2026	* "lapic" specified.
2027	*/
2028	if (!force_enable_local_apic) {
2029	pr_info("Local APIC disabled by BIOS -- "
2030	"you can enable it with \"lapic\"\n");
2031	return false;
2032	}
2033	if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
2034	return false;
2035	} else {
2036	if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
2037	return false;
2038	}
2039
2040	apic_pm_activate();
2041
2042	return true;
2043
2044	no_apic:
2045	pr_info("No local APIC present or hardware disabled\n");
2046	return false;
2047	}
2048	#endif
2049
2050	/**
2051	* init_apic_mappings - initialize APIC mappings
2052	*/
2053	void __init init_apic_mappings(void)
2054	{
2055	if (apic_validate_deadline_timer())
2056	pr_info("TSC deadline timer available\n");
2057
2058	if (x2apic_mode)
2059	return;
2060
2061	if (!smp_found_config) {
2062	if (!detect_init_APIC()) {
2063	pr_info("APIC: disable apic facility\n");
2064	apic_disable();
2065	}
2066	}
2067	}
2068
2069	static __init void apic_set_fixmap(bool read_apic)
2070	{
2071	set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
2072	apic_mmio_base = APIC_BASE;
2073	apic_pr_verbose("Mapped APIC to %16lx (%16lx)\n", apic_mmio_base, mp_lapic_addr);
2074	if (read_apic)
2075	apic_read_boot_cpu_id(x2apic: false);
2076	}
2077
2078	void __init register_lapic_address(unsigned long address)
2079	{
2080	/* This should only happen once */
2081	WARN_ON_ONCE(mp_lapic_addr);
2082	mp_lapic_addr = address;
2083
2084	if (!x2apic_mode)
2085	apic_set_fixmap(read_apic: true);
2086	}
2087
2088	/*
2089	* Local APIC interrupts
2090	*/
2091
2092	/*
2093	* Common handling code for spurious_interrupt and spurious_vector entry
2094	* points below. No point in allowing the compiler to inline it twice.
2095	*/
2096	static noinline void handle_spurious_interrupt(u8 vector)
2097	{
2098	u32 v;
2099
2100	trace_spurious_apic_entry(vector);
2101
2102	inc_irq_stat(irq_spurious_count);
2103
2104	/*
2105	* If this is a spurious interrupt then do not acknowledge
2106	*/
2107	if (vector == SPURIOUS_APIC_VECTOR) {
2108	/* See SDM vol 3 */
2109	pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2110	smp_processor_id());
2111	goto out;
2112	}
2113
2114	/*
2115	* If it is a vectored one, verify it's set in the ISR. If set,
2116	* acknowledge it.
2117	*/
2118	v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2119	if (v & (1 << (vector & 0x1f))) {
2120	pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2121	vector, smp_processor_id());
2122	apic_eoi();
2123	} else {
2124	pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2125	vector, smp_processor_id());
2126	}
2127	out:
2128	trace_spurious_apic_exit(vector);
2129	}
2130
2131	/**
2132	* spurious_interrupt - Catch all for interrupts raised on unused vectors
2133	* @regs: Pointer to pt_regs on stack
2134	* @vector: The vector number
2135	*
2136	* This is invoked from ASM entry code to catch all interrupts which
2137	* trigger on an entry which is routed to the common_spurious idtentry
2138	* point.
2139	*/
2140	DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2141	{
2142	handle_spurious_interrupt(vector);
2143	}
2144
2145	DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2146	{
2147	handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2148	}
2149
2150	/*
2151	* This interrupt should never happen with our APIC/SMP architecture
2152	*/
2153	DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2154	{
2155	static const char * const error_interrupt_reason[] = {
2156	"Send CS error", /* APIC Error Bit 0 */
2157	"Receive CS error", /* APIC Error Bit 1 */
2158	"Send accept error", /* APIC Error Bit 2 */
2159	"Receive accept error", /* APIC Error Bit 3 */
2160	"Redirectable IPI", /* APIC Error Bit 4 */
2161	"Send illegal vector", /* APIC Error Bit 5 */
2162	"Received illegal vector", /* APIC Error Bit 6 */
2163	"Illegal register address", /* APIC Error Bit 7 */
2164	};
2165	u32 v, i = 0;
2166
2167	trace_error_apic_entry(ERROR_APIC_VECTOR);
2168
2169	/* First tickle the hardware, only then report what went on. -- REW */
2170	if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2171	apic_write(APIC_ESR, val: 0);
2172	v = apic_read(APIC_ESR);
2173	apic_eoi();
2174	atomic_inc(v: &irq_err_count);
2175
2176	apic_pr_debug("APIC error on CPU%d: %02x", smp_processor_id(), v);
2177
2178	v &= 0xff;
2179	while (v) {
2180	if (v & 0x1)
2181	apic_pr_debug_cont(" : %s", error_interrupt_reason[i]);
2182	i++;
2183	v >>= 1;
2184	}
2185
2186	apic_pr_debug_cont("\n");
2187
2188	trace_error_apic_exit(ERROR_APIC_VECTOR);
2189	}
2190
2191	/**
2192	* connect_bsp_APIC - attach the APIC to the interrupt system
2193	*/
2194	static void __init connect_bsp_APIC(void)
2195	{
2196	#ifdef CONFIG_X86_32
2197	if (pic_mode) {
2198	/*
2199	* Do not trust the local APIC being empty at bootup.
2200	*/
2201	clear_local_APIC();
2202	/*
2203	* PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2204	* local APIC to INT and NMI lines.
2205	*/
2206	apic_pr_verbose("Leaving PIC mode, enabling APIC mode.\n");
2207	imcr_pic_to_apic();
2208	}
2209	#endif
2210	}
2211
2212	/**
2213	* disconnect_bsp_APIC - detach the APIC from the interrupt system
2214	* @virt_wire_setup: indicates, whether virtual wire mode is selected
2215	*
2216	* Virtual wire mode is necessary to deliver legacy interrupts even when the
2217	* APIC is disabled.
2218	*/
2219	void disconnect_bsp_APIC(int virt_wire_setup)
2220	{
2221	unsigned int value;
2222
2223	#ifdef CONFIG_X86_32
2224	if (pic_mode) {
2225	/*
2226	* Put the board back into PIC mode (has an effect only on
2227	* certain older boards). Note that APIC interrupts, including
2228	* IPIs, won't work beyond this point! The only exception are
2229	* INIT IPIs.
2230	*/
2231	apic_pr_verbose("Disabling APIC mode, entering PIC mode.\n");
2232	imcr_apic_to_pic();
2233	return;
2234	}
2235	#endif
2236
2237	/* Go back to Virtual Wire compatibility mode */
2238
2239	/* For the spurious interrupt use vector F, and enable it */
2240	value = apic_read(APIC_SPIV);
2241	value &= ~APIC_VECTOR_MASK;
2242	value |= APIC_SPIV_APIC_ENABLED;
2243	value |= 0xf;
2244	apic_write(APIC_SPIV, val: value);
2245
2246	if (!virt_wire_setup) {
2247	/*
2248	* For LVT0 make it edge triggered, active high,
2249	* external and enabled
2250	*/
2251	value = apic_read(APIC_LVT0);
2252	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2253	APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2254	APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2255	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2256	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2257	apic_write(APIC_LVT0, val: value);
2258	} else {
2259	/* Disable LVT0 */
2260	apic_write(APIC_LVT0, APIC_LVT_MASKED);
2261	}
2262
2263	/*
2264	* For LVT1 make it edge triggered, active high,
2265	* nmi and enabled
2266	*/
2267	value = apic_read(APIC_LVT1);
2268	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2269	APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2270	APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2271	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2272	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2273	apic_write(APIC_LVT1, val: value);
2274	}
2275
2276	void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2277	bool dmar)
2278	{
2279	memset(msg, 0, sizeof(*msg));
2280
2281	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
2282	msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
2283	msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2284
2285	msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
2286	msg->arch_data.vector = cfg->vector;
2287
2288	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2289	/*
2290	* Only the IOMMU itself can use the trick of putting destination
2291	* APIC ID into the high bits of the address. Anything else would
2292	* just be writing to memory if it tried that, and needs IR to
2293	* address APICs which can't be addressed in the normal 32-bit
2294	* address range at 0xFFExxxxx. That is typically just 8 bits, but
2295	* some hypervisors allow the extended destination ID field in bits
2296	* 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2297	*/
2298	if (dmar)
2299	msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2300	else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2301	msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2302	else
2303	WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2304	}
2305
2306	u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
2307	{
2308	u32 dest = msg->arch_addr_lo.destid_0_7;
2309
2310	if (extid)
2311	dest |= msg->arch_addr_hi.destid_8_31 << 8;
2312	return dest;
2313	}
2314	EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
2315
2316	static void __init apic_bsp_up_setup(void)
2317	{
2318	reset_phys_cpu_present_map(apicid: boot_cpu_physical_apicid);
2319	}
2320
2321	/**
2322	* apic_bsp_setup - Setup function for local apic and io-apic
2323	* @upmode: Force UP mode (for APIC_init_uniprocessor)
2324	*/
2325	static void __init apic_bsp_setup(bool upmode)
2326	{
2327	connect_bsp_APIC();
2328	if (upmode)
2329	apic_bsp_up_setup();
2330	setup_local_APIC();
2331
2332	enable_IO_APIC();
2333	end_local_APIC_setup();
2334	irq_remap_enable_fault_handling();
2335	setup_IO_APIC();
2336	lapic_update_legacy_vectors();
2337	}
2338
2339	#ifdef CONFIG_UP_LATE_INIT
2340	void __init up_late_init(void)
2341	{
2342	if (apic_intr_mode == APIC_PIC)
2343	return;
2344
2345	/* Setup local timer */
2346	x86_init.timers.setup_percpu_clockev();
2347	}
2348	#endif
2349
2350	/*
2351	* Power management
2352	*/
2353	#ifdef CONFIG_PM
2354
2355	static struct {
2356	/*
2357	* 'active' is true if the local APIC was enabled by us and
2358	* not the BIOS; this signifies that we are also responsible
2359	* for disabling it before entering apm/acpi suspend
2360	*/
2361	int active;
2362	/* r/w apic fields */
2363	u32 apic_id;
2364	unsigned int apic_taskpri;
2365	unsigned int apic_ldr;
2366	unsigned int apic_dfr;
2367	unsigned int apic_spiv;
2368	unsigned int apic_lvtt;
2369	unsigned int apic_lvtpc;
2370	unsigned int apic_lvt0;
2371	unsigned int apic_lvt1;
2372	unsigned int apic_lvterr;
2373	unsigned int apic_tmict;
2374	unsigned int apic_tdcr;
2375	unsigned int apic_thmr;
2376	unsigned int apic_cmci;
2377	} apic_pm_state;
2378
2379	static int lapic_suspend(void)
2380	{
2381	unsigned long flags;
2382	int maxlvt;
2383
2384	if (!apic_pm_state.active)
2385	return 0;
2386
2387	maxlvt = lapic_get_maxlvt();
2388
2389	apic_pm_state.apic_id = apic_read(APIC_ID);
2390	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2391	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2392	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2393	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2394	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2395	if (maxlvt >= 4)
2396	apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2397	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2398	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2399	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2400	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2401	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2402	#ifdef CONFIG_X86_THERMAL_VECTOR
2403	if (maxlvt >= 5)
2404	apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2405	#endif
2406	#ifdef CONFIG_X86_MCE_INTEL
2407	if (maxlvt >= 6)
2408	apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2409	#endif
2410
2411	local_irq_save(flags);
2412
2413	/*
2414	* Mask IOAPIC before disabling the local APIC to prevent stale IRR
2415	* entries on some implementations.
2416	*/
2417	mask_ioapic_entries();
2418
2419	disable_local_APIC();
2420
2421	irq_remapping_disable();
2422
2423	local_irq_restore(flags);
2424	return 0;
2425	}
2426
2427	static void lapic_resume(void)
2428	{
2429	unsigned int l, h;
2430	unsigned long flags;
2431	int maxlvt;
2432
2433	if (!apic_pm_state.active)
2434	return;
2435
2436	local_irq_save(flags);
2437
2438	/*
2439	* IO-APIC and PIC have their own resume routines.
2440	* We just mask them here to make sure the interrupt
2441	* subsystem is completely quiet while we enable x2apic
2442	* and interrupt-remapping.
2443	*/
2444	mask_ioapic_entries();
2445	legacy_pic->mask_all();
2446
2447	if (x2apic_mode) {
2448	__x2apic_enable();
2449	} else {
2450	/*
2451	* Make sure the APICBASE points to the right address
2452	*
2453	* FIXME! This will be wrong if we ever support suspend on
2454	* SMP! We'll need to do this as part of the CPU restore!
2455	*/
2456	if (boot_cpu_data.x86 >= 6) {
2457	rdmsr(MSR_IA32_APICBASE, l, h);
2458	l &= ~MSR_IA32_APICBASE_BASE;
2459	l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2460	wrmsr(MSR_IA32_APICBASE, low: l, high: h);
2461	}
2462	}
2463
2464	maxlvt = lapic_get_maxlvt();
2465	apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2466	apic_write(APIC_ID, val: apic_pm_state.apic_id);
2467	apic_write(APIC_DFR, val: apic_pm_state.apic_dfr);
2468	apic_write(APIC_LDR, val: apic_pm_state.apic_ldr);
2469	apic_write(APIC_TASKPRI, val: apic_pm_state.apic_taskpri);
2470	apic_write(APIC_SPIV, val: apic_pm_state.apic_spiv);
2471	apic_write(APIC_LVT0, val: apic_pm_state.apic_lvt0);
2472	apic_write(APIC_LVT1, val: apic_pm_state.apic_lvt1);
2473	#ifdef CONFIG_X86_THERMAL_VECTOR
2474	if (maxlvt >= 5)
2475	apic_write(APIC_LVTTHMR, val: apic_pm_state.apic_thmr);
2476	#endif
2477	#ifdef CONFIG_X86_MCE_INTEL
2478	if (maxlvt >= 6)
2479	apic_write(APIC_LVTCMCI, val: apic_pm_state.apic_cmci);
2480	#endif
2481	if (maxlvt >= 4)
2482	apic_write(APIC_LVTPC, val: apic_pm_state.apic_lvtpc);
2483	apic_write(APIC_LVTT, val: apic_pm_state.apic_lvtt);
2484	apic_write(APIC_TDCR, val: apic_pm_state.apic_tdcr);
2485	apic_write(APIC_TMICT, val: apic_pm_state.apic_tmict);
2486	apic_write(APIC_ESR, val: 0);
2487	apic_read(APIC_ESR);
2488	apic_write(APIC_LVTERR, val: apic_pm_state.apic_lvterr);
2489	apic_write(APIC_ESR, val: 0);
2490	apic_read(APIC_ESR);
2491
2492	irq_remapping_reenable(x2apic_mode);
2493
2494	local_irq_restore(flags);
2495	}
2496
2497	/*
2498	* This device has no shutdown method - fully functioning local APICs
2499	* are needed on every CPU up until machine_halt/restart/poweroff.
2500	*/
2501
2502	static struct syscore_ops lapic_syscore_ops = {
2503	.resume = lapic_resume,
2504	.suspend = lapic_suspend,
2505	};
2506
2507	static void apic_pm_activate(void)
2508	{
2509	apic_pm_state.active = 1;
2510	}
2511
2512	static int __init init_lapic_sysfs(void)
2513	{
2514	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2515	if (boot_cpu_has(X86_FEATURE_APIC))
2516	register_syscore_ops(ops: &lapic_syscore_ops);
2517
2518	return 0;
2519	}
2520
2521	/* local apic needs to resume before other devices access its registers. */
2522	core_initcall(init_lapic_sysfs);
2523
2524	#else /* CONFIG_PM */
2525
2526	static void apic_pm_activate(void) { }
2527
2528	#endif /* CONFIG_PM */
2529
2530	#ifdef CONFIG_X86_64
2531
2532	static int multi_checked;
2533	static int multi;
2534
2535	static int set_multi(const struct dmi_system_id *d)
2536	{
2537	if (multi)
2538	return 0;
2539	pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2540	multi = 1;
2541	return 0;
2542	}
2543
2544	static const struct dmi_system_id multi_dmi_table[] = {
2545	{
2546	.callback = set_multi,
2547	.ident = "IBM System Summit2",
2548	.matches = {
2549	DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2550	DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2551	},
2552	},
2553	{}
2554	};
2555
2556	static void dmi_check_multi(void)
2557	{
2558	if (multi_checked)
2559	return;
2560
2561	dmi_check_system(list: multi_dmi_table);
2562	multi_checked = 1;
2563	}
2564
2565	/*
2566	* apic_is_clustered_box() -- Check if we can expect good TSC
2567	*
2568	* Thus far, the major user of this is IBM's Summit2 series:
2569	* Clustered boxes may have unsynced TSC problems if they are
2570	* multi-chassis.
2571	* Use DMI to check them
2572	*/
2573	int apic_is_clustered_box(void)
2574	{
2575	dmi_check_multi();
2576	return multi;
2577	}
2578	#endif
2579
2580	/*
2581	* APIC command line parameters
2582	*/
2583	static int __init setup_nolapic(char *arg)
2584	{
2585	apic_is_disabled = true;
2586	setup_clear_cpu_cap(X86_FEATURE_APIC);
2587	return 0;
2588	}
2589	early_param("nolapic", setup_nolapic);
2590
2591	static int __init parse_lapic_timer_c2_ok(char *arg)
2592	{
2593	local_apic_timer_c2_ok = 1;
2594	return 0;
2595	}
2596	early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2597
2598	static int __init parse_disable_apic_timer(char *arg)
2599	{
2600	disable_apic_timer = 1;
2601	return 0;
2602	}
2603	early_param("noapictimer", parse_disable_apic_timer);
2604
2605	static int __init parse_nolapic_timer(char *arg)
2606	{
2607	disable_apic_timer = 1;
2608	return 0;
2609	}
2610	early_param("nolapic_timer", parse_nolapic_timer);
2611
2612	static int __init apic_set_verbosity(char *arg)
2613	{
2614	if (!arg) {
2615	if (IS_ENABLED(CONFIG_X86_32))
2616	return -EINVAL;
2617
2618	ioapic_is_disabled = false;
2619	return 0;
2620	}
2621
2622	if (strcmp("debug", arg) == 0)
2623	apic_verbosity = APIC_DEBUG;
2624	else if (strcmp("verbose", arg) == 0)
2625	apic_verbosity = APIC_VERBOSE;
2626	#ifdef CONFIG_X86_64
2627	else {
2628	pr_warn("APIC Verbosity level %s not recognised"
2629	" use apic=verbose or apic=debug\n", arg);
2630	return -EINVAL;
2631	}
2632	#endif
2633
2634	return 0;
2635	}
2636	early_param("apic", apic_set_verbosity);
2637
2638	static int __init lapic_insert_resource(void)
2639	{
2640	if (!apic_mmio_base)
2641	return -1;
2642
2643	/* Put local APIC into the resource map. */
2644	lapic_resource.start = apic_mmio_base;
2645	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2646	insert_resource(parent: &iomem_resource, new: &lapic_resource);
2647
2648	return 0;
2649	}
2650
2651	/*
2652	* need call insert after e820__reserve_resources()
2653	* that is using request_resource
2654	*/
2655	late_initcall(lapic_insert_resource);
2656
2657	static int __init apic_set_extnmi(char *arg)
2658	{
2659	if (!arg)
2660	return -EINVAL;
2661
2662	if (!strncmp("all", arg, 3))
2663	apic_extnmi = APIC_EXTNMI_ALL;
2664	else if (!strncmp("none", arg, 4))
2665	apic_extnmi = APIC_EXTNMI_NONE;
2666	else if (!strncmp("bsp", arg, 3))
2667	apic_extnmi = APIC_EXTNMI_BSP;
2668	else {
2669	pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2670	return -EINVAL;
2671	}
2672
2673	return 0;
2674	}
2675	early_param("apic_extnmi", apic_set_extnmi);
2676