1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Local APIC handling, local APIC timers
4	*
5	* (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6	*
7	* Fixes
8	* Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9	* thanks to Eric Gilmore
10	* and Rolf G. Tews
11	* for testing these extensively.
12	* Maciej W. Rozycki : Various updates and fixes.
13	* Mikael Pettersson : Power Management for UP-APIC.
14	* Pavel Machek and
15	* Mikael Pettersson : PM converted to driver model.
16	*/
17
18	#include <linux/perf_event.h>
19	#include <linux/kernel_stat.h>
20	#include <linux/mc146818rtc.h>
21	#include <linux/acpi_pmtmr.h>
22	#include <linux/bitmap.h>
23	#include <linux/clockchips.h>
24	#include <linux/interrupt.h>
25	#include <linux/memblock.h>
26	#include <linux/ftrace.h>
27	#include <linux/ioport.h>
28	#include <linux/export.h>
29	#include <linux/syscore_ops.h>
30	#include <linux/delay.h>
31	#include <linux/timex.h>
32	#include <linux/i8253.h>
33	#include <linux/dmar.h>
34	#include <linux/init.h>
35	#include <linux/cpu.h>
36	#include <linux/dmi.h>
37	#include <linux/smp.h>
38	#include <linux/mm.h>
39	#include <linux/kvm_types.h>
40
41	#include <xen/xen.h>
42
43	#include <asm/trace/irq_vectors.h>
44	#include <asm/irq_remapping.h>
45	#include <asm/pc-conf-reg.h>
46	#include <asm/perf_event.h>
47	#include <asm/x86_init.h>
48	#include <linux/atomic.h>
49	#include <asm/barrier.h>
50	#include <asm/mpspec.h>
51	#include <asm/i8259.h>
52	#include <asm/proto.h>
53	#include <asm/traps.h>
54	#include <asm/apic.h>
55	#include <asm/acpi.h>
56	#include <asm/io_apic.h>
57	#include <asm/desc.h>
58	#include <asm/hpet.h>
59	#include <asm/mtrr.h>
60	#include <asm/time.h>
61	#include <asm/smp.h>
62	#include <asm/mce.h>
63	#include <asm/msr.h>
64	#include <asm/tsc.h>
65	#include <asm/hypervisor.h>
66	#include <asm/cpu_device_id.h>
67	#include <asm/intel-family.h>
68	#include <asm/irq_regs.h>
69	#include <asm/cpu.h>
70
71	#include "local.h"
72
73	/* Processor that is doing the boot up */
74	u32 boot_cpu_physical_apicid __ro_after_init = BAD_APICID;
75	EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
76
77	u8 boot_cpu_apic_version __ro_after_init;
78
79	/*
80	* This variable controls which CPUs receive external NMIs. By default,
81	* external NMIs are delivered only to the BSP.
82	*/
83	static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
84
85	/*
86	* Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
87	*/
88	static bool virt_ext_dest_id __ro_after_init;
89
90	/* For parallel bootup. */
91	unsigned long apic_mmio_base __ro_after_init;
92
93	static inline bool apic_accessible(void)
94	{
95	return x2apic_mode || apic_mmio_base;
96	}
97
98	#ifdef CONFIG_X86_32
99	/* Local APIC was disabled by the BIOS and enabled by the kernel */
100	static int enabled_via_apicbase __ro_after_init;
101
102	/*
103	* Handle interrupt mode configuration register (IMCR).
104	* This register controls whether the interrupt signals
105	* that reach the BSP come from the master PIC or from the
106	* local APIC. Before entering Symmetric I/O Mode, either
107	* the BIOS or the operating system must switch out of
108	* PIC Mode by changing the IMCR.
109	*/
110	static inline void imcr_pic_to_apic(void)
111	{
112	/* NMI and 8259 INTR go through APIC */
113	pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
114	}
115
116	static inline void imcr_apic_to_pic(void)
117	{
118	/* NMI and 8259 INTR go directly to BSP */
119	pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
120	}
121	#endif
122
123	/*
124	* Knob to control our willingness to enable the local APIC.
125	*
126	* +1=force-enable
127	*/
128	static int force_enable_local_apic __initdata;
129
130	/*
131	* APIC command line parameters
132	*/
133	static int __init parse_lapic(char *arg)
134	{
135	if (IS_ENABLED(CONFIG_X86_32) && !arg)
136	force_enable_local_apic = 1;
137	else if (arg && !strncmp(arg, "notscdeadline", 13))
138	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
139	return 0;
140	}
141	early_param("lapic", parse_lapic);
142
143	#ifdef CONFIG_X86_64
144	static int apic_calibrate_pmtmr __initdata;
145	static __init int setup_apicpmtimer(char *s)
146	{
147	apic_calibrate_pmtmr = 1;
148	notsc_setup(NULL);
149	return 1;
150	}
151	__setup("apicpmtimer", setup_apicpmtimer);
152	#endif
153
154	static unsigned long mp_lapic_addr __ro_after_init;
155	bool apic_is_disabled __ro_after_init;
156	/* Disable local APIC timer from the kernel commandline or via dmi quirk */
157	static int disable_apic_timer __initdata;
158	/* Local APIC timer works in C2 */
159	int local_apic_timer_c2_ok __ro_after_init;
160	EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
161
162	/*
163	* Debug level, exported for io_apic.c
164	*/
165	int apic_verbosity __ro_after_init;
166
167	int pic_mode __ro_after_init;
168
169	/* Have we found an MP table */
170	int smp_found_config __ro_after_init;
171
172	static struct resource lapic_resource = {
173	.name = "Local APIC",
174	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
175	};
176
177	/* Measured in ticks per HZ. */
178	unsigned int lapic_timer_period = 0;
179
180	static void apic_pm_activate(void);
181
182	/*
183	* Get the LAPIC version
184	*/
185	static inline int lapic_get_version(void)
186	{
187	return GET_APIC_VERSION(apic_read(APIC_LVR));
188	}
189
190	/*
191	* Check, if the APIC is integrated or a separate chip
192	*/
193	static inline int lapic_is_integrated(void)
194	{
195	return APIC_INTEGRATED(lapic_get_version());
196	}
197
198	/*
199	* Check, whether this is a modern or a first generation APIC
200	*/
201	static int modern_apic(void)
202	{
203	/* AMD systems use old APIC versions, so check the CPU */
204	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
205	boot_cpu_data.x86 >= 0xf)
206	return 1;
207
208	/* Hygon systems use modern APIC */
209	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
210	return 1;
211
212	return lapic_get_version() >= 0x14;
213	}
214
215	/*
216	* right after this call apic become NOOP driven
217	* so apic->write/read doesn't do anything
218	*/
219	static void __init apic_disable(void)
220	{
221	apic_install_driver(driver: &apic_noop);
222	}
223
224	void native_apic_icr_write(u32 low, u32 id)
225	{
226	unsigned long flags;
227
228	local_irq_save(flags);
229	apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id));
230	apic_write(APIC_ICR, val: low);
231	local_irq_restore(flags);
232	}
233
234	u64 native_apic_icr_read(void)
235	{
236	u32 icr1, icr2;
237
238	icr2 = apic_read(APIC_ICR2);
239	icr1 = apic_read(APIC_ICR);
240
241	return icr1 | ((u64)icr2 << 32);
242	}
243
244	/**
245	* lapic_get_maxlvt - get the maximum number of local vector table entries
246	*/
247	int lapic_get_maxlvt(void)
248	{
249	/*
250	* - we always have APIC integrated on 64bit mode
251	* - 82489DXs do not report # of LVT entries
252	*/
253	return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
254	}
255
256	/*
257	* Local APIC timer
258	*/
259
260	/* Clock divisor */
261	#define APIC_DIVISOR 16
262	#define TSC_DIVISOR 8
263
264	/* i82489DX specific */
265	#define I82489DX_BASE_DIVIDER (((0x2) << 18))
266
267	/*
268	* This function sets up the local APIC timer, with a timeout of
269	* 'clocks' APIC bus clock. During calibration we actually call
270	* this function twice on the boot CPU, once with a bogus timeout
271	* value, second time for real. The other (noncalibrating) CPUs
272	* call this function only once, with the real, calibrated value.
273	*
274	* We do reads before writes even if unnecessary, to get around the
275	* P5 APIC double write bug.
276	*/
277	static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
278	{
279	unsigned int lvtt_value, tmp_value;
280
281	lvtt_value = LOCAL_TIMER_VECTOR;
282	if (!oneshot)
283	lvtt_value |= APIC_LVT_TIMER_PERIODIC;
284	else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
285	lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
286
287	/*
288	* The i82489DX APIC uses bit 18 and 19 for the base divider. This
289	* overlaps with bit 18 on integrated APICs, but is not documented
290	* in the SDM. No problem though. i82489DX equipped systems do not
291	* have TSC deadline timer.
292	*/
293	if (!lapic_is_integrated())
294	lvtt_value |= I82489DX_BASE_DIVIDER;
295
296	if (!irqen)
297	lvtt_value |= APIC_LVT_MASKED;
298
299	apic_write(APIC_LVTT, val: lvtt_value);
300
301	if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
302	/*
303	* See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
304	* writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
305	* According to Intel, MFENCE can do the serialization here.
306	*/
307	asm volatile("mfence" : : : "memory");
308	return;
309	}
310
311	/*
312	* Divide PICLK by 16
313	*/
314	tmp_value = apic_read(APIC_TDCR);
315	apic_write(APIC_TDCR,
316	val: (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
317	APIC_TDR_DIV_16);
318
319	if (!oneshot)
320	apic_write(APIC_TMICT, val: clocks / APIC_DIVISOR);
321	}
322
323	/*
324	* Setup extended LVT, AMD specific
325	*
326	* Software should use the LVT offsets the BIOS provides. The offsets
327	* are determined by the subsystems using it like those for MCE
328	* threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
329	* are supported. Beginning with family 10h at least 4 offsets are
330	* available.
331	*
332	* Since the offsets must be consistent for all cores, we keep track
333	* of the LVT offsets in software and reserve the offset for the same
334	* vector also to be used on other cores. An offset is freed by
335	* setting the entry to APIC_EILVT_MASKED.
336	*
337	* If the BIOS is right, there should be no conflicts. Otherwise a
338	* "[Firmware Bug]: ..." error message is generated. However, if
339	* software does not properly determines the offsets, it is not
340	* necessarily a BIOS bug.
341	*/
342
343	static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
344
345	static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
346	{
347	return (old & APIC_EILVT_MASKED)
348	|| (new == APIC_EILVT_MASKED)
349	|| ((new & ~APIC_EILVT_MASKED) == old);
350	}
351
352	static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
353	{
354	unsigned int rsvd, vector;
355
356	if (offset >= APIC_EILVT_NR_MAX)
357	return ~0;
358
359	rsvd = atomic_read(v: &eilvt_offsets[offset]);
360	do {
361	vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
362	if (vector && !eilvt_entry_is_changeable(old: vector, new))
363	/* may not change if vectors are different */
364	return rsvd;
365	} while (!atomic_try_cmpxchg(v: &eilvt_offsets[offset], old: &rsvd, new));
366
367	rsvd = new & ~APIC_EILVT_MASKED;
368	if (rsvd && rsvd != vector)
369	pr_info("LVT offset %d assigned for vector 0x%02x\n",
370	offset, rsvd);
371
372	return new;
373	}
374
375	/*
376	* If mask=1, the LVT entry does not generate interrupts while mask=0
377	* enables the vector. See also the BKDGs. Must be called with
378	* preemption disabled.
379	*/
380
381	int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
382	{
383	unsigned long reg = APIC_EILVTn(offset);
384	unsigned int new, old, reserved;
385
386	new = (mask << 16) | (msg_type << 8) | vector;
387	old = apic_read(reg);
388	reserved = reserve_eilvt_offset(offset, new);
389
390	if (reserved != new) {
391	pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
392	"vector 0x%x, but the register is already in use for "
393	"vector 0x%x on another cpu\n",
394	smp_processor_id(), reg, offset, new, reserved);
395	return -EINVAL;
396	}
397
398	if (!eilvt_entry_is_changeable(old, new)) {
399	pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
400	"vector 0x%x, but the register is already in use for "
401	"vector 0x%x on this cpu\n",
402	smp_processor_id(), reg, offset, new, old);
403	return -EBUSY;
404	}
405
406	apic_write(reg, val: new);
407
408	return 0;
409	}
410	EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
411
412	/*
413	* Program the next event, relative to now
414	*/
415	static int lapic_next_event(unsigned long delta,
416	struct clock_event_device *evt)
417	{
418	apic_write(APIC_TMICT, val: delta);
419	return 0;
420	}
421
422	static int lapic_next_deadline(unsigned long delta,
423	struct clock_event_device *evt)
424	{
425	u64 tsc;
426
427	/* This MSR is special and need a special fence: */
428	weak_wrmsr_fence();
429
430	tsc = rdtsc();
431	wrmsrq(MSR_IA32_TSC_DEADLINE, val: tsc + (((u64) delta) * TSC_DIVISOR));
432	return 0;
433	}
434
435	static int lapic_timer_shutdown(struct clock_event_device *evt)
436	{
437	unsigned int v;
438
439	/* Lapic used as dummy for broadcast ? */
440	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
441	return 0;
442
443	v = apic_read(APIC_LVTT);
444	v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
445	apic_write(APIC_LVTT, val: v);
446
447	/*
448	* Setting APIC_LVT_MASKED (above) should be enough to tell
449	* the hardware that this timer will never fire. But AMD
450	* erratum 411 and some Intel CPU behavior circa 2024 say
451	* otherwise. Time for belt and suspenders programming: mask
452	* the timer _and_ zero the counter registers:
453	*/
454	if (v & APIC_LVT_TIMER_TSCDEADLINE)
455	wrmsrq(MSR_IA32_TSC_DEADLINE, val: 0);
456	else
457	apic_write(APIC_TMICT, val: 0);
458
459	return 0;
460	}
461
462	static inline int
463	lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
464	{
465	/* Lapic used as dummy for broadcast ? */
466	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
467	return 0;
468
469	__setup_APIC_LVTT(clocks: lapic_timer_period, oneshot, irqen: 1);
470	return 0;
471	}
472
473	static int lapic_timer_set_periodic(struct clock_event_device *evt)
474	{
475	return lapic_timer_set_periodic_oneshot(evt, oneshot: false);
476	}
477
478	static int lapic_timer_set_oneshot(struct clock_event_device *evt)
479	{
480	return lapic_timer_set_periodic_oneshot(evt, oneshot: true);
481	}
482
483	/*
484	* Local APIC timer broadcast function
485	*/
486	static void lapic_timer_broadcast(const struct cpumask *mask)
487	{
488	#ifdef CONFIG_SMP
489	__apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
490	#endif
491	}
492
493
494	/*
495	* The local apic timer can be used for any function which is CPU local.
496	*/
497	static struct clock_event_device lapic_clockevent = {
498	.name = "lapic",
499	.features = CLOCK_EVT_FEAT_PERIODIC |
500	CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
501	| CLOCK_EVT_FEAT_DUMMY,
502	.shift = 32,
503	.set_state_shutdown = lapic_timer_shutdown,
504	.set_state_periodic = lapic_timer_set_periodic,
505	.set_state_oneshot = lapic_timer_set_oneshot,
506	.set_state_oneshot_stopped = lapic_timer_shutdown,
507	.set_next_event = lapic_next_event,
508	.broadcast = lapic_timer_broadcast,
509	.rating = 100,
510	.irq = -1,
511	};
512	static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
513
514	static const struct x86_cpu_id deadline_match[] __initconst = {
515	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X, 0x2, 0x2, 0x3a), /* EP */
516	X86_MATCH_VFM_STEPS(INTEL_HASWELL_X, 0x4, 0x4, 0x0f), /* EX */
517
518	X86_MATCH_VFM(INTEL_BROADWELL_X, 0x0b000020),
519
520	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x2, 0x2, 0x00000011),
521	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x3, 0x3, 0x0700000e),
522	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x4, 0x4, 0x0f00000c),
523	X86_MATCH_VFM_STEPS(INTEL_BROADWELL_D, 0x5, 0x5, 0x0e000003),
524
525	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x3, 0x3, 0x01000136),
526	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x4, 0x4, 0x02000014),
527	X86_MATCH_VFM_STEPS(INTEL_SKYLAKE_X, 0x5, 0xf, 0),
528
529	X86_MATCH_VFM(INTEL_HASWELL, 0x22),
530	X86_MATCH_VFM(INTEL_HASWELL_L, 0x20),
531	X86_MATCH_VFM(INTEL_HASWELL_G, 0x17),
532
533	X86_MATCH_VFM(INTEL_BROADWELL, 0x25),
534	X86_MATCH_VFM(INTEL_BROADWELL_G, 0x17),
535
536	X86_MATCH_VFM(INTEL_SKYLAKE_L, 0xb2),
537	X86_MATCH_VFM(INTEL_SKYLAKE, 0xb2),
538
539	X86_MATCH_VFM(INTEL_KABYLAKE_L, 0x52),
540	X86_MATCH_VFM(INTEL_KABYLAKE, 0x52),
541
542	{},
543	};
544
545	static __init bool apic_validate_deadline_timer(void)
546	{
547	const struct x86_cpu_id *m;
548	u32 rev;
549
550	if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
551	return false;
552	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
553	return true;
554
555	m = x86_match_cpu(match: deadline_match);
556	if (!m)
557	return true;
558
559	rev = (u32)m->driver_data;
560
561	if (boot_cpu_data.microcode >= rev)
562	return true;
563
564	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
565	pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
566	"please update microcode to version: 0x%x (or later)\n", rev);
567	return false;
568	}
569
570	/*
571	* Setup the local APIC timer for this CPU. Copy the initialized values
572	* of the boot CPU and register the clock event in the framework.
573	*/
574	static void setup_APIC_timer(void)
575	{
576	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
577
578	if (this_cpu_has(X86_FEATURE_ARAT)) {
579	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
580	/* Make LAPIC timer preferable over percpu HPET */
581	lapic_clockevent.rating = 150;
582	}
583
584	memcpy(levt, &lapic_clockevent, sizeof(*levt));
585	levt->cpumask = cpumask_of(smp_processor_id());
586
587	if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
588	levt->name = "lapic-deadline";
589	levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
590	CLOCK_EVT_FEAT_DUMMY);
591	levt->set_next_event = lapic_next_deadline;
592	clockevents_config_and_register(dev: levt,
593	freq: tsc_khz * (1000 / TSC_DIVISOR),
594	min_delta: 0xF, max_delta: ~0UL);
595	} else
596	clockevents_register_device(dev: levt);
597
598	apic_update_vector(smp_processor_id(), LOCAL_TIMER_VECTOR, set: true);
599	}
600
601	/*
602	* Install the updated TSC frequency from recalibration at the TSC
603	* deadline clockevent devices.
604	*/
605	static void __lapic_update_tsc_freq(void *info)
606	{
607	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
608
609	if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
610	return;
611
612	clockevents_update_freq(ce: levt, freq: tsc_khz * (1000 / TSC_DIVISOR));
613	}
614
615	void lapic_update_tsc_freq(void)
616	{
617	/*
618	* The clockevent device's ->mult and ->shift can both be
619	* changed. In order to avoid races, schedule the frequency
620	* update code on each CPU.
621	*/
622	on_each_cpu(func: __lapic_update_tsc_freq, NULL, wait: 0);
623	}
624
625	/*
626	* In this functions we calibrate APIC bus clocks to the external timer.
627	*
628	* We want to do the calibration only once since we want to have local timer
629	* irqs synchronous. CPUs connected by the same APIC bus have the very same bus
630	* frequency.
631	*
632	* This was previously done by reading the PIT/HPET and waiting for a wrap
633	* around to find out, that a tick has elapsed. I have a box, where the PIT
634	* readout is broken, so it never gets out of the wait loop again. This was
635	* also reported by others.
636	*
637	* Monitoring the jiffies value is inaccurate and the clockevents
638	* infrastructure allows us to do a simple substitution of the interrupt
639	* handler.
640	*
641	* The calibration routine also uses the pm_timer when possible, as the PIT
642	* happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
643	* back to normal later in the boot process).
644	*/
645
646	#define LAPIC_CAL_LOOPS (HZ/10)
647
648	static __initdata int lapic_cal_loops = -1;
649	static __initdata long lapic_cal_t1, lapic_cal_t2;
650	static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
651	static __initdata u32 lapic_cal_pm1, lapic_cal_pm2;
652	static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
653
654	/*
655	* Temporary interrupt handler and polled calibration function.
656	*/
657	static void __init lapic_cal_handler(struct clock_event_device *dev)
658	{
659	unsigned long long tsc = 0;
660	long tapic = apic_read(APIC_TMCCT);
661	u32 pm = acpi_pm_read_early();
662
663	if (boot_cpu_has(X86_FEATURE_TSC))
664	tsc = rdtsc();
665
666	switch (lapic_cal_loops++) {
667	case 0:
668	lapic_cal_t1 = tapic;
669	lapic_cal_tsc1 = tsc;
670	lapic_cal_pm1 = pm;
671	lapic_cal_j1 = jiffies;
672	break;
673
674	case LAPIC_CAL_LOOPS:
675	lapic_cal_t2 = tapic;
676	lapic_cal_tsc2 = tsc;
677	if (pm < lapic_cal_pm1)
678	pm += ACPI_PM_OVRRUN;
679	lapic_cal_pm2 = pm;
680	lapic_cal_j2 = jiffies;
681	break;
682	}
683	}
684
685	static int __init
686	calibrate_by_pmtimer(u32 deltapm, long *delta, long *deltatsc)
687	{
688	const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
689	const long pm_thresh = pm_100ms / 100;
690	unsigned long mult;
691	u64 res;
692
693	#ifndef CONFIG_X86_PM_TIMER
694	return -1;
695	#endif
696
697	apic_pr_verbose("... PM-Timer delta = %u\n", deltapm);
698
699	/* Check, if the PM timer is available */
700	if (!deltapm)
701	return -1;
702
703	mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, shift_constant: 22);
704
705	if (deltapm > (pm_100ms - pm_thresh) &&
706	deltapm < (pm_100ms + pm_thresh)) {
707	apic_pr_verbose("... PM-Timer result ok\n");
708	return 0;
709	}
710
711	res = (((u64)deltapm) * mult) >> 22;
712	do_div(res, 1000000);
713	pr_warn("APIC calibration not consistent with PM-Timer: %ldms instead of 100ms\n",
714	(long)res);
715
716	/* Correct the lapic counter value */
717	res = (((u64)(*delta)) * pm_100ms);
718	do_div(res, deltapm);
719	pr_info("APIC delta adjusted to PM-Timer: "
720	"%lu (%ld)\n", (unsigned long)res, *delta);
721	*delta = (long)res;
722
723	/* Correct the tsc counter value */
724	if (boot_cpu_has(X86_FEATURE_TSC)) {
725	res = (((u64)(*deltatsc)) * pm_100ms);
726	do_div(res, deltapm);
727	apic_pr_verbose("TSC delta adjusted to PM-Timer: %lu (%ld)\n",
728	(unsigned long)res, *deltatsc);
729	*deltatsc = (long)res;
730	}
731
732	return 0;
733	}
734
735	static int __init lapic_init_clockevent(void)
736	{
737	if (!lapic_timer_period)
738	return -1;
739
740	/* Calculate the scaled math multiplication factor */
741	lapic_clockevent.mult = div_sc(ticks: lapic_timer_period/APIC_DIVISOR,
742	TICK_NSEC, shift: lapic_clockevent.shift);
743	lapic_clockevent.max_delta_ns =
744	clockevent_delta2ns(latch: 0x7FFFFFFF, evt: &lapic_clockevent);
745	lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
746	lapic_clockevent.min_delta_ns =
747	clockevent_delta2ns(latch: 0xF, evt: &lapic_clockevent);
748	lapic_clockevent.min_delta_ticks = 0xF;
749
750	return 0;
751	}
752
753	bool __init apic_needs_pit(void)
754	{
755	/*
756	* If the frequencies are not known, PIT is required for both TSC
757	* and apic timer calibration.
758	*/
759	if (!tsc_khz || !cpu_khz)
760	return true;
761
762	/* Is there an APIC at all or is it disabled? */
763	if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
764	return true;
765
766	/*
767	* If interrupt delivery mode is legacy PIC or virtual wire without
768	* configuration, the local APIC timer won't be set up. Make sure
769	* that the PIT is initialized.
770	*/
771	if (apic_intr_mode == APIC_PIC ||
772	apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
773	return true;
774
775	/* Virt guests may lack ARAT, but still have DEADLINE */
776	if (!boot_cpu_has(X86_FEATURE_ARAT))
777	return true;
778
779	/* Deadline timer is based on TSC so no further PIT action required */
780	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
781	return false;
782
783	/* APIC timer disabled? */
784	if (disable_apic_timer)
785	return true;
786	/*
787	* The APIC timer frequency is known already, no PIT calibration
788	* required. If unknown, let the PIT be initialized.
789	*/
790	return lapic_timer_period == 0;
791	}
792
793	static int __init calibrate_APIC_clock(void)
794	{
795	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
796	u64 tsc_perj = 0, tsc_start = 0;
797	long delta_tsc_khz, bus_khz;
798	unsigned long jif_start;
799	unsigned long deltaj;
800	long delta, deltatsc;
801	int pm_referenced = 0;
802
803	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
804	return 0;
805
806	/*
807	* Check if lapic timer has already been calibrated by platform
808	* specific routine, such as tsc calibration code. If so just fill
809	* in the clockevent structure and return.
810	*/
811	if (!lapic_init_clockevent()) {
812	apic_pr_verbose("lapic timer already calibrated %d\n", lapic_timer_period);
813	/*
814	* Direct calibration methods must have an always running
815	* local APIC timer, no need for broadcast timer.
816	*/
817	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
818	return 0;
819	}
820
821	apic_pr_verbose("Using local APIC timer interrupts. Calibrating APIC timer ...\n");
822
823	/*
824	* There are platforms w/o global clockevent devices. Instead of
825	* making the calibration conditional on that, use a polling based
826	* approach everywhere.
827	*/
828	local_irq_disable();
829
830	/*
831	* Setup the APIC counter to maximum. There is no way the lapic
832	* can underflow in the 100ms detection time frame
833	*/
834	__setup_APIC_LVTT(clocks: 0xffffffff, oneshot: 0, irqen: 0);
835
836	/*
837	* Methods to terminate the calibration loop:
838	* 1) Global clockevent if available (jiffies)
839	* 2) TSC if available and frequency is known
840	*/
841	jif_start = READ_ONCE(jiffies);
842
843	if (tsc_khz) {
844	tsc_start = rdtsc();
845	tsc_perj = div_u64(dividend: (u64)tsc_khz * 1000, HZ);
846	}
847
848	/*
849	* Enable interrupts so the tick can fire, if a global
850	* clockevent device is available
851	*/
852	local_irq_enable();
853
854	while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
855	/* Wait for a tick to elapse */
856	while (1) {
857	if (tsc_khz) {
858	u64 tsc_now = rdtsc();
859	if ((tsc_now - tsc_start) >= tsc_perj) {
860	tsc_start += tsc_perj;
861	break;
862	}
863	} else {
864	unsigned long jif_now = READ_ONCE(jiffies);
865
866	if (time_after(jif_now, jif_start)) {
867	jif_start = jif_now;
868	break;
869	}
870	}
871	cpu_relax();
872	}
873
874	/* Invoke the calibration routine */
875	local_irq_disable();
876	lapic_cal_handler(NULL);
877	local_irq_enable();
878	}
879
880	local_irq_disable();
881
882	/* Build delta t1-t2 as apic timer counts down */
883	delta = lapic_cal_t1 - lapic_cal_t2;
884	apic_pr_verbose("... lapic delta = %ld\n", delta);
885
886	deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
887
888	/* we trust the PM based calibration if possible */
889	pm_referenced = !calibrate_by_pmtimer(deltapm: lapic_cal_pm2 - lapic_cal_pm1,
890	delta: &delta, deltatsc: &deltatsc);
891
892	lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
893	lapic_init_clockevent();
894
895	apic_pr_verbose("..... delta %ld\n", delta);
896	apic_pr_verbose("..... mult: %u\n", lapic_clockevent.mult);
897	apic_pr_verbose("..... calibration result: %u\n", lapic_timer_period);
898
899	if (boot_cpu_has(X86_FEATURE_TSC)) {
900	delta_tsc_khz = (deltatsc * HZ) / (1000 * LAPIC_CAL_LOOPS);
901
902	apic_pr_verbose("..... CPU clock speed is %ld.%03ld MHz.\n",
903	delta_tsc_khz / 1000, delta_tsc_khz % 1000);
904	}
905
906	bus_khz = (long)lapic_timer_period * HZ / 1000;
907	apic_pr_verbose("..... host bus clock speed is %ld.%03ld MHz.\n",
908	bus_khz / 1000, bus_khz % 1000);
909
910	/*
911	* Do a sanity check on the APIC calibration result
912	*/
913	if (lapic_timer_period < (1000000 / HZ)) {
914	local_irq_enable();
915	pr_warn("APIC frequency too slow, disabling apic timer\n");
916	return -1;
917	}
918
919	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
920
921	/*
922	* PM timer calibration failed or not turned on so lets try APIC
923	* timer based calibration, if a global clockevent device is
924	* available.
925	*/
926	if (!pm_referenced && global_clock_event) {
927	apic_pr_verbose("... verify APIC timer\n");
928
929	/*
930	* Setup the apic timer manually
931	*/
932	levt->event_handler = lapic_cal_handler;
933	lapic_timer_set_periodic(evt: levt);
934	lapic_cal_loops = -1;
935
936	/* Let the interrupts run */
937	local_irq_enable();
938
939	while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
940	cpu_relax();
941
942	/* Stop the lapic timer */
943	local_irq_disable();
944	lapic_timer_shutdown(evt: levt);
945
946	/* Jiffies delta */
947	deltaj = lapic_cal_j2 - lapic_cal_j1;
948	apic_pr_verbose("... jiffies delta = %lu\n", deltaj);
949
950	/* Check, if the jiffies result is consistent */
951	if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
952	apic_pr_verbose("... jiffies result ok\n");
953	else
954	levt->features |= CLOCK_EVT_FEAT_DUMMY;
955	}
956	local_irq_enable();
957
958	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
959	pr_warn("APIC timer disabled due to verification failure\n");
960	return -1;
961	}
962
963	return 0;
964	}
965
966	/*
967	* Setup the boot APIC
968	*
969	* Calibrate and verify the result.
970	*/
971	void __init setup_boot_APIC_clock(void)
972	{
973	/*
974	* The local apic timer can be disabled via the kernel
975	* commandline or from the CPU detection code. Register the lapic
976	* timer as a dummy clock event source on SMP systems, so the
977	* broadcast mechanism is used. On UP systems simply ignore it.
978	*/
979	if (disable_apic_timer) {
980	pr_info("Disabling APIC timer\n");
981	/* No broadcast on UP ! */
982	if (num_possible_cpus() > 1) {
983	lapic_clockevent.mult = 1;
984	setup_APIC_timer();
985	}
986	return;
987	}
988
989	if (calibrate_APIC_clock()) {
990	/* No broadcast on UP ! */
991	if (num_possible_cpus() > 1)
992	setup_APIC_timer();
993	return;
994	}
995
996	/*
997	* If nmi_watchdog is set to IO_APIC, we need the
998	* PIT/HPET going. Otherwise register lapic as a dummy
999	* device.
1000	*/
1001	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
1002
1003	/* Setup the lapic or request the broadcast */
1004	setup_APIC_timer();
1005	amd_e400_c1e_apic_setup();
1006	}
1007
1008	void setup_secondary_APIC_clock(void)
1009	{
1010	setup_APIC_timer();
1011	amd_e400_c1e_apic_setup();
1012	}
1013
1014	/*
1015	* The guts of the apic timer interrupt
1016	*/
1017	static void local_apic_timer_interrupt(void)
1018	{
1019	struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1020
1021	/*
1022	* Normally we should not be here till LAPIC has been initialized but
1023	* in some cases like kdump, its possible that there is a pending LAPIC
1024	* timer interrupt from previous kernel's context and is delivered in
1025	* new kernel the moment interrupts are enabled.
1026	*
1027	* Interrupts are enabled early and LAPIC is setup much later, hence
1028	* its possible that when we get here evt->event_handler is NULL.
1029	* Check for event_handler being NULL and discard the interrupt as
1030	* spurious.
1031	*/
1032	if (!evt->event_handler) {
1033	pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1034	smp_processor_id());
1035	/* Switch it off */
1036	lapic_timer_shutdown(evt);
1037	return;
1038	}
1039
1040	/*
1041	* the NMI deadlock-detector uses this.
1042	*/
1043	inc_irq_stat(apic_timer_irqs);
1044
1045	evt->event_handler(evt);
1046	}
1047
1048	/*
1049	* Local APIC timer interrupt. This is the most natural way for doing
1050	* local interrupts, but local timer interrupts can be emulated by
1051	* broadcast interrupts too. [in case the hw doesn't support APIC timers]
1052	*
1053	* [ if a single-CPU system runs an SMP kernel then we call the local
1054	* interrupt as well. Thus we cannot inline the local irq ... ]
1055	*/
1056	DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1057	{
1058	struct pt_regs *old_regs = set_irq_regs(regs);
1059
1060	apic_eoi();
1061	trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1062	local_apic_timer_interrupt();
1063	trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1064
1065	set_irq_regs(old_regs);
1066	}
1067
1068	/*
1069	* Local APIC start and shutdown
1070	*/
1071
1072	/**
1073	* clear_local_APIC - shutdown the local APIC
1074	*
1075	* This is called, when a CPU is disabled and before rebooting, so the state of
1076	* the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1077	* leftovers during boot.
1078	*/
1079	void clear_local_APIC(void)
1080	{
1081	int maxlvt;
1082	u32 v;
1083
1084	if (!apic_accessible())
1085	return;
1086
1087	maxlvt = lapic_get_maxlvt();
1088	/*
1089	* Masking an LVT entry can trigger a local APIC error
1090	* if the vector is zero. Mask LVTERR first to prevent this.
1091	*/
1092	if (maxlvt >= 3) {
1093	v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1094	apic_write(APIC_LVTERR, val: v | APIC_LVT_MASKED);
1095	}
1096	/*
1097	* Careful: we have to set masks only first to deassert
1098	* any level-triggered sources.
1099	*/
1100	v = apic_read(APIC_LVTT);
1101	apic_write(APIC_LVTT, val: v | APIC_LVT_MASKED);
1102	v = apic_read(APIC_LVT0);
1103	apic_write(APIC_LVT0, val: v | APIC_LVT_MASKED);
1104	v = apic_read(APIC_LVT1);
1105	apic_write(APIC_LVT1, val: v | APIC_LVT_MASKED);
1106	if (maxlvt >= 4) {
1107	v = apic_read(APIC_LVTPC);
1108	apic_write(APIC_LVTPC, val: v | APIC_LVT_MASKED);
1109	}
1110
1111	/* lets not touch this if we didn't frob it */
1112	#ifdef CONFIG_X86_THERMAL_VECTOR
1113	if (maxlvt >= 5) {
1114	v = apic_read(APIC_LVTTHMR);
1115	apic_write(APIC_LVTTHMR, val: v | APIC_LVT_MASKED);
1116	}
1117	#endif
1118	#ifdef CONFIG_X86_MCE_INTEL
1119	if (maxlvt >= 6) {
1120	v = apic_read(APIC_LVTCMCI);
1121	if (!(v & APIC_LVT_MASKED))
1122	apic_write(APIC_LVTCMCI, val: v | APIC_LVT_MASKED);
1123	}
1124	#endif
1125
1126	/*
1127	* Clean APIC state for other OSs:
1128	*/
1129	apic_write(APIC_LVTT, APIC_LVT_MASKED);
1130	apic_write(APIC_LVT0, APIC_LVT_MASKED);
1131	apic_write(APIC_LVT1, APIC_LVT_MASKED);
1132	if (maxlvt >= 3)
1133	apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1134	if (maxlvt >= 4)
1135	apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1136
1137	/* Integrated APIC (!82489DX) ? */
1138	if (lapic_is_integrated()) {
1139	if (maxlvt > 3)
1140	/* Clear ESR due to Pentium errata 3AP and 11AP */
1141	apic_write(APIC_ESR, val: 0);
1142	apic_read(APIC_ESR);
1143	}
1144	}
1145
1146	/**
1147	* apic_soft_disable - Clears and software disables the local APIC on hotplug
1148	*
1149	* Contrary to disable_local_APIC() this does not touch the enable bit in
1150	* MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1151	* bus would require a hardware reset as the APIC would lose track of bus
1152	* arbitration. On systems with FSB delivery APICBASE could be disabled,
1153	* but it has to be guaranteed that no interrupt is sent to the APIC while
1154	* in that state and it's not clear from the SDM whether it still responds
1155	* to INIT/SIPI messages. Stay on the safe side and use software disable.
1156	*/
1157	void apic_soft_disable(void)
1158	{
1159	u32 value;
1160
1161	clear_local_APIC();
1162
1163	/* Soft disable APIC (implies clearing of registers for 82489DX!). */
1164	value = apic_read(APIC_SPIV);
1165	value &= ~APIC_SPIV_APIC_ENABLED;
1166	apic_write(APIC_SPIV, val: value);
1167	}
1168
1169	/**
1170	* disable_local_APIC - clear and disable the local APIC
1171	*/
1172	void disable_local_APIC(void)
1173	{
1174	if (!apic_accessible())
1175	return;
1176
1177	if (apic->teardown)
1178	apic->teardown();
1179
1180	apic_soft_disable();
1181
1182	#ifdef CONFIG_X86_32
1183	/*
1184	* When LAPIC was disabled by the BIOS and enabled by the kernel,
1185	* restore the disabled state.
1186	*/
1187	if (enabled_via_apicbase) {
1188	unsigned int l, h;
1189
1190	rdmsr(MSR_IA32_APICBASE, l, h);
1191	l &= ~MSR_IA32_APICBASE_ENABLE;
1192	wrmsr(MSR_IA32_APICBASE, l, h);
1193	}
1194	#endif
1195	}
1196
1197	/*
1198	* If Linux enabled the LAPIC against the BIOS default disable it down before
1199	* re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1200	* not power-off. Additionally clear all LVT entries before disable_local_APIC
1201	* for the case where Linux didn't enable the LAPIC.
1202	*/
1203	void lapic_shutdown(void)
1204	{
1205	unsigned long flags;
1206
1207	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1208	return;
1209
1210	local_irq_save(flags);
1211
1212	#ifdef CONFIG_X86_32
1213	if (!enabled_via_apicbase)
1214	clear_local_APIC();
1215	else
1216	#endif
1217	disable_local_APIC();
1218
1219
1220	local_irq_restore(flags);
1221	}
1222
1223	/**
1224	* sync_Arb_IDs - synchronize APIC bus arbitration IDs
1225	*/
1226	void __init sync_Arb_IDs(void)
1227	{
1228	/*
1229	* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1230	* needed on AMD.
1231	*/
1232	if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1233	return;
1234
1235	/*
1236	* Wait for idle.
1237	*/
1238	apic_wait_icr_idle();
1239
1240	apic_pr_debug("Synchronizing Arb IDs.\n");
1241	apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG | APIC_DM_INIT);
1242	}
1243
1244	enum apic_intr_mode_id apic_intr_mode __ro_after_init;
1245
1246	static int __init __apic_intr_mode_select(void)
1247	{
1248	/* Check kernel option */
1249	if (apic_is_disabled) {
1250	pr_info("APIC disabled via kernel command line\n");
1251	return APIC_PIC;
1252	}
1253
1254	/* Check BIOS */
1255	#ifdef CONFIG_X86_64
1256	/* On 64-bit, the APIC must be integrated, Check local APIC only */
1257	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1258	apic_is_disabled = true;
1259	pr_info("APIC disabled by BIOS\n");
1260	return APIC_PIC;
1261	}
1262	#else
1263	/* On 32-bit, the APIC may be integrated APIC or 82489DX */
1264
1265	/* Neither 82489DX nor integrated APIC ? */
1266	if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
1267	apic_is_disabled = true;
1268	return APIC_PIC;
1269	}
1270
1271	/* If the BIOS pretends there is an integrated APIC ? */
1272	if (!boot_cpu_has(X86_FEATURE_APIC) &&
1273	APIC_INTEGRATED(boot_cpu_apic_version)) {
1274	apic_is_disabled = true;
1275	pr_err(FW_BUG "Local APIC not detected, force emulation\n");
1276	return APIC_PIC;
1277	}
1278	#endif
1279
1280	/* Check MP table or ACPI MADT configuration */
1281	if (!smp_found_config) {
1282	disable_ioapic_support();
1283	if (!acpi_lapic) {
1284	pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1285	return APIC_VIRTUAL_WIRE_NO_CONFIG;
1286	}
1287	return APIC_VIRTUAL_WIRE;
1288	}
1289
1290	#ifdef CONFIG_SMP
1291	/* If SMP should be disabled, then really disable it! */
1292	if (!setup_max_cpus) {
1293	pr_info("APIC: SMP mode deactivated\n");
1294	return APIC_SYMMETRIC_IO_NO_ROUTING;
1295	}
1296	#endif
1297
1298	return APIC_SYMMETRIC_IO;
1299	}
1300
1301	/* Select the interrupt delivery mode for the BSP */
1302	void __init apic_intr_mode_select(void)
1303	{
1304	apic_intr_mode = __apic_intr_mode_select();
1305	}
1306
1307	/*
1308	* An initial setup of the virtual wire mode.
1309	*/
1310	void __init init_bsp_APIC(void)
1311	{
1312	unsigned int value;
1313
1314	/*
1315	* Don't do the setup now if we have a SMP BIOS as the
1316	* through-I/O-APIC virtual wire mode might be active.
1317	*/
1318	if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1319	return;
1320
1321	/*
1322	* Do not trust the local APIC being empty at bootup.
1323	*/
1324	clear_local_APIC();
1325
1326	/*
1327	* Enable APIC.
1328	*/
1329	value = apic_read(APIC_SPIV);
1330	value &= ~APIC_VECTOR_MASK;
1331	value |= APIC_SPIV_APIC_ENABLED;
1332
1333	#ifdef CONFIG_X86_32
1334	/* This bit is reserved on P4/Xeon and should be cleared */
1335	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1336	(boot_cpu_data.x86 == 15))
1337	value &= ~APIC_SPIV_FOCUS_DISABLED;
1338	else
1339	#endif
1340	value |= APIC_SPIV_FOCUS_DISABLED;
1341	value |= SPURIOUS_APIC_VECTOR;
1342	apic_write(APIC_SPIV, val: value);
1343
1344	/*
1345	* Set up the virtual wire mode.
1346	*/
1347	apic_write(APIC_LVT0, APIC_DM_EXTINT);
1348	value = APIC_DM_NMI;
1349	if (!lapic_is_integrated()) /* 82489DX */
1350	value |= APIC_LVT_LEVEL_TRIGGER;
1351	if (apic_extnmi == APIC_EXTNMI_NONE)
1352	value |= APIC_LVT_MASKED;
1353	apic_write(APIC_LVT1, val: value);
1354	}
1355
1356	static void __init apic_bsp_setup(bool upmode);
1357
1358	/* Init the interrupt delivery mode for the BSP */
1359	void __init apic_intr_mode_init(void)
1360	{
1361	bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
1362
1363	switch (apic_intr_mode) {
1364	case APIC_PIC:
1365	pr_info("APIC: Keep in PIC mode(8259)\n");
1366	return;
1367	case APIC_VIRTUAL_WIRE:
1368	pr_info("APIC: Switch to virtual wire mode setup\n");
1369	break;
1370	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1371	pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1372	upmode = true;
1373	break;
1374	case APIC_SYMMETRIC_IO:
1375	pr_info("APIC: Switch to symmetric I/O mode setup\n");
1376	break;
1377	case APIC_SYMMETRIC_IO_NO_ROUTING:
1378	pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1379	break;
1380	}
1381
1382	x86_64_probe_apic();
1383
1384	if (x86_platform.apic_post_init)
1385	x86_platform.apic_post_init();
1386
1387	apic_bsp_setup(upmode);
1388	}
1389
1390	static void lapic_setup_esr(void)
1391	{
1392	unsigned int oldvalue, value, maxlvt;
1393
1394	if (!lapic_is_integrated()) {
1395	pr_info("No ESR for 82489DX.\n");
1396	return;
1397	}
1398
1399	if (apic->disable_esr) {
1400	/*
1401	* Something untraceable is creating bad interrupts on
1402	* secondary quads ... for the moment, just leave the
1403	* ESR disabled - we can't do anything useful with the
1404	* errors anyway - mbligh
1405	*/
1406	pr_info("Leaving ESR disabled.\n");
1407	return;
1408	}
1409
1410	maxlvt = lapic_get_maxlvt();
1411	if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1412	apic_write(APIC_ESR, val: 0);
1413	oldvalue = apic_read(APIC_ESR);
1414
1415	/* enables sending errors */
1416	value = ERROR_APIC_VECTOR;
1417	apic_write(APIC_LVTERR, val: value);
1418
1419	/*
1420	* spec says clear errors after enabling vector.
1421	*/
1422	if (maxlvt > 3)
1423	apic_write(APIC_ESR, val: 0);
1424	value = apic_read(APIC_ESR);
1425	if (value != oldvalue) {
1426	apic_pr_verbose("ESR value before enabling vector: 0x%08x after: 0x%08x\n",
1427	oldvalue, value);
1428	}
1429	}
1430
1431	#define APIC_IR_REGS APIC_ISR_NR
1432	#define APIC_IR_BITS (APIC_IR_REGS * 32)
1433	#define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
1434
1435	union apic_ir {
1436	unsigned long map[APIC_IR_MAPSIZE];
1437	u32 regs[APIC_IR_REGS];
1438	};
1439
1440	static bool apic_check_and_eoi_isr(union apic_ir *isr)
1441	{
1442	int i, bit;
1443
1444	/* Read the ISRs */
1445	for (i = 0; i < APIC_IR_REGS; i++)
1446	isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1447
1448	/* If the ISR map empty, nothing to do here. */
1449	if (bitmap_empty(src: isr->map, APIC_IR_BITS))
1450	return true;
1451
1452	/*
1453	* There can be multiple ISR bits set when a high priority
1454	* interrupt preempted a lower priority one. Issue an EOI for each
1455	* set bit. The priority traversal order does not matter as there
1456	* can't be new ISR bits raised at this point. What matters is that
1457	* an EOI is issued for each ISR bit.
1458	*/
1459	for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1460	apic_eoi();
1461
1462	/* Reread the ISRs, they should be empty now */
1463	for (i = 0; i < APIC_IR_REGS; i++)
1464	isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1465
1466	return bitmap_empty(src: isr->map, APIC_IR_BITS);
1467	}
1468
1469	/*
1470	* If a CPU services an interrupt and crashes before issuing EOI to the
1471	* local APIC, the corresponding ISR bit is still set when the crashing CPU
1472	* jumps into a crash kernel. Read the ISR and issue an EOI for each set
1473	* bit to acknowledge it as otherwise these slots would be locked forever
1474	* waiting for an EOI.
1475	*
1476	* If there are pending bits in the IRR, then they won't be converted into
1477	* ISR bits as the CPU has interrupts disabled. They will be delivered once
1478	* the CPU enables interrupts and there is nothing which can prevent that.
1479	*
1480	* In the worst case this results in spurious interrupt warnings.
1481	*/
1482	static void apic_clear_isr(void)
1483	{
1484	union apic_ir ir;
1485	unsigned int i;
1486
1487	if (!apic_check_and_eoi_isr(isr: &ir))
1488	pr_warn("APIC: Stale ISR: %256pb\n", ir.map);
1489
1490	for (i = 0; i < APIC_IR_REGS; i++)
1491	ir.regs[i] = apic_read(APIC_IRR + i * 0x10);
1492
1493	if (!bitmap_empty(src: ir.map, APIC_IR_BITS))
1494	pr_warn("APIC: Stale IRR: %256pb\n", ir.map);
1495	}
1496
1497	/**
1498	* setup_local_APIC - setup the local APIC
1499	*
1500	* Used to setup local APIC while initializing BSP or bringing up APs.
1501	* Always called with preemption disabled.
1502	*/
1503	static void setup_local_APIC(void)
1504	{
1505	int cpu = smp_processor_id();
1506	unsigned int value;
1507
1508	if (apic_is_disabled) {
1509	disable_ioapic_support();
1510	return;
1511	}
1512
1513	if (apic->setup)
1514	apic->setup();
1515
1516	/*
1517	* If this comes from kexec/kcrash the APIC might be enabled in
1518	* SPIV. Soft disable it before doing further initialization.
1519	*/
1520	value = apic_read(APIC_SPIV);
1521	value &= ~APIC_SPIV_APIC_ENABLED;
1522	apic_write(APIC_SPIV, val: value);
1523
1524	#ifdef CONFIG_X86_32
1525	/* Pound the ESR really hard over the head with a big hammer - mbligh */
1526	if (lapic_is_integrated() && apic->disable_esr) {
1527	apic_write(APIC_ESR, 0);
1528	apic_write(APIC_ESR, 0);
1529	apic_write(APIC_ESR, 0);
1530	apic_write(APIC_ESR, 0);
1531	}
1532	#endif
1533	/*
1534	* Intel recommends to set DFR, LDR and TPR before enabling
1535	* an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1536	* document number 292116).
1537	*
1538	* Except for APICs which operate in physical destination mode.
1539	*/
1540	if (apic->init_apic_ldr)
1541	apic->init_apic_ldr();
1542
1543	/*
1544	* Set Task Priority to 'accept all except vectors 0-31'. An APIC
1545	* vector in the 16-31 range could be delivered if TPR == 0, but we
1546	* would think it's an exception and terrible things will happen. We
1547	* never change this later on.
1548	*/
1549	value = apic_read(APIC_TASKPRI);
1550	value &= ~APIC_TPRI_MASK;
1551	value |= 0x10;
1552	apic_write(APIC_TASKPRI, val: value);
1553
1554	apic_clear_isr();
1555
1556	/*
1557	* Now that we are all set up, enable the APIC
1558	*/
1559	value = apic_read(APIC_SPIV);
1560	value &= ~APIC_VECTOR_MASK;
1561	/*
1562	* Enable APIC
1563	*/
1564	value |= APIC_SPIV_APIC_ENABLED;
1565
1566	#ifdef CONFIG_X86_32
1567	/*
1568	* Some unknown Intel IO/APIC (or APIC) errata is biting us with
1569	* certain networking cards. If high frequency interrupts are
1570	* happening on a particular IOAPIC pin, plus the IOAPIC routing
1571	* entry is masked/unmasked at a high rate as well then sooner or
1572	* later IOAPIC line gets 'stuck', no more interrupts are received
1573	* from the device. If focus CPU is disabled then the hang goes
1574	* away, oh well :-(
1575	*
1576	* [ This bug can be reproduced easily with a level-triggered
1577	* PCI Ne2000 networking cards and PII/PIII processors, dual
1578	* BX chipset. ]
1579	*/
1580	/*
1581	* Actually disabling the focus CPU check just makes the hang less
1582	* frequent as it makes the interrupt distribution model be more
1583	* like LRU than MRU (the short-term load is more even across CPUs).
1584	*/
1585
1586	/*
1587	* - enable focus processor (bit==0)
1588	* - 64bit mode always use processor focus
1589	* so no need to set it
1590	*/
1591	value &= ~APIC_SPIV_FOCUS_DISABLED;
1592	#endif
1593
1594	/*
1595	* Set spurious IRQ vector
1596	*/
1597	value |= SPURIOUS_APIC_VECTOR;
1598	apic_write(APIC_SPIV, val: value);
1599
1600	perf_events_lapic_init();
1601
1602	/*
1603	* Set up LVT0, LVT1:
1604	*
1605	* set up through-local-APIC on the boot CPU's LINT0. This is not
1606	* strictly necessary in pure symmetric-IO mode, but sometimes
1607	* we delegate interrupts to the 8259A.
1608	*/
1609	/*
1610	* TODO: set up through-local-APIC from through-I/O-APIC? --macro
1611	*/
1612	value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1613	if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
1614	value = APIC_DM_EXTINT;
1615	apic_pr_verbose("Enabled ExtINT on CPU#%d\n", cpu);
1616	} else {
1617	value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1618	apic_pr_verbose("Masked ExtINT on CPU#%d\n", cpu);
1619	}
1620	apic_write(APIC_LVT0, val: value);
1621
1622	/*
1623	* Only the BSP sees the LINT1 NMI signal by default. This can be
1624	* modified by apic_extnmi= boot option.
1625	*/
1626	if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1627	apic_extnmi == APIC_EXTNMI_ALL)
1628	value = APIC_DM_NMI;
1629	else
1630	value = APIC_DM_NMI | APIC_LVT_MASKED;
1631
1632	/* Is 82489DX ? */
1633	if (!lapic_is_integrated())
1634	value |= APIC_LVT_LEVEL_TRIGGER;
1635	apic_write(APIC_LVT1, val: value);
1636
1637	#ifdef CONFIG_X86_MCE_INTEL
1638	/* Recheck CMCI information after local APIC is up on CPU #0 */
1639	if (!cpu)
1640	cmci_recheck();
1641	#endif
1642	}
1643
1644	static void end_local_APIC_setup(void)
1645	{
1646	lapic_setup_esr();
1647
1648	#ifdef CONFIG_X86_32
1649	{
1650	unsigned int value;
1651	/* Disable the local apic timer */
1652	value = apic_read(APIC_LVTT);
1653	value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1654	apic_write(APIC_LVTT, value);
1655	}
1656	#endif
1657
1658	apic_pm_activate();
1659	}
1660
1661	/*
1662	* APIC setup function for application processors. Called from smpboot.c
1663	*/
1664	void apic_ap_setup(void)
1665	{
1666	setup_local_APIC();
1667	end_local_APIC_setup();
1668	}
1669
1670	static __init void apic_read_boot_cpu_id(bool x2apic)
1671	{
1672	/*
1673	* This can be invoked from check_x2apic() before the APIC has been
1674	* selected. But that code knows for sure that the BIOS enabled
1675	* X2APIC.
1676	*/
1677	if (x2apic) {
1678	boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
1679	boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
1680	} else {
1681	boot_cpu_physical_apicid = read_apic_id();
1682	boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1683	}
1684	topology_register_boot_apic(apic_id: boot_cpu_physical_apicid);
1685	}
1686
1687	#ifdef CONFIG_X86_X2APIC
1688	int x2apic_mode;
1689	EXPORT_SYMBOL_GPL(x2apic_mode);
1690
1691	enum {
1692	X2APIC_OFF,
1693	X2APIC_DISABLED,
1694	/* All states below here have X2APIC enabled */
1695	X2APIC_ON,
1696	X2APIC_ON_LOCKED
1697	};
1698	static int x2apic_state;
1699
1700	static bool x2apic_hw_locked(void)
1701	{
1702	u64 x86_arch_cap_msr;
1703	u64 msr;
1704
1705	x86_arch_cap_msr = x86_read_arch_cap_msr();
1706	if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) {
1707	rdmsrq(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
1708	return (msr & LEGACY_XAPIC_DISABLED);
1709	}
1710	return false;
1711	}
1712
1713	static void __x2apic_disable(void)
1714	{
1715	u64 msr;
1716
1717	if (!boot_cpu_has(X86_FEATURE_APIC))
1718	return;
1719
1720	rdmsrq(MSR_IA32_APICBASE, msr);
1721	if (!(msr & X2APIC_ENABLE))
1722	return;
1723	/* Disable xapic and x2apic first and then reenable xapic mode */
1724	wrmsrq(MSR_IA32_APICBASE, val: msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1725	wrmsrq(MSR_IA32_APICBASE, val: msr & ~X2APIC_ENABLE);
1726	printk_once(KERN_INFO "x2apic disabled\n");
1727	}
1728
1729	static void __x2apic_enable(void)
1730	{
1731	u64 msr;
1732
1733	rdmsrq(MSR_IA32_APICBASE, msr);
1734	if (msr & X2APIC_ENABLE)
1735	return;
1736	wrmsrq(MSR_IA32_APICBASE, val: msr | X2APIC_ENABLE);
1737	printk_once(KERN_INFO "x2apic enabled\n");
1738	}
1739
1740	static int __init setup_nox2apic(char *str)
1741	{
1742	if (x2apic_enabled()) {
1743	u32 apicid = native_apic_msr_read(APIC_ID);
1744
1745	if (apicid >= 255) {
1746	pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1747	apicid);
1748	return 0;
1749	}
1750	if (x2apic_hw_locked()) {
1751	pr_warn("APIC locked in x2apic mode, can't disable\n");
1752	return 0;
1753	}
1754	pr_warn("x2apic already enabled.\n");
1755	__x2apic_disable();
1756	}
1757	setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1758	x2apic_state = X2APIC_DISABLED;
1759	x2apic_mode = 0;
1760	return 0;
1761	}
1762	early_param("nox2apic", setup_nox2apic);
1763
1764	/* Called from cpu_init() to enable x2apic on (secondary) cpus */
1765	void x2apic_setup(void)
1766	{
1767	/*
1768	* Try to make the AP's APIC state match that of the BSP, but if the
1769	* BSP is unlocked and the AP is locked then there is a state mismatch.
1770	* Warn about the mismatch in case a GP fault occurs due to a locked AP
1771	* trying to be turned off.
1772	*/
1773	if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
1774	pr_warn("x2apic lock mismatch between BSP and AP.\n");
1775	/*
1776	* If x2apic is not in ON or LOCKED state, disable it if already enabled
1777	* from BIOS.
1778	*/
1779	if (x2apic_state < X2APIC_ON) {
1780	__x2apic_disable();
1781	return;
1782	}
1783	__x2apic_enable();
1784	}
1785
1786	static __init void apic_set_fixmap(bool read_apic);
1787
1788	static __init void x2apic_disable(void)
1789	{
1790	u32 x2apic_id;
1791
1792	if (x2apic_state < X2APIC_ON)
1793	return;
1794
1795	x2apic_id = read_apic_id();
1796	if (x2apic_id >= 255)
1797	panic(fmt: "Cannot disable x2apic, id: %08x\n", x2apic_id);
1798
1799	if (x2apic_hw_locked()) {
1800	pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
1801	return;
1802	}
1803
1804	__x2apic_disable();
1805
1806	x2apic_mode = 0;
1807	x2apic_state = X2APIC_DISABLED;
1808
1809	/*
1810	* Don't reread the APIC ID as it was already done from
1811	* check_x2apic() and the APIC driver still is a x2APIC variant,
1812	* which fails to do the read after x2APIC was disabled.
1813	*/
1814	apic_set_fixmap(read_apic: false);
1815	}
1816
1817	static __init void x2apic_enable(void)
1818	{
1819	if (x2apic_state != X2APIC_OFF)
1820	return;
1821
1822	x2apic_mode = 1;
1823	x2apic_state = X2APIC_ON;
1824	__x2apic_enable();
1825	}
1826
1827	static __init void try_to_enable_x2apic(int remap_mode)
1828	{
1829	if (x2apic_state == X2APIC_DISABLED)
1830	return;
1831
1832	if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1833	u32 apic_limit = 255;
1834
1835	/*
1836	* Using X2APIC without IR is not architecturally supported
1837	* on bare metal but may be supported in guests.
1838	*/
1839	if (!x86_init.hyper.x2apic_available()) {
1840	pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1841	x2apic_disable();
1842	return;
1843	}
1844
1845	/*
1846	* If the hypervisor supports extended destination ID in
1847	* MSI, that increases the maximum APIC ID that can be
1848	* used for non-remapped IRQ domains.
1849	*/
1850	if (x86_init.hyper.msi_ext_dest_id()) {
1851	virt_ext_dest_id = 1;
1852	apic_limit = 32767;
1853	}
1854
1855	/*
1856	* Without IR, all CPUs can be addressed by IOAPIC/MSI only
1857	* in physical mode, and CPUs with an APIC ID that cannot
1858	* be addressed must not be brought online.
1859	*/
1860	x2apic_set_max_apicid(apicid: apic_limit);
1861	x2apic_phys = 1;
1862	}
1863	x2apic_enable();
1864	}
1865
1866	void __init check_x2apic(void)
1867	{
1868	if (x2apic_enabled()) {
1869	pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1870	x2apic_mode = 1;
1871	if (x2apic_hw_locked())
1872	x2apic_state = X2APIC_ON_LOCKED;
1873	else
1874	x2apic_state = X2APIC_ON;
1875	apic_read_boot_cpu_id(x2apic: true);
1876	} else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1877	x2apic_state = X2APIC_DISABLED;
1878	}
1879	}
1880	#else /* CONFIG_X86_X2APIC */
1881	void __init check_x2apic(void)
1882	{
1883	if (!apic_is_x2apic_enabled())
1884	return;
1885	/*
1886	* Checkme: Can we simply turn off x2APIC here instead of disabling the APIC?
1887	*/
1888	pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
1889	pr_err("Disabling APIC, expect reduced performance and functionality.\n");
1890
1891	apic_is_disabled = true;
1892	setup_clear_cpu_cap(X86_FEATURE_APIC);
1893	}
1894
1895	static inline void try_to_enable_x2apic(int remap_mode) { }
1896	static inline void __x2apic_enable(void) { }
1897	#endif /* !CONFIG_X86_X2APIC */
1898
1899	void __init enable_IR_x2apic(void)
1900	{
1901	unsigned long flags;
1902	int ret, ir_stat;
1903
1904	if (ioapic_is_disabled) {
1905	pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1906	return;
1907	}
1908
1909	ir_stat = irq_remapping_prepare();
1910	if (ir_stat < 0 && !x2apic_supported())
1911	return;
1912
1913	ret = save_ioapic_entries();
1914	if (ret) {
1915	pr_info("Saving IO-APIC state failed: %d\n", ret);
1916	return;
1917	}
1918
1919	local_irq_save(flags);
1920	legacy_pic->mask_all();
1921	mask_ioapic_entries();
1922
1923	/* If irq_remapping_prepare() succeeded, try to enable it */
1924	if (ir_stat >= 0)
1925	ir_stat = irq_remapping_enable();
1926	/* ir_stat contains the remap mode or an error code */
1927	try_to_enable_x2apic(remap_mode: ir_stat);
1928
1929	if (ir_stat < 0)
1930	restore_ioapic_entries();
1931	legacy_pic->restore_mask();
1932	local_irq_restore(flags);
1933	}
1934
1935	#ifdef CONFIG_X86_64
1936	/*
1937	* Detect and enable local APICs on non-SMP boards.
1938	* Original code written by Keir Fraser.
1939	* On AMD64 we trust the BIOS - if it says no APIC it is likely
1940	* not correctly set up (usually the APIC timer won't work etc.)
1941	*/
1942	static bool __init detect_init_APIC(void)
1943	{
1944	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1945	pr_info("No local APIC present\n");
1946	return false;
1947	}
1948
1949	register_lapic_address(APIC_DEFAULT_PHYS_BASE);
1950	return true;
1951	}
1952	#else
1953
1954	static bool __init apic_verify(unsigned long addr)
1955	{
1956	u32 features, h, l;
1957
1958	/*
1959	* The APIC feature bit should now be enabled
1960	* in `cpuid'
1961	*/
1962	features = cpuid_edx(1);
1963	if (!(features & (1 << X86_FEATURE_APIC))) {
1964	pr_warn("Could not enable APIC!\n");
1965	return false;
1966	}
1967	set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1968
1969	/* The BIOS may have set up the APIC at some other address */
1970	if (boot_cpu_data.x86 >= 6) {
1971	rdmsr(MSR_IA32_APICBASE, l, h);
1972	if (l & MSR_IA32_APICBASE_ENABLE)
1973	addr = l & MSR_IA32_APICBASE_BASE;
1974	}
1975
1976	register_lapic_address(addr);
1977	pr_info("Found and enabled local APIC!\n");
1978	return true;
1979	}
1980
1981	bool __init apic_force_enable(unsigned long addr)
1982	{
1983	u32 h, l;
1984
1985	if (apic_is_disabled)
1986	return false;
1987
1988	/*
1989	* Some BIOSes disable the local APIC in the APIC_BASE
1990	* MSR. This can only be done in software for Intel P6 or later
1991	* and AMD K7 (Model > 1) or later.
1992	*/
1993	if (boot_cpu_data.x86 >= 6) {
1994	rdmsr(MSR_IA32_APICBASE, l, h);
1995	if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1996	pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1997	l &= ~MSR_IA32_APICBASE_BASE;
1998	l |= MSR_IA32_APICBASE_ENABLE | addr;
1999	wrmsr(MSR_IA32_APICBASE, l, h);
2000	enabled_via_apicbase = 1;
2001	}
2002	}
2003	return apic_verify(addr);
2004	}
2005
2006	/*
2007	* Detect and initialize APIC
2008	*/
2009	static bool __init detect_init_APIC(void)
2010	{
2011	/* Disabled by kernel option? */
2012	if (apic_is_disabled)
2013	return false;
2014
2015	switch (boot_cpu_data.x86_vendor) {
2016	case X86_VENDOR_AMD:
2017	if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2018	(boot_cpu_data.x86 >= 15))
2019	break;
2020	goto no_apic;
2021	case X86_VENDOR_HYGON:
2022	break;
2023	case X86_VENDOR_INTEL:
2024	if ((boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)) ||
2025	boot_cpu_data.x86_vfm >= INTEL_PENTIUM_PRO)
2026	break;
2027	goto no_apic;
2028	default:
2029	goto no_apic;
2030	}
2031
2032	if (!boot_cpu_has(X86_FEATURE_APIC)) {
2033	/*
2034	* Over-ride BIOS and try to enable the local APIC only if
2035	* "lapic" specified.
2036	*/
2037	if (!force_enable_local_apic) {
2038	pr_info("Local APIC disabled by BIOS -- "
2039	"you can enable it with \"lapic\"\n");
2040	return false;
2041	}
2042	if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
2043	return false;
2044	} else {
2045	if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
2046	return false;
2047	}
2048
2049	apic_pm_activate();
2050
2051	return true;
2052
2053	no_apic:
2054	pr_info("No local APIC present or hardware disabled\n");
2055	return false;
2056	}
2057	#endif
2058
2059	/**
2060	* init_apic_mappings - initialize APIC mappings
2061	*/
2062	void __init init_apic_mappings(void)
2063	{
2064	if (apic_validate_deadline_timer())
2065	pr_info("TSC deadline timer available\n");
2066
2067	if (x2apic_mode)
2068	return;
2069
2070	if (!smp_found_config) {
2071	if (!detect_init_APIC()) {
2072	pr_info("APIC: disable apic facility\n");
2073	apic_disable();
2074	}
2075	}
2076	}
2077
2078	static __init void apic_set_fixmap(bool read_apic)
2079	{
2080	set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
2081	apic_mmio_base = APIC_BASE;
2082	apic_pr_verbose("Mapped APIC to %16lx (%16lx)\n", apic_mmio_base, mp_lapic_addr);
2083	if (read_apic)
2084	apic_read_boot_cpu_id(x2apic: false);
2085	}
2086
2087	void __init register_lapic_address(unsigned long address)
2088	{
2089	/* This should only happen once */
2090	WARN_ON_ONCE(mp_lapic_addr);
2091	mp_lapic_addr = address;
2092
2093	if (!x2apic_mode)
2094	apic_set_fixmap(read_apic: true);
2095	}
2096
2097	/*
2098	* Local APIC interrupts
2099	*/
2100
2101	/*
2102	* Common handling code for spurious_interrupt and spurious_vector entry
2103	* points below. No point in allowing the compiler to inline it twice.
2104	*/
2105	static noinline void handle_spurious_interrupt(u8 vector)
2106	{
2107	u32 v;
2108
2109	trace_spurious_apic_entry(vector);
2110
2111	inc_irq_stat(irq_spurious_count);
2112
2113	/*
2114	* If this is a spurious interrupt then do not acknowledge
2115	*/
2116	if (vector == SPURIOUS_APIC_VECTOR) {
2117	/* See SDM vol 3 */
2118	pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2119	smp_processor_id());
2120	goto out;
2121	}
2122
2123	/*
2124	* If it is a vectored one, verify it's set in the ISR. If set,
2125	* acknowledge it.
2126	*/
2127	v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2128	if (v & (1 << (vector & 0x1f))) {
2129	pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2130	vector, smp_processor_id());
2131	apic_eoi();
2132	} else {
2133	pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2134	vector, smp_processor_id());
2135	}
2136	out:
2137	trace_spurious_apic_exit(vector);
2138	}
2139
2140	/**
2141	* spurious_interrupt - Catch all for interrupts raised on unused vectors
2142	* @regs: Pointer to pt_regs on stack
2143	* @vector: The vector number
2144	*
2145	* This is invoked from ASM entry code to catch all interrupts which
2146	* trigger on an entry which is routed to the common_spurious idtentry
2147	* point.
2148	*/
2149	DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2150	{
2151	handle_spurious_interrupt(vector);
2152	}
2153
2154	DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2155	{
2156	handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2157	}
2158
2159	/*
2160	* This interrupt should never happen with our APIC/SMP architecture
2161	*/
2162	DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2163	{
2164	static const char * const error_interrupt_reason[] = {
2165	"Send CS error", /* APIC Error Bit 0 */
2166	"Receive CS error", /* APIC Error Bit 1 */
2167	"Send accept error", /* APIC Error Bit 2 */
2168	"Receive accept error", /* APIC Error Bit 3 */
2169	"Redirectable IPI", /* APIC Error Bit 4 */
2170	"Send illegal vector", /* APIC Error Bit 5 */
2171	"Received illegal vector", /* APIC Error Bit 6 */
2172	"Illegal register address", /* APIC Error Bit 7 */
2173	};
2174	u32 v, i = 0;
2175
2176	trace_error_apic_entry(ERROR_APIC_VECTOR);
2177
2178	/* First tickle the hardware, only then report what went on. -- REW */
2179	if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2180	apic_write(APIC_ESR, val: 0);
2181	v = apic_read(APIC_ESR);
2182	apic_eoi();
2183	atomic_inc(v: &irq_err_count);
2184
2185	apic_pr_debug("APIC error on CPU%d: %02x", smp_processor_id(), v);
2186
2187	v &= 0xff;
2188	while (v) {
2189	if (v & 0x1)
2190	apic_pr_debug_cont(" : %s", error_interrupt_reason[i]);
2191	i++;
2192	v >>= 1;
2193	}
2194
2195	apic_pr_debug_cont("\n");
2196
2197	trace_error_apic_exit(ERROR_APIC_VECTOR);
2198	}
2199
2200	/**
2201	* connect_bsp_APIC - attach the APIC to the interrupt system
2202	*/
2203	static void __init connect_bsp_APIC(void)
2204	{
2205	#ifdef CONFIG_X86_32
2206	if (pic_mode) {
2207	/*
2208	* Do not trust the local APIC being empty at bootup.
2209	*/
2210	clear_local_APIC();
2211	/*
2212	* PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2213	* local APIC to INT and NMI lines.
2214	*/
2215	apic_pr_verbose("Leaving PIC mode, enabling APIC mode.\n");
2216	imcr_pic_to_apic();
2217	}
2218	#endif
2219	}
2220
2221	/**
2222	* disconnect_bsp_APIC - detach the APIC from the interrupt system
2223	* @virt_wire_setup: indicates, whether virtual wire mode is selected
2224	*
2225	* Virtual wire mode is necessary to deliver legacy interrupts even when the
2226	* APIC is disabled.
2227	*/
2228	void disconnect_bsp_APIC(int virt_wire_setup)
2229	{
2230	unsigned int value;
2231
2232	#ifdef CONFIG_X86_32
2233	if (pic_mode) {
2234	/*
2235	* Put the board back into PIC mode (has an effect only on
2236	* certain older boards). Note that APIC interrupts, including
2237	* IPIs, won't work beyond this point! The only exception are
2238	* INIT IPIs.
2239	*/
2240	apic_pr_verbose("Disabling APIC mode, entering PIC mode.\n");
2241	imcr_apic_to_pic();
2242	return;
2243	}
2244	#endif
2245
2246	/* Go back to Virtual Wire compatibility mode */
2247
2248	/* For the spurious interrupt use vector F, and enable it */
2249	value = apic_read(APIC_SPIV);
2250	value &= ~APIC_VECTOR_MASK;
2251	value |= APIC_SPIV_APIC_ENABLED;
2252	value |= 0xf;
2253	apic_write(APIC_SPIV, val: value);
2254
2255	if (!virt_wire_setup) {
2256	/*
2257	* For LVT0 make it edge triggered, active high,
2258	* external and enabled
2259	*/
2260	value = apic_read(APIC_LVT0);
2261	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2262	APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2263	APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2264	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2265	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2266	apic_write(APIC_LVT0, val: value);
2267	} else {
2268	/* Disable LVT0 */
2269	apic_write(APIC_LVT0, APIC_LVT_MASKED);
2270	}
2271
2272	/*
2273	* For LVT1 make it edge triggered, active high,
2274	* nmi and enabled
2275	*/
2276	value = apic_read(APIC_LVT1);
2277	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2278	APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2279	APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2280	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2281	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2282	apic_write(APIC_LVT1, val: value);
2283	}
2284
2285	void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2286	bool dmar)
2287	{
2288	memset(msg, 0, sizeof(*msg));
2289
2290	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
2291	msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
2292	msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2293
2294	msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
2295	msg->arch_data.vector = cfg->vector;
2296
2297	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2298	/*
2299	* Only the IOMMU itself can use the trick of putting destination
2300	* APIC ID into the high bits of the address. Anything else would
2301	* just be writing to memory if it tried that, and needs IR to
2302	* address APICs which can't be addressed in the normal 32-bit
2303	* address range at 0xFFExxxxx. That is typically just 8 bits, but
2304	* some hypervisors allow the extended destination ID field in bits
2305	* 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2306	*/
2307	if (dmar)
2308	msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2309	else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2310	msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2311	else
2312	WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2313	}
2314
2315	u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
2316	{
2317	u32 dest = msg->arch_addr_lo.destid_0_7;
2318
2319	if (extid)
2320	dest |= msg->arch_addr_hi.destid_8_31 << 8;
2321	return dest;
2322	}
2323	EXPORT_SYMBOL_FOR_KVM(x86_msi_msg_get_destid);
2324
2325	static void __init apic_bsp_up_setup(void)
2326	{
2327	reset_phys_cpu_present_map(apicid: boot_cpu_physical_apicid);
2328	}
2329
2330	/**
2331	* apic_bsp_setup - Setup function for local apic and io-apic
2332	* @upmode: Force UP mode (for APIC_init_uniprocessor)
2333	*/
2334	static void __init apic_bsp_setup(bool upmode)
2335	{
2336	connect_bsp_APIC();
2337	if (upmode)
2338	apic_bsp_up_setup();
2339	setup_local_APIC();
2340
2341	enable_IO_APIC();
2342	end_local_APIC_setup();
2343	irq_remap_enable_fault_handling();
2344	setup_IO_APIC();
2345	lapic_update_legacy_vectors();
2346	}
2347
2348	#ifdef CONFIG_UP_LATE_INIT
2349	void __init up_late_init(void)
2350	{
2351	if (apic_intr_mode == APIC_PIC)
2352	return;
2353
2354	/* Setup local timer */
2355	x86_init.timers.setup_percpu_clockev();
2356	}
2357	#endif
2358
2359	/*
2360	* Power management
2361	*/
2362	#ifdef CONFIG_PM
2363
2364	static struct {
2365	/*
2366	* 'active' is true if the local APIC was enabled by us and
2367	* not the BIOS; this signifies that we are also responsible
2368	* for disabling it before entering apm/acpi suspend
2369	*/
2370	int active;
2371	/* r/w apic fields */
2372	u32 apic_id;
2373	unsigned int apic_taskpri;
2374	unsigned int apic_ldr;
2375	unsigned int apic_dfr;
2376	unsigned int apic_spiv;
2377	unsigned int apic_lvtt;
2378	unsigned int apic_lvtpc;
2379	unsigned int apic_lvt0;
2380	unsigned int apic_lvt1;
2381	unsigned int apic_lvterr;
2382	unsigned int apic_tmict;
2383	unsigned int apic_tdcr;
2384	unsigned int apic_thmr;
2385	unsigned int apic_cmci;
2386	} apic_pm_state;
2387
2388	static int lapic_suspend(void *data)
2389	{
2390	unsigned long flags;
2391	int maxlvt;
2392
2393	if (!apic_pm_state.active)
2394	return 0;
2395
2396	maxlvt = lapic_get_maxlvt();
2397
2398	apic_pm_state.apic_id = apic_read(APIC_ID);
2399	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2400	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2401	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2402	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2403	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2404	if (maxlvt >= 4)
2405	apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2406	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2407	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2408	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2409	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2410	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2411	#ifdef CONFIG_X86_THERMAL_VECTOR
2412	if (maxlvt >= 5)
2413	apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2414	#endif
2415	#ifdef CONFIG_X86_MCE_INTEL
2416	if (maxlvt >= 6)
2417	apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2418	#endif
2419
2420	local_irq_save(flags);
2421
2422	/*
2423	* Mask IOAPIC before disabling the local APIC to prevent stale IRR
2424	* entries on some implementations.
2425	*/
2426	mask_ioapic_entries();
2427
2428	disable_local_APIC();
2429
2430	irq_remapping_disable();
2431
2432	local_irq_restore(flags);
2433	return 0;
2434	}
2435
2436	static void lapic_resume(void *data)
2437	{
2438	unsigned int l, h;
2439	unsigned long flags;
2440	int maxlvt;
2441
2442	if (!apic_pm_state.active)
2443	return;
2444
2445	local_irq_save(flags);
2446
2447	/*
2448	* IO-APIC and PIC have their own resume routines.
2449	* We just mask them here to make sure the interrupt
2450	* subsystem is completely quiet while we enable x2apic
2451	* and interrupt-remapping.
2452	*/
2453	mask_ioapic_entries();
2454	legacy_pic->mask_all();
2455
2456	if (x2apic_mode) {
2457	__x2apic_enable();
2458	} else {
2459	/*
2460	* Make sure the APICBASE points to the right address
2461	*
2462	* FIXME! This will be wrong if we ever support suspend on
2463	* SMP! We'll need to do this as part of the CPU restore!
2464	*/
2465	if (boot_cpu_data.x86 >= 6) {
2466	rdmsr(MSR_IA32_APICBASE, l, h);
2467	l &= ~MSR_IA32_APICBASE_BASE;
2468	l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2469	wrmsr(MSR_IA32_APICBASE, low: l, high: h);
2470	}
2471	}
2472
2473	maxlvt = lapic_get_maxlvt();
2474	apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2475	apic_write(APIC_ID, val: apic_pm_state.apic_id);
2476	apic_write(APIC_DFR, val: apic_pm_state.apic_dfr);
2477	apic_write(APIC_LDR, val: apic_pm_state.apic_ldr);
2478	apic_write(APIC_TASKPRI, val: apic_pm_state.apic_taskpri);
2479	apic_write(APIC_SPIV, val: apic_pm_state.apic_spiv);
2480	apic_write(APIC_LVT0, val: apic_pm_state.apic_lvt0);
2481	apic_write(APIC_LVT1, val: apic_pm_state.apic_lvt1);
2482	#ifdef CONFIG_X86_THERMAL_VECTOR
2483	if (maxlvt >= 5)
2484	apic_write(APIC_LVTTHMR, val: apic_pm_state.apic_thmr);
2485	#endif
2486	#ifdef CONFIG_X86_MCE_INTEL
2487	if (maxlvt >= 6)
2488	apic_write(APIC_LVTCMCI, val: apic_pm_state.apic_cmci);
2489	#endif
2490	if (maxlvt >= 4)
2491	apic_write(APIC_LVTPC, val: apic_pm_state.apic_lvtpc);
2492	apic_write(APIC_LVTT, val: apic_pm_state.apic_lvtt);
2493	apic_write(APIC_TDCR, val: apic_pm_state.apic_tdcr);
2494	apic_write(APIC_TMICT, val: apic_pm_state.apic_tmict);
2495	apic_write(APIC_ESR, val: 0);
2496	apic_read(APIC_ESR);
2497	apic_write(APIC_LVTERR, val: apic_pm_state.apic_lvterr);
2498	apic_write(APIC_ESR, val: 0);
2499	apic_read(APIC_ESR);
2500
2501	irq_remapping_reenable(x2apic_mode);
2502
2503	local_irq_restore(flags);
2504	}
2505
2506	/*
2507	* This device has no shutdown method - fully functioning local APICs
2508	* are needed on every CPU up until machine_halt/restart/poweroff.
2509	*/
2510
2511	static const struct syscore_ops lapic_syscore_ops = {
2512	.resume = lapic_resume,
2513	.suspend = lapic_suspend,
2514	};
2515
2516	static struct syscore lapic_syscore = {
2517	.ops = &lapic_syscore_ops,
2518	};
2519
2520	static void apic_pm_activate(void)
2521	{
2522	apic_pm_state.active = 1;
2523	}
2524
2525	static int __init init_lapic_sysfs(void)
2526	{
2527	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2528	if (boot_cpu_has(X86_FEATURE_APIC))
2529	register_syscore(syscore: &lapic_syscore);
2530
2531	return 0;
2532	}
2533
2534	/* local apic needs to resume before other devices access its registers. */
2535	core_initcall(init_lapic_sysfs);
2536
2537	#else /* CONFIG_PM */
2538
2539	static void apic_pm_activate(void) { }
2540
2541	#endif /* CONFIG_PM */
2542
2543	#ifdef CONFIG_X86_64
2544
2545	static int multi_checked;
2546	static int multi;
2547
2548	static int set_multi(const struct dmi_system_id *d)
2549	{
2550	if (multi)
2551	return 0;
2552	pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2553	multi = 1;
2554	return 0;
2555	}
2556
2557	static const struct dmi_system_id multi_dmi_table[] = {
2558	{
2559	.callback = set_multi,
2560	.ident = "IBM System Summit2",
2561	.matches = {
2562	DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2563	DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2564	},
2565	},
2566	{}
2567	};
2568
2569	static void dmi_check_multi(void)
2570	{
2571	if (multi_checked)
2572	return;
2573
2574	dmi_check_system(list: multi_dmi_table);
2575	multi_checked = 1;
2576	}
2577
2578	/*
2579	* apic_is_clustered_box() -- Check if we can expect good TSC
2580	*
2581	* Thus far, the major user of this is IBM's Summit2 series:
2582	* Clustered boxes may have unsynced TSC problems if they are
2583	* multi-chassis.
2584	* Use DMI to check them
2585	*/
2586	int apic_is_clustered_box(void)
2587	{
2588	dmi_check_multi();
2589	return multi;
2590	}
2591	#endif
2592
2593	/*
2594	* APIC command line parameters
2595	*/
2596	static int __init setup_nolapic(char *arg)
2597	{
2598	apic_is_disabled = true;
2599	setup_clear_cpu_cap(X86_FEATURE_APIC);
2600	return 0;
2601	}
2602	early_param("nolapic", setup_nolapic);
2603
2604	static int __init parse_lapic_timer_c2_ok(char *arg)
2605	{
2606	local_apic_timer_c2_ok = 1;
2607	return 0;
2608	}
2609	early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2610
2611	static int __init parse_disable_apic_timer(char *arg)
2612	{
2613	disable_apic_timer = 1;
2614	return 0;
2615	}
2616	early_param("noapictimer", parse_disable_apic_timer);
2617
2618	static int __init parse_nolapic_timer(char *arg)
2619	{
2620	disable_apic_timer = 1;
2621	return 0;
2622	}
2623	early_param("nolapic_timer", parse_nolapic_timer);
2624
2625	static int __init apic_set_verbosity(char *arg)
2626	{
2627	if (!arg) {
2628	if (IS_ENABLED(CONFIG_X86_32))
2629	return -EINVAL;
2630
2631	ioapic_is_disabled = false;
2632	return 0;
2633	}
2634
2635	if (strcmp("debug", arg) == 0)
2636	apic_verbosity = APIC_DEBUG;
2637	else if (strcmp("verbose", arg) == 0)
2638	apic_verbosity = APIC_VERBOSE;
2639	#ifdef CONFIG_X86_64
2640	else {
2641	pr_warn("APIC Verbosity level %s not recognised"
2642	" use apic=verbose or apic=debug\n", arg);
2643	return -EINVAL;
2644	}
2645	#endif
2646
2647	return 0;
2648	}
2649	early_param("apic", apic_set_verbosity);
2650
2651	static int __init lapic_insert_resource(void)
2652	{
2653	if (!apic_mmio_base)
2654	return -1;
2655
2656	/* Put local APIC into the resource map. */
2657	lapic_resource.start = apic_mmio_base;
2658	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2659	insert_resource(parent: &iomem_resource, new: &lapic_resource);
2660
2661	return 0;
2662	}
2663
2664	/*
2665	* need call insert after e820__reserve_resources()
2666	* that is using request_resource
2667	*/
2668	late_initcall(lapic_insert_resource);
2669
2670	static int __init apic_set_extnmi(char *arg)
2671	{
2672	if (!arg)
2673	return -EINVAL;
2674
2675	if (!strncmp("all", arg, 3))
2676	apic_extnmi = APIC_EXTNMI_ALL;
2677	else if (!strncmp("none", arg, 4))
2678	apic_extnmi = APIC_EXTNMI_NONE;
2679	else if (!strncmp("bsp", arg, 3))
2680	apic_extnmi = APIC_EXTNMI_BSP;
2681	else {
2682	pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2683	return -EINVAL;
2684	}
2685
2686	return 0;
2687	}
2688	early_param("apic_extnmi", apic_set_extnmi);
2689