1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Local APIC handling, local APIC timers
4	*
5	* (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6	*
7	* Fixes
8	* Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9	* thanks to Eric Gilmore
10	* and Rolf G. Tews
11	* for testing these extensively.
12	* Maciej W. Rozycki : Various updates and fixes.
13	* Mikael Pettersson : Power Management for UP-APIC.
14	* Pavel Machek and
15	* Mikael Pettersson : PM converted to driver model.
16	*/
17
18	#include <linux/perf_event.h>
19	#include <linux/kernel_stat.h>
20	#include <linux/mc146818rtc.h>
21	#include <linux/acpi_pmtmr.h>
22	#include <linux/bitmap.h>
23	#include <linux/clockchips.h>
24	#include <linux/interrupt.h>
25	#include <linux/memblock.h>
26	#include <linux/ftrace.h>
27	#include <linux/ioport.h>
28	#include <linux/export.h>
29	#include <linux/syscore_ops.h>
30	#include <linux/delay.h>
31	#include <linux/timex.h>
32	#include <linux/i8253.h>
33	#include <linux/dmar.h>
34	#include <linux/init.h>
35	#include <linux/cpu.h>
36	#include <linux/dmi.h>
37	#include <linux/smp.h>
38	#include <linux/mm.h>
39
40	#include <xen/xen.h>
41
42	#include <asm/trace/irq_vectors.h>
43	#include <asm/irq_remapping.h>
44	#include <asm/pc-conf-reg.h>
45	#include <asm/perf_event.h>
46	#include <asm/x86_init.h>
47	#include <linux/atomic.h>
48	#include <asm/barrier.h>
49	#include <asm/mpspec.h>
50	#include <asm/i8259.h>
51	#include <asm/proto.h>
52	#include <asm/traps.h>
53	#include <asm/apic.h>
54	#include <asm/acpi.h>
55	#include <asm/io_apic.h>
56	#include <asm/desc.h>
57	#include <asm/hpet.h>
58	#include <asm/mtrr.h>
59	#include <asm/time.h>
60	#include <asm/smp.h>
61	#include <asm/mce.h>
62	#include <asm/tsc.h>
63	#include <asm/hypervisor.h>
64	#include <asm/cpu_device_id.h>
65	#include <asm/intel-family.h>
66	#include <asm/irq_regs.h>
67	#include <asm/cpu.h>
68
69	#include "local.h"
70
71	/* Processor that is doing the boot up */
72	u32 boot_cpu_physical_apicid __ro_after_init = BAD_APICID;
73	EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
74
75	u8 boot_cpu_apic_version __ro_after_init;
76
77	/*
78	* This variable controls which CPUs receive external NMIs. By default,
79	* external NMIs are delivered only to the BSP.
80	*/
81	static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
82
83	/*
84	* Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
85	*/
86	static bool virt_ext_dest_id __ro_after_init;
87
88	/* For parallel bootup. */
89	unsigned long apic_mmio_base __ro_after_init;
90
91	static inline bool apic_accessible(void)
92	{
93	return x2apic_mode || apic_mmio_base;
94	}
95
96	#ifdef CONFIG_X86_32
97	/* Local APIC was disabled by the BIOS and enabled by the kernel */
98	static int enabled_via_apicbase __ro_after_init;
99
100	/*
101	* Handle interrupt mode configuration register (IMCR).
102	* This register controls whether the interrupt signals
103	* that reach the BSP come from the master PIC or from the
104	* local APIC. Before entering Symmetric I/O Mode, either
105	* the BIOS or the operating system must switch out of
106	* PIC Mode by changing the IMCR.
107	*/
108	static inline void imcr_pic_to_apic(void)
109	{
110	/* NMI and 8259 INTR go through APIC */
111	pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
112	}
113
114	static inline void imcr_apic_to_pic(void)
115	{
116	/* NMI and 8259 INTR go directly to BSP */
117	pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
118	}
119	#endif
120
121	/*
122	* Knob to control our willingness to enable the local APIC.
123	*
124	* +1=force-enable
125	*/
126	static int force_enable_local_apic __initdata;
127
128	/*
129	* APIC command line parameters
130	*/
131	static int __init parse_lapic(char *arg)
132	{
133	if (IS_ENABLED(CONFIG_X86_32) && !arg)
134	force_enable_local_apic = 1;
135	else if (arg && !strncmp(arg, "notscdeadline", 13))
136	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
137	return 0;
138	}
139	early_param("lapic", parse_lapic);
140
141	#ifdef CONFIG_X86_64
142	static int apic_calibrate_pmtmr __initdata;
143	static __init int setup_apicpmtimer(char *s)
144	{
145	apic_calibrate_pmtmr = 1;
146	notsc_setup(NULL);
147	return 1;
148	}
149	__setup("apicpmtimer", setup_apicpmtimer);
150	#endif
151
152	static unsigned long mp_lapic_addr __ro_after_init;
153	bool apic_is_disabled __ro_after_init;
154	/* Disable local APIC timer from the kernel commandline or via dmi quirk */
155	static int disable_apic_timer __initdata;
156	/* Local APIC timer works in C2 */
157	int local_apic_timer_c2_ok __ro_after_init;
158	EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
159
160	/*
161	* Debug level, exported for io_apic.c
162	*/
163	int apic_verbosity __ro_after_init;
164
165	int pic_mode __ro_after_init;
166
167	/* Have we found an MP table */
168	int smp_found_config __ro_after_init;
169
170	static struct resource lapic_resource = {
171	.name = "Local APIC",
172	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
173	};
174
175	unsigned int lapic_timer_period = 0;
176
177	static void apic_pm_activate(void);
178
179	/*
180	* Get the LAPIC version
181	*/
182	static inline int lapic_get_version(void)
183	{
184	return GET_APIC_VERSION(apic_read(APIC_LVR));
185	}
186
187	/*
188	* Check, if the APIC is integrated or a separate chip
189	*/
190	static inline int lapic_is_integrated(void)
191	{
192	return APIC_INTEGRATED(lapic_get_version());
193	}
194
195	/*
196	* Check, whether this is a modern or a first generation APIC
197	*/
198	static int modern_apic(void)
199	{
200	/* AMD systems use old APIC versions, so check the CPU */
201	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
202	boot_cpu_data.x86 >= 0xf)
203	return 1;
204
205	/* Hygon systems use modern APIC */
206	if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
207	return 1;
208
209	return lapic_get_version() >= 0x14;
210	}
211
212	/*
213	* right after this call apic become NOOP driven
214	* so apic->write/read doesn't do anything
215	*/
216	static void __init apic_disable(void)
217	{
218	apic_install_driver(driver: &apic_noop);
219	}
220
221	void native_apic_icr_write(u32 low, u32 id)
222	{
223	unsigned long flags;
224
225	local_irq_save(flags);
226	apic_write(APIC_ICR2, SET_XAPIC_DEST_FIELD(id));
227	apic_write(APIC_ICR, val: low);
228	local_irq_restore(flags);
229	}
230
231	u64 native_apic_icr_read(void)
232	{
233	u32 icr1, icr2;
234
235	icr2 = apic_read(APIC_ICR2);
236	icr1 = apic_read(APIC_ICR);
237
238	return icr1 | ((u64)icr2 << 32);
239	}
240
241	/**
242	* lapic_get_maxlvt - get the maximum number of local vector table entries
243	*/
244	int lapic_get_maxlvt(void)
245	{
246	/*
247	* - we always have APIC integrated on 64bit mode
248	* - 82489DXs do not report # of LVT entries
249	*/
250	return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
251	}
252
253	/*
254	* Local APIC timer
255	*/
256
257	/* Clock divisor */
258	#define APIC_DIVISOR 16
259	#define TSC_DIVISOR 8
260
261	/* i82489DX specific */
262	#define I82489DX_BASE_DIVIDER (((0x2) << 18))
263
264	/*
265	* This function sets up the local APIC timer, with a timeout of
266	* 'clocks' APIC bus clock. During calibration we actually call
267	* this function twice on the boot CPU, once with a bogus timeout
268	* value, second time for real. The other (noncalibrating) CPUs
269	* call this function only once, with the real, calibrated value.
270	*
271	* We do reads before writes even if unnecessary, to get around the
272	* P5 APIC double write bug.
273	*/
274	static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
275	{
276	unsigned int lvtt_value, tmp_value;
277
278	lvtt_value = LOCAL_TIMER_VECTOR;
279	if (!oneshot)
280	lvtt_value |= APIC_LVT_TIMER_PERIODIC;
281	else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
282	lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
283
284	/*
285	* The i82489DX APIC uses bit 18 and 19 for the base divider. This
286	* overlaps with bit 18 on integrated APICs, but is not documented
287	* in the SDM. No problem though. i82489DX equipped systems do not
288	* have TSC deadline timer.
289	*/
290	if (!lapic_is_integrated())
291	lvtt_value |= I82489DX_BASE_DIVIDER;
292
293	if (!irqen)
294	lvtt_value |= APIC_LVT_MASKED;
295
296	apic_write(APIC_LVTT, val: lvtt_value);
297
298	if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
299	/*
300	* See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
301	* writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
302	* According to Intel, MFENCE can do the serialization here.
303	*/
304	asm volatile("mfence" : : : "memory");
305	return;
306	}
307
308	/*
309	* Divide PICLK by 16
310	*/
311	tmp_value = apic_read(APIC_TDCR);
312	apic_write(APIC_TDCR,
313	val: (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
314	APIC_TDR_DIV_16);
315
316	if (!oneshot)
317	apic_write(APIC_TMICT, val: clocks / APIC_DIVISOR);
318	}
319
320	/*
321	* Setup extended LVT, AMD specific
322	*
323	* Software should use the LVT offsets the BIOS provides. The offsets
324	* are determined by the subsystems using it like those for MCE
325	* threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
326	* are supported. Beginning with family 10h at least 4 offsets are
327	* available.
328	*
329	* Since the offsets must be consistent for all cores, we keep track
330	* of the LVT offsets in software and reserve the offset for the same
331	* vector also to be used on other cores. An offset is freed by
332	* setting the entry to APIC_EILVT_MASKED.
333	*
334	* If the BIOS is right, there should be no conflicts. Otherwise a
335	* "[Firmware Bug]: ..." error message is generated. However, if
336	* software does not properly determines the offsets, it is not
337	* necessarily a BIOS bug.
338	*/
339
340	static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
341
342	static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
343	{
344	return (old & APIC_EILVT_MASKED)
345	|| (new == APIC_EILVT_MASKED)
346	|| ((new & ~APIC_EILVT_MASKED) == old);
347	}
348
349	static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
350	{
351	unsigned int rsvd, vector;
352
353	if (offset >= APIC_EILVT_NR_MAX)
354	return ~0;
355
356	rsvd = atomic_read(v: &eilvt_offsets[offset]);
357	do {
358	vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
359	if (vector && !eilvt_entry_is_changeable(old: vector, new))
360	/* may not change if vectors are different */
361	return rsvd;
362	} while (!atomic_try_cmpxchg(v: &eilvt_offsets[offset], old: &rsvd, new));
363
364	rsvd = new & ~APIC_EILVT_MASKED;
365	if (rsvd && rsvd != vector)
366	pr_info("LVT offset %d assigned for vector 0x%02x\n",
367	offset, rsvd);
368
369	return new;
370	}
371
372	/*
373	* If mask=1, the LVT entry does not generate interrupts while mask=0
374	* enables the vector. See also the BKDGs. Must be called with
375	* preemption disabled.
376	*/
377
378	int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
379	{
380	unsigned long reg = APIC_EILVTn(offset);
381	unsigned int new, old, reserved;
382
383	new = (mask << 16) | (msg_type << 8) | vector;
384	old = apic_read(reg);
385	reserved = reserve_eilvt_offset(offset, new);
386
387	if (reserved != new) {
388	pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
389	"vector 0x%x, but the register is already in use for "
390	"vector 0x%x on another cpu\n",
391	smp_processor_id(), reg, offset, new, reserved);
392	return -EINVAL;
393	}
394
395	if (!eilvt_entry_is_changeable(old, new)) {
396	pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
397	"vector 0x%x, but the register is already in use for "
398	"vector 0x%x on this cpu\n",
399	smp_processor_id(), reg, offset, new, old);
400	return -EBUSY;
401	}
402
403	apic_write(reg, val: new);
404
405	return 0;
406	}
407	EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
408
409	/*
410	* Program the next event, relative to now
411	*/
412	static int lapic_next_event(unsigned long delta,
413	struct clock_event_device *evt)
414	{
415	apic_write(APIC_TMICT, val: delta);
416	return 0;
417	}
418
419	static int lapic_next_deadline(unsigned long delta,
420	struct clock_event_device *evt)
421	{
422	u64 tsc;
423
424	/* This MSR is special and need a special fence: */
425	weak_wrmsr_fence();
426
427	tsc = rdtsc();
428	wrmsrl(MSR_IA32_TSC_DEADLINE, val: tsc + (((u64) delta) * TSC_DIVISOR));
429	return 0;
430	}
431
432	static int lapic_timer_shutdown(struct clock_event_device *evt)
433	{
434	unsigned int v;
435
436	/* Lapic used as dummy for broadcast ? */
437	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
438	return 0;
439
440	v = apic_read(APIC_LVTT);
441	v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
442	apic_write(APIC_LVTT, val: v);
443	apic_write(APIC_TMICT, val: 0);
444	return 0;
445	}
446
447	static inline int
448	lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
449	{
450	/* Lapic used as dummy for broadcast ? */
451	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
452	return 0;
453
454	__setup_APIC_LVTT(clocks: lapic_timer_period, oneshot, irqen: 1);
455	return 0;
456	}
457
458	static int lapic_timer_set_periodic(struct clock_event_device *evt)
459	{
460	return lapic_timer_set_periodic_oneshot(evt, oneshot: false);
461	}
462
463	static int lapic_timer_set_oneshot(struct clock_event_device *evt)
464	{
465	return lapic_timer_set_periodic_oneshot(evt, oneshot: true);
466	}
467
468	/*
469	* Local APIC timer broadcast function
470	*/
471	static void lapic_timer_broadcast(const struct cpumask *mask)
472	{
473	#ifdef CONFIG_SMP
474	__apic_send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
475	#endif
476	}
477
478
479	/*
480	* The local apic timer can be used for any function which is CPU local.
481	*/
482	static struct clock_event_device lapic_clockevent = {
483	.name = "lapic",
484	.features = CLOCK_EVT_FEAT_PERIODIC |
485	CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
486	| CLOCK_EVT_FEAT_DUMMY,
487	.shift = 32,
488	.set_state_shutdown = lapic_timer_shutdown,
489	.set_state_periodic = lapic_timer_set_periodic,
490	.set_state_oneshot = lapic_timer_set_oneshot,
491	.set_state_oneshot_stopped = lapic_timer_shutdown,
492	.set_next_event = lapic_next_event,
493	.broadcast = lapic_timer_broadcast,
494	.rating = 100,
495	.irq = -1,
496	};
497	static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
498
499	static const struct x86_cpu_id deadline_match[] __initconst = {
500	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
501	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */
502
503	X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020),
504
505	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011),
506	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e),
507	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c),
508	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003),
509
510	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136),
511	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014),
512	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0),
513
514	X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22),
515	X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20),
516	X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17),
517
518	X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25),
519	X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17),
520
521	X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2),
522	X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2),
523
524	X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52),
525	X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52),
526
527	{},
528	};
529
530	static __init bool apic_validate_deadline_timer(void)
531	{
532	const struct x86_cpu_id *m;
533	u32 rev;
534
535	if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
536	return false;
537	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
538	return true;
539
540	m = x86_match_cpu(match: deadline_match);
541	if (!m)
542	return true;
543
544	rev = (u32)m->driver_data;
545
546	if (boot_cpu_data.microcode >= rev)
547	return true;
548
549	setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
550	pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
551	"please update microcode to version: 0x%x (or later)\n", rev);
552	return false;
553	}
554
555	/*
556	* Setup the local APIC timer for this CPU. Copy the initialized values
557	* of the boot CPU and register the clock event in the framework.
558	*/
559	static void setup_APIC_timer(void)
560	{
561	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
562
563	if (this_cpu_has(X86_FEATURE_ARAT)) {
564	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
565	/* Make LAPIC timer preferable over percpu HPET */
566	lapic_clockevent.rating = 150;
567	}
568
569	memcpy(levt, &lapic_clockevent, sizeof(*levt));
570	levt->cpumask = cpumask_of(smp_processor_id());
571
572	if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
573	levt->name = "lapic-deadline";
574	levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
575	CLOCK_EVT_FEAT_DUMMY);
576	levt->set_next_event = lapic_next_deadline;
577	clockevents_config_and_register(dev: levt,
578	freq: tsc_khz * (1000 / TSC_DIVISOR),
579	min_delta: 0xF, max_delta: ~0UL);
580	} else
581	clockevents_register_device(dev: levt);
582	}
583
584	/*
585	* Install the updated TSC frequency from recalibration at the TSC
586	* deadline clockevent devices.
587	*/
588	static void __lapic_update_tsc_freq(void *info)
589	{
590	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
591
592	if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
593	return;
594
595	clockevents_update_freq(ce: levt, freq: tsc_khz * (1000 / TSC_DIVISOR));
596	}
597
598	void lapic_update_tsc_freq(void)
599	{
600	/*
601	* The clockevent device's ->mult and ->shift can both be
602	* changed. In order to avoid races, schedule the frequency
603	* update code on each CPU.
604	*/
605	on_each_cpu(func: __lapic_update_tsc_freq, NULL, wait: 0);
606	}
607
608	/*
609	* In this functions we calibrate APIC bus clocks to the external timer.
610	*
611	* We want to do the calibration only once since we want to have local timer
612	* irqs synchronous. CPUs connected by the same APIC bus have the very same bus
613	* frequency.
614	*
615	* This was previously done by reading the PIT/HPET and waiting for a wrap
616	* around to find out, that a tick has elapsed. I have a box, where the PIT
617	* readout is broken, so it never gets out of the wait loop again. This was
618	* also reported by others.
619	*
620	* Monitoring the jiffies value is inaccurate and the clockevents
621	* infrastructure allows us to do a simple substitution of the interrupt
622	* handler.
623	*
624	* The calibration routine also uses the pm_timer when possible, as the PIT
625	* happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
626	* back to normal later in the boot process).
627	*/
628
629	#define LAPIC_CAL_LOOPS (HZ/10)
630
631	static __initdata int lapic_cal_loops = -1;
632	static __initdata long lapic_cal_t1, lapic_cal_t2;
633	static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
634	static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
635	static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
636
637	/*
638	* Temporary interrupt handler and polled calibration function.
639	*/
640	static void __init lapic_cal_handler(struct clock_event_device *dev)
641	{
642	unsigned long long tsc = 0;
643	long tapic = apic_read(APIC_TMCCT);
644	unsigned long pm = acpi_pm_read_early();
645
646	if (boot_cpu_has(X86_FEATURE_TSC))
647	tsc = rdtsc();
648
649	switch (lapic_cal_loops++) {
650	case 0:
651	lapic_cal_t1 = tapic;
652	lapic_cal_tsc1 = tsc;
653	lapic_cal_pm1 = pm;
654	lapic_cal_j1 = jiffies;
655	break;
656
657	case LAPIC_CAL_LOOPS:
658	lapic_cal_t2 = tapic;
659	lapic_cal_tsc2 = tsc;
660	if (pm < lapic_cal_pm1)
661	pm += ACPI_PM_OVRRUN;
662	lapic_cal_pm2 = pm;
663	lapic_cal_j2 = jiffies;
664	break;
665	}
666	}
667
668	static int __init
669	calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
670	{
671	const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
672	const long pm_thresh = pm_100ms / 100;
673	unsigned long mult;
674	u64 res;
675
676	#ifndef CONFIG_X86_PM_TIMER
677	return -1;
678	#endif
679
680	apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
681
682	/* Check, if the PM timer is available */
683	if (!deltapm)
684	return -1;
685
686	mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, shift_constant: 22);
687
688	if (deltapm > (pm_100ms - pm_thresh) &&
689	deltapm < (pm_100ms + pm_thresh)) {
690	apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
691	return 0;
692	}
693
694	res = (((u64)deltapm) * mult) >> 22;
695	do_div(res, 1000000);
696	pr_warn("APIC calibration not consistent "
697	"with PM-Timer: %ldms instead of 100ms\n", (long)res);
698
699	/* Correct the lapic counter value */
700	res = (((u64)(*delta)) * pm_100ms);
701	do_div(res, deltapm);
702	pr_info("APIC delta adjusted to PM-Timer: "
703	"%lu (%ld)\n", (unsigned long)res, *delta);
704	*delta = (long)res;
705
706	/* Correct the tsc counter value */
707	if (boot_cpu_has(X86_FEATURE_TSC)) {
708	res = (((u64)(*deltatsc)) * pm_100ms);
709	do_div(res, deltapm);
710	apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
711	"PM-Timer: %lu (%ld)\n",
712	(unsigned long)res, *deltatsc);
713	*deltatsc = (long)res;
714	}
715
716	return 0;
717	}
718
719	static int __init lapic_init_clockevent(void)
720	{
721	if (!lapic_timer_period)
722	return -1;
723
724	/* Calculate the scaled math multiplication factor */
725	lapic_clockevent.mult = div_sc(ticks: lapic_timer_period/APIC_DIVISOR,
726	TICK_NSEC, shift: lapic_clockevent.shift);
727	lapic_clockevent.max_delta_ns =
728	clockevent_delta2ns(latch: 0x7FFFFFFF, evt: &lapic_clockevent);
729	lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
730	lapic_clockevent.min_delta_ns =
731	clockevent_delta2ns(latch: 0xF, evt: &lapic_clockevent);
732	lapic_clockevent.min_delta_ticks = 0xF;
733
734	return 0;
735	}
736
737	bool __init apic_needs_pit(void)
738	{
739	/*
740	* If the frequencies are not known, PIT is required for both TSC
741	* and apic timer calibration.
742	*/
743	if (!tsc_khz || !cpu_khz)
744	return true;
745
746	/* Is there an APIC at all or is it disabled? */
747	if (!boot_cpu_has(X86_FEATURE_APIC) || apic_is_disabled)
748	return true;
749
750	/*
751	* If interrupt delivery mode is legacy PIC or virtual wire without
752	* configuration, the local APIC timer won't be set up. Make sure
753	* that the PIT is initialized.
754	*/
755	if (apic_intr_mode == APIC_PIC ||
756	apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
757	return true;
758
759	/* Virt guests may lack ARAT, but still have DEADLINE */
760	if (!boot_cpu_has(X86_FEATURE_ARAT))
761	return true;
762
763	/* Deadline timer is based on TSC so no further PIT action required */
764	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
765	return false;
766
767	/* APIC timer disabled? */
768	if (disable_apic_timer)
769	return true;
770	/*
771	* The APIC timer frequency is known already, no PIT calibration
772	* required. If unknown, let the PIT be initialized.
773	*/
774	return lapic_timer_period == 0;
775	}
776
777	static int __init calibrate_APIC_clock(void)
778	{
779	struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
780	u64 tsc_perj = 0, tsc_start = 0;
781	unsigned long jif_start;
782	unsigned long deltaj;
783	long delta, deltatsc;
784	int pm_referenced = 0;
785
786	if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
787	return 0;
788
789	/*
790	* Check if lapic timer has already been calibrated by platform
791	* specific routine, such as tsc calibration code. If so just fill
792	* in the clockevent structure and return.
793	*/
794	if (!lapic_init_clockevent()) {
795	apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
796	lapic_timer_period);
797	/*
798	* Direct calibration methods must have an always running
799	* local APIC timer, no need for broadcast timer.
800	*/
801	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
802	return 0;
803	}
804
805	apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
806	"calibrating APIC timer ...\n");
807
808	/*
809	* There are platforms w/o global clockevent devices. Instead of
810	* making the calibration conditional on that, use a polling based
811	* approach everywhere.
812	*/
813	local_irq_disable();
814
815	/*
816	* Setup the APIC counter to maximum. There is no way the lapic
817	* can underflow in the 100ms detection time frame
818	*/
819	__setup_APIC_LVTT(clocks: 0xffffffff, oneshot: 0, irqen: 0);
820
821	/*
822	* Methods to terminate the calibration loop:
823	* 1) Global clockevent if available (jiffies)
824	* 2) TSC if available and frequency is known
825	*/
826	jif_start = READ_ONCE(jiffies);
827
828	if (tsc_khz) {
829	tsc_start = rdtsc();
830	tsc_perj = div_u64(dividend: (u64)tsc_khz * 1000, HZ);
831	}
832
833	/*
834	* Enable interrupts so the tick can fire, if a global
835	* clockevent device is available
836	*/
837	local_irq_enable();
838
839	while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
840	/* Wait for a tick to elapse */
841	while (1) {
842	if (tsc_khz) {
843	u64 tsc_now = rdtsc();
844	if ((tsc_now - tsc_start) >= tsc_perj) {
845	tsc_start += tsc_perj;
846	break;
847	}
848	} else {
849	unsigned long jif_now = READ_ONCE(jiffies);
850
851	if (time_after(jif_now, jif_start)) {
852	jif_start = jif_now;
853	break;
854	}
855	}
856	cpu_relax();
857	}
858
859	/* Invoke the calibration routine */
860	local_irq_disable();
861	lapic_cal_handler(NULL);
862	local_irq_enable();
863	}
864
865	local_irq_disable();
866
867	/* Build delta t1-t2 as apic timer counts down */
868	delta = lapic_cal_t1 - lapic_cal_t2;
869	apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
870
871	deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
872
873	/* we trust the PM based calibration if possible */
874	pm_referenced = !calibrate_by_pmtimer(deltapm: lapic_cal_pm2 - lapic_cal_pm1,
875	delta: &delta, deltatsc: &deltatsc);
876
877	lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
878	lapic_init_clockevent();
879
880	apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
881	apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
882	apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
883	lapic_timer_period);
884
885	if (boot_cpu_has(X86_FEATURE_TSC)) {
886	apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
887	"%ld.%04ld MHz.\n",
888	(deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
889	(deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
890	}
891
892	apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
893	"%u.%04u MHz.\n",
894	lapic_timer_period / (1000000 / HZ),
895	lapic_timer_period % (1000000 / HZ));
896
897	/*
898	* Do a sanity check on the APIC calibration result
899	*/
900	if (lapic_timer_period < (1000000 / HZ)) {
901	local_irq_enable();
902	pr_warn("APIC frequency too slow, disabling apic timer\n");
903	return -1;
904	}
905
906	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
907
908	/*
909	* PM timer calibration failed or not turned on so lets try APIC
910	* timer based calibration, if a global clockevent device is
911	* available.
912	*/
913	if (!pm_referenced && global_clock_event) {
914	apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
915
916	/*
917	* Setup the apic timer manually
918	*/
919	levt->event_handler = lapic_cal_handler;
920	lapic_timer_set_periodic(evt: levt);
921	lapic_cal_loops = -1;
922
923	/* Let the interrupts run */
924	local_irq_enable();
925
926	while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
927	cpu_relax();
928
929	/* Stop the lapic timer */
930	local_irq_disable();
931	lapic_timer_shutdown(evt: levt);
932
933	/* Jiffies delta */
934	deltaj = lapic_cal_j2 - lapic_cal_j1;
935	apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
936
937	/* Check, if the jiffies result is consistent */
938	if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
939	apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
940	else
941	levt->features |= CLOCK_EVT_FEAT_DUMMY;
942	}
943	local_irq_enable();
944
945	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
946	pr_warn("APIC timer disabled due to verification failure\n");
947	return -1;
948	}
949
950	return 0;
951	}
952
953	/*
954	* Setup the boot APIC
955	*
956	* Calibrate and verify the result.
957	*/
958	void __init setup_boot_APIC_clock(void)
959	{
960	/*
961	* The local apic timer can be disabled via the kernel
962	* commandline or from the CPU detection code. Register the lapic
963	* timer as a dummy clock event source on SMP systems, so the
964	* broadcast mechanism is used. On UP systems simply ignore it.
965	*/
966	if (disable_apic_timer) {
967	pr_info("Disabling APIC timer\n");
968	/* No broadcast on UP ! */
969	if (num_possible_cpus() > 1) {
970	lapic_clockevent.mult = 1;
971	setup_APIC_timer();
972	}
973	return;
974	}
975
976	if (calibrate_APIC_clock()) {
977	/* No broadcast on UP ! */
978	if (num_possible_cpus() > 1)
979	setup_APIC_timer();
980	return;
981	}
982
983	/*
984	* If nmi_watchdog is set to IO_APIC, we need the
985	* PIT/HPET going. Otherwise register lapic as a dummy
986	* device.
987	*/
988	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
989
990	/* Setup the lapic or request the broadcast */
991	setup_APIC_timer();
992	amd_e400_c1e_apic_setup();
993	}
994
995	void setup_secondary_APIC_clock(void)
996	{
997	setup_APIC_timer();
998	amd_e400_c1e_apic_setup();
999	}
1000
1001	/*
1002	* The guts of the apic timer interrupt
1003	*/
1004	static void local_apic_timer_interrupt(void)
1005	{
1006	struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1007
1008	/*
1009	* Normally we should not be here till LAPIC has been initialized but
1010	* in some cases like kdump, its possible that there is a pending LAPIC
1011	* timer interrupt from previous kernel's context and is delivered in
1012	* new kernel the moment interrupts are enabled.
1013	*
1014	* Interrupts are enabled early and LAPIC is setup much later, hence
1015	* its possible that when we get here evt->event_handler is NULL.
1016	* Check for event_handler being NULL and discard the interrupt as
1017	* spurious.
1018	*/
1019	if (!evt->event_handler) {
1020	pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1021	smp_processor_id());
1022	/* Switch it off */
1023	lapic_timer_shutdown(evt);
1024	return;
1025	}
1026
1027	/*
1028	* the NMI deadlock-detector uses this.
1029	*/
1030	inc_irq_stat(apic_timer_irqs);
1031
1032	evt->event_handler(evt);
1033	}
1034
1035	/*
1036	* Local APIC timer interrupt. This is the most natural way for doing
1037	* local interrupts, but local timer interrupts can be emulated by
1038	* broadcast interrupts too. [in case the hw doesn't support APIC timers]
1039	*
1040	* [ if a single-CPU system runs an SMP kernel then we call the local
1041	* interrupt as well. Thus we cannot inline the local irq ... ]
1042	*/
1043	DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1044	{
1045	struct pt_regs *old_regs = set_irq_regs(regs);
1046
1047	apic_eoi();
1048	trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1049	local_apic_timer_interrupt();
1050	trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1051
1052	set_irq_regs(old_regs);
1053	}
1054
1055	/*
1056	* Local APIC start and shutdown
1057	*/
1058
1059	/**
1060	* clear_local_APIC - shutdown the local APIC
1061	*
1062	* This is called, when a CPU is disabled and before rebooting, so the state of
1063	* the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1064	* leftovers during boot.
1065	*/
1066	void clear_local_APIC(void)
1067	{
1068	int maxlvt;
1069	u32 v;
1070
1071	if (!apic_accessible())
1072	return;
1073
1074	maxlvt = lapic_get_maxlvt();
1075	/*
1076	* Masking an LVT entry can trigger a local APIC error
1077	* if the vector is zero. Mask LVTERR first to prevent this.
1078	*/
1079	if (maxlvt >= 3) {
1080	v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1081	apic_write(APIC_LVTERR, val: v | APIC_LVT_MASKED);
1082	}
1083	/*
1084	* Careful: we have to set masks only first to deassert
1085	* any level-triggered sources.
1086	*/
1087	v = apic_read(APIC_LVTT);
1088	apic_write(APIC_LVTT, val: v | APIC_LVT_MASKED);
1089	v = apic_read(APIC_LVT0);
1090	apic_write(APIC_LVT0, val: v | APIC_LVT_MASKED);
1091	v = apic_read(APIC_LVT1);
1092	apic_write(APIC_LVT1, val: v | APIC_LVT_MASKED);
1093	if (maxlvt >= 4) {
1094	v = apic_read(APIC_LVTPC);
1095	apic_write(APIC_LVTPC, val: v | APIC_LVT_MASKED);
1096	}
1097
1098	/* lets not touch this if we didn't frob it */
1099	#ifdef CONFIG_X86_THERMAL_VECTOR
1100	if (maxlvt >= 5) {
1101	v = apic_read(APIC_LVTTHMR);
1102	apic_write(APIC_LVTTHMR, val: v | APIC_LVT_MASKED);
1103	}
1104	#endif
1105	#ifdef CONFIG_X86_MCE_INTEL
1106	if (maxlvt >= 6) {
1107	v = apic_read(APIC_LVTCMCI);
1108	if (!(v & APIC_LVT_MASKED))
1109	apic_write(APIC_LVTCMCI, val: v | APIC_LVT_MASKED);
1110	}
1111	#endif
1112
1113	/*
1114	* Clean APIC state for other OSs:
1115	*/
1116	apic_write(APIC_LVTT, APIC_LVT_MASKED);
1117	apic_write(APIC_LVT0, APIC_LVT_MASKED);
1118	apic_write(APIC_LVT1, APIC_LVT_MASKED);
1119	if (maxlvt >= 3)
1120	apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1121	if (maxlvt >= 4)
1122	apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1123
1124	/* Integrated APIC (!82489DX) ? */
1125	if (lapic_is_integrated()) {
1126	if (maxlvt > 3)
1127	/* Clear ESR due to Pentium errata 3AP and 11AP */
1128	apic_write(APIC_ESR, val: 0);
1129	apic_read(APIC_ESR);
1130	}
1131	}
1132
1133	/**
1134	* apic_soft_disable - Clears and software disables the local APIC on hotplug
1135	*
1136	* Contrary to disable_local_APIC() this does not touch the enable bit in
1137	* MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1138	* bus would require a hardware reset as the APIC would lose track of bus
1139	* arbitration. On systems with FSB delivery APICBASE could be disabled,
1140	* but it has to be guaranteed that no interrupt is sent to the APIC while
1141	* in that state and it's not clear from the SDM whether it still responds
1142	* to INIT/SIPI messages. Stay on the safe side and use software disable.
1143	*/
1144	void apic_soft_disable(void)
1145	{
1146	u32 value;
1147
1148	clear_local_APIC();
1149
1150	/* Soft disable APIC (implies clearing of registers for 82489DX!). */
1151	value = apic_read(APIC_SPIV);
1152	value &= ~APIC_SPIV_APIC_ENABLED;
1153	apic_write(APIC_SPIV, val: value);
1154	}
1155
1156	/**
1157	* disable_local_APIC - clear and disable the local APIC
1158	*/
1159	void disable_local_APIC(void)
1160	{
1161	if (!apic_accessible())
1162	return;
1163
1164	apic_soft_disable();
1165
1166	#ifdef CONFIG_X86_32
1167	/*
1168	* When LAPIC was disabled by the BIOS and enabled by the kernel,
1169	* restore the disabled state.
1170	*/
1171	if (enabled_via_apicbase) {
1172	unsigned int l, h;
1173
1174	rdmsr(MSR_IA32_APICBASE, l, h);
1175	l &= ~MSR_IA32_APICBASE_ENABLE;
1176	wrmsr(MSR_IA32_APICBASE, l, h);
1177	}
1178	#endif
1179	}
1180
1181	/*
1182	* If Linux enabled the LAPIC against the BIOS default disable it down before
1183	* re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1184	* not power-off. Additionally clear all LVT entries before disable_local_APIC
1185	* for the case where Linux didn't enable the LAPIC.
1186	*/
1187	void lapic_shutdown(void)
1188	{
1189	unsigned long flags;
1190
1191	if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1192	return;
1193
1194	local_irq_save(flags);
1195
1196	#ifdef CONFIG_X86_32
1197	if (!enabled_via_apicbase)
1198	clear_local_APIC();
1199	else
1200	#endif
1201	disable_local_APIC();
1202
1203
1204	local_irq_restore(flags);
1205	}
1206
1207	/**
1208	* sync_Arb_IDs - synchronize APIC bus arbitration IDs
1209	*/
1210	void __init sync_Arb_IDs(void)
1211	{
1212	/*
1213	* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1214	* needed on AMD.
1215	*/
1216	if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1217	return;
1218
1219	/*
1220	* Wait for idle.
1221	*/
1222	apic_wait_icr_idle();
1223
1224	apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1225	apic_write(APIC_ICR, APIC_DEST_ALLINC |
1226	APIC_INT_LEVELTRIG | APIC_DM_INIT);
1227	}
1228
1229	enum apic_intr_mode_id apic_intr_mode __ro_after_init;
1230
1231	static int __init __apic_intr_mode_select(void)
1232	{
1233	/* Check kernel option */
1234	if (apic_is_disabled) {
1235	pr_info("APIC disabled via kernel command line\n");
1236	return APIC_PIC;
1237	}
1238
1239	/* Check BIOS */
1240	#ifdef CONFIG_X86_64
1241	/* On 64-bit, the APIC must be integrated, Check local APIC only */
1242	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1243	apic_is_disabled = true;
1244	pr_info("APIC disabled by BIOS\n");
1245	return APIC_PIC;
1246	}
1247	#else
1248	/* On 32-bit, the APIC may be integrated APIC or 82489DX */
1249
1250	/* Neither 82489DX nor integrated APIC ? */
1251	if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
1252	apic_is_disabled = true;
1253	return APIC_PIC;
1254	}
1255
1256	/* If the BIOS pretends there is an integrated APIC ? */
1257	if (!boot_cpu_has(X86_FEATURE_APIC) &&
1258	APIC_INTEGRATED(boot_cpu_apic_version)) {
1259	apic_is_disabled = true;
1260	pr_err(FW_BUG "Local APIC not detected, force emulation\n");
1261	return APIC_PIC;
1262	}
1263	#endif
1264
1265	/* Check MP table or ACPI MADT configuration */
1266	if (!smp_found_config) {
1267	disable_ioapic_support();
1268	if (!acpi_lapic) {
1269	pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1270	return APIC_VIRTUAL_WIRE_NO_CONFIG;
1271	}
1272	return APIC_VIRTUAL_WIRE;
1273	}
1274
1275	#ifdef CONFIG_SMP
1276	/* If SMP should be disabled, then really disable it! */
1277	if (!setup_max_cpus) {
1278	pr_info("APIC: SMP mode deactivated\n");
1279	return APIC_SYMMETRIC_IO_NO_ROUTING;
1280	}
1281	#endif
1282
1283	return APIC_SYMMETRIC_IO;
1284	}
1285
1286	/* Select the interrupt delivery mode for the BSP */
1287	void __init apic_intr_mode_select(void)
1288	{
1289	apic_intr_mode = __apic_intr_mode_select();
1290	}
1291
1292	/*
1293	* An initial setup of the virtual wire mode.
1294	*/
1295	void __init init_bsp_APIC(void)
1296	{
1297	unsigned int value;
1298
1299	/*
1300	* Don't do the setup now if we have a SMP BIOS as the
1301	* through-I/O-APIC virtual wire mode might be active.
1302	*/
1303	if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1304	return;
1305
1306	/*
1307	* Do not trust the local APIC being empty at bootup.
1308	*/
1309	clear_local_APIC();
1310
1311	/*
1312	* Enable APIC.
1313	*/
1314	value = apic_read(APIC_SPIV);
1315	value &= ~APIC_VECTOR_MASK;
1316	value |= APIC_SPIV_APIC_ENABLED;
1317
1318	#ifdef CONFIG_X86_32
1319	/* This bit is reserved on P4/Xeon and should be cleared */
1320	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1321	(boot_cpu_data.x86 == 15))
1322	value &= ~APIC_SPIV_FOCUS_DISABLED;
1323	else
1324	#endif
1325	value |= APIC_SPIV_FOCUS_DISABLED;
1326	value |= SPURIOUS_APIC_VECTOR;
1327	apic_write(APIC_SPIV, val: value);
1328
1329	/*
1330	* Set up the virtual wire mode.
1331	*/
1332	apic_write(APIC_LVT0, APIC_DM_EXTINT);
1333	value = APIC_DM_NMI;
1334	if (!lapic_is_integrated()) /* 82489DX */
1335	value |= APIC_LVT_LEVEL_TRIGGER;
1336	if (apic_extnmi == APIC_EXTNMI_NONE)
1337	value |= APIC_LVT_MASKED;
1338	apic_write(APIC_LVT1, val: value);
1339	}
1340
1341	static void __init apic_bsp_setup(bool upmode);
1342
1343	/* Init the interrupt delivery mode for the BSP */
1344	void __init apic_intr_mode_init(void)
1345	{
1346	bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
1347
1348	switch (apic_intr_mode) {
1349	case APIC_PIC:
1350	pr_info("APIC: Keep in PIC mode(8259)\n");
1351	return;
1352	case APIC_VIRTUAL_WIRE:
1353	pr_info("APIC: Switch to virtual wire mode setup\n");
1354	break;
1355	case APIC_VIRTUAL_WIRE_NO_CONFIG:
1356	pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1357	upmode = true;
1358	break;
1359	case APIC_SYMMETRIC_IO:
1360	pr_info("APIC: Switch to symmetric I/O mode setup\n");
1361	break;
1362	case APIC_SYMMETRIC_IO_NO_ROUTING:
1363	pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1364	break;
1365	}
1366
1367	x86_64_probe_apic();
1368
1369	x86_32_install_bigsmp();
1370
1371	if (x86_platform.apic_post_init)
1372	x86_platform.apic_post_init();
1373
1374	apic_bsp_setup(upmode);
1375	}
1376
1377	static void lapic_setup_esr(void)
1378	{
1379	unsigned int oldvalue, value, maxlvt;
1380
1381	if (!lapic_is_integrated()) {
1382	pr_info("No ESR for 82489DX.\n");
1383	return;
1384	}
1385
1386	if (apic->disable_esr) {
1387	/*
1388	* Something untraceable is creating bad interrupts on
1389	* secondary quads ... for the moment, just leave the
1390	* ESR disabled - we can't do anything useful with the
1391	* errors anyway - mbligh
1392	*/
1393	pr_info("Leaving ESR disabled.\n");
1394	return;
1395	}
1396
1397	maxlvt = lapic_get_maxlvt();
1398	if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1399	apic_write(APIC_ESR, val: 0);
1400	oldvalue = apic_read(APIC_ESR);
1401
1402	/* enables sending errors */
1403	value = ERROR_APIC_VECTOR;
1404	apic_write(APIC_LVTERR, val: value);
1405
1406	/*
1407	* spec says clear errors after enabling vector.
1408	*/
1409	if (maxlvt > 3)
1410	apic_write(APIC_ESR, val: 0);
1411	value = apic_read(APIC_ESR);
1412	if (value != oldvalue)
1413	apic_printk(APIC_VERBOSE, "ESR value before enabling "
1414	"vector: 0x%08x after: 0x%08x\n",
1415	oldvalue, value);
1416	}
1417
1418	#define APIC_IR_REGS APIC_ISR_NR
1419	#define APIC_IR_BITS (APIC_IR_REGS * 32)
1420	#define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
1421
1422	union apic_ir {
1423	unsigned long map[APIC_IR_MAPSIZE];
1424	u32 regs[APIC_IR_REGS];
1425	};
1426
1427	static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
1428	{
1429	int i, bit;
1430
1431	/* Read the IRRs */
1432	for (i = 0; i < APIC_IR_REGS; i++)
1433	irr->regs[i] = apic_read(APIC_IRR + i * 0x10);
1434
1435	/* Read the ISRs */
1436	for (i = 0; i < APIC_IR_REGS; i++)
1437	isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1438
1439	/*
1440	* If the ISR map is not empty. ACK the APIC and run another round
1441	* to verify whether a pending IRR has been unblocked and turned
1442	* into a ISR.
1443	*/
1444	if (!bitmap_empty(src: isr->map, APIC_IR_BITS)) {
1445	/*
1446	* There can be multiple ISR bits set when a high priority
1447	* interrupt preempted a lower priority one. Issue an ACK
1448	* per set bit.
1449	*/
1450	for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1451	apic_eoi();
1452	return true;
1453	}
1454
1455	return !bitmap_empty(src: irr->map, APIC_IR_BITS);
1456	}
1457
1458	/*
1459	* After a crash, we no longer service the interrupts and a pending
1460	* interrupt from previous kernel might still have ISR bit set.
1461	*
1462	* Most probably by now the CPU has serviced that pending interrupt and it
1463	* might not have done the apic_eoi() because it thought, interrupt
1464	* came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
1465	* the ISR bit and cpu thinks it has already serviced the interrupt. Hence
1466	* a vector might get locked. It was noticed for timer irq (vector
1467	* 0x31). Issue an extra EOI to clear ISR.
1468	*
1469	* If there are pending IRR bits they turn into ISR bits after a higher
1470	* priority ISR bit has been acked.
1471	*/
1472	static void apic_pending_intr_clear(void)
1473	{
1474	union apic_ir irr, isr;
1475	unsigned int i;
1476
1477	/* 512 loops are way oversized and give the APIC a chance to obey. */
1478	for (i = 0; i < 512; i++) {
1479	if (!apic_check_and_ack(irr: &irr, isr: &isr))
1480	return;
1481	}
1482	/* Dump the IRR/ISR content if that failed */
1483	pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map);
1484	}
1485
1486	/**
1487	* setup_local_APIC - setup the local APIC
1488	*
1489	* Used to setup local APIC while initializing BSP or bringing up APs.
1490	* Always called with preemption disabled.
1491	*/
1492	static void setup_local_APIC(void)
1493	{
1494	int cpu = smp_processor_id();
1495	unsigned int value;
1496
1497	if (apic_is_disabled) {
1498	disable_ioapic_support();
1499	return;
1500	}
1501
1502	/*
1503	* If this comes from kexec/kcrash the APIC might be enabled in
1504	* SPIV. Soft disable it before doing further initialization.
1505	*/
1506	value = apic_read(APIC_SPIV);
1507	value &= ~APIC_SPIV_APIC_ENABLED;
1508	apic_write(APIC_SPIV, val: value);
1509
1510	#ifdef CONFIG_X86_32
1511	/* Pound the ESR really hard over the head with a big hammer - mbligh */
1512	if (lapic_is_integrated() && apic->disable_esr) {
1513	apic_write(APIC_ESR, 0);
1514	apic_write(APIC_ESR, 0);
1515	apic_write(APIC_ESR, 0);
1516	apic_write(APIC_ESR, 0);
1517	}
1518	#endif
1519	/*
1520	* Intel recommends to set DFR, LDR and TPR before enabling
1521	* an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1522	* document number 292116).
1523	*
1524	* Except for APICs which operate in physical destination mode.
1525	*/
1526	if (apic->init_apic_ldr)
1527	apic->init_apic_ldr();
1528
1529	/*
1530	* Set Task Priority to 'accept all except vectors 0-31'. An APIC
1531	* vector in the 16-31 range could be delivered if TPR == 0, but we
1532	* would think it's an exception and terrible things will happen. We
1533	* never change this later on.
1534	*/
1535	value = apic_read(APIC_TASKPRI);
1536	value &= ~APIC_TPRI_MASK;
1537	value |= 0x10;
1538	apic_write(APIC_TASKPRI, val: value);
1539
1540	/* Clear eventually stale ISR/IRR bits */
1541	apic_pending_intr_clear();
1542
1543	/*
1544	* Now that we are all set up, enable the APIC
1545	*/
1546	value = apic_read(APIC_SPIV);
1547	value &= ~APIC_VECTOR_MASK;
1548	/*
1549	* Enable APIC
1550	*/
1551	value |= APIC_SPIV_APIC_ENABLED;
1552
1553	#ifdef CONFIG_X86_32
1554	/*
1555	* Some unknown Intel IO/APIC (or APIC) errata is biting us with
1556	* certain networking cards. If high frequency interrupts are
1557	* happening on a particular IOAPIC pin, plus the IOAPIC routing
1558	* entry is masked/unmasked at a high rate as well then sooner or
1559	* later IOAPIC line gets 'stuck', no more interrupts are received
1560	* from the device. If focus CPU is disabled then the hang goes
1561	* away, oh well :-(
1562	*
1563	* [ This bug can be reproduced easily with a level-triggered
1564	* PCI Ne2000 networking cards and PII/PIII processors, dual
1565	* BX chipset. ]
1566	*/
1567	/*
1568	* Actually disabling the focus CPU check just makes the hang less
1569	* frequent as it makes the interrupt distribution model be more
1570	* like LRU than MRU (the short-term load is more even across CPUs).
1571	*/
1572
1573	/*
1574	* - enable focus processor (bit==0)
1575	* - 64bit mode always use processor focus
1576	* so no need to set it
1577	*/
1578	value &= ~APIC_SPIV_FOCUS_DISABLED;
1579	#endif
1580
1581	/*
1582	* Set spurious IRQ vector
1583	*/
1584	value |= SPURIOUS_APIC_VECTOR;
1585	apic_write(APIC_SPIV, val: value);
1586
1587	perf_events_lapic_init();
1588
1589	/*
1590	* Set up LVT0, LVT1:
1591	*
1592	* set up through-local-APIC on the boot CPU's LINT0. This is not
1593	* strictly necessary in pure symmetric-IO mode, but sometimes
1594	* we delegate interrupts to the 8259A.
1595	*/
1596	/*
1597	* TODO: set up through-local-APIC from through-I/O-APIC? --macro
1598	*/
1599	value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1600	if (!cpu && (pic_mode || !value || ioapic_is_disabled)) {
1601	value = APIC_DM_EXTINT;
1602	apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1603	} else {
1604	value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1605	apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1606	}
1607	apic_write(APIC_LVT0, val: value);
1608
1609	/*
1610	* Only the BSP sees the LINT1 NMI signal by default. This can be
1611	* modified by apic_extnmi= boot option.
1612	*/
1613	if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1614	apic_extnmi == APIC_EXTNMI_ALL)
1615	value = APIC_DM_NMI;
1616	else
1617	value = APIC_DM_NMI | APIC_LVT_MASKED;
1618
1619	/* Is 82489DX ? */
1620	if (!lapic_is_integrated())
1621	value |= APIC_LVT_LEVEL_TRIGGER;
1622	apic_write(APIC_LVT1, val: value);
1623
1624	#ifdef CONFIG_X86_MCE_INTEL
1625	/* Recheck CMCI information after local APIC is up on CPU #0 */
1626	if (!cpu)
1627	cmci_recheck();
1628	#endif
1629	}
1630
1631	static void end_local_APIC_setup(void)
1632	{
1633	lapic_setup_esr();
1634
1635	#ifdef CONFIG_X86_32
1636	{
1637	unsigned int value;
1638	/* Disable the local apic timer */
1639	value = apic_read(APIC_LVTT);
1640	value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1641	apic_write(APIC_LVTT, value);
1642	}
1643	#endif
1644
1645	apic_pm_activate();
1646	}
1647
1648	/*
1649	* APIC setup function for application processors. Called from smpboot.c
1650	*/
1651	void apic_ap_setup(void)
1652	{
1653	setup_local_APIC();
1654	end_local_APIC_setup();
1655	}
1656
1657	static __init void apic_read_boot_cpu_id(bool x2apic)
1658	{
1659	/*
1660	* This can be invoked from check_x2apic() before the APIC has been
1661	* selected. But that code knows for sure that the BIOS enabled
1662	* X2APIC.
1663	*/
1664	if (x2apic) {
1665	boot_cpu_physical_apicid = native_apic_msr_read(APIC_ID);
1666	boot_cpu_apic_version = GET_APIC_VERSION(native_apic_msr_read(APIC_LVR));
1667	} else {
1668	boot_cpu_physical_apicid = read_apic_id();
1669	boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
1670	}
1671	topology_register_boot_apic(apic_id: boot_cpu_physical_apicid);
1672	x86_32_probe_bigsmp_early();
1673	}
1674
1675	#ifdef CONFIG_X86_X2APIC
1676	int x2apic_mode;
1677	EXPORT_SYMBOL_GPL(x2apic_mode);
1678
1679	enum {
1680	X2APIC_OFF,
1681	X2APIC_DISABLED,
1682	/* All states below here have X2APIC enabled */
1683	X2APIC_ON,
1684	X2APIC_ON_LOCKED
1685	};
1686	static int x2apic_state;
1687
1688	static bool x2apic_hw_locked(void)
1689	{
1690	u64 x86_arch_cap_msr;
1691	u64 msr;
1692
1693	x86_arch_cap_msr = x86_read_arch_cap_msr();
1694	if (x86_arch_cap_msr & ARCH_CAP_XAPIC_DISABLE) {
1695	rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
1696	return (msr & LEGACY_XAPIC_DISABLED);
1697	}
1698	return false;
1699	}
1700
1701	static void __x2apic_disable(void)
1702	{
1703	u64 msr;
1704
1705	if (!boot_cpu_has(X86_FEATURE_APIC))
1706	return;
1707
1708	rdmsrl(MSR_IA32_APICBASE, msr);
1709	if (!(msr & X2APIC_ENABLE))
1710	return;
1711	/* Disable xapic and x2apic first and then reenable xapic mode */
1712	wrmsrl(MSR_IA32_APICBASE, val: msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1713	wrmsrl(MSR_IA32_APICBASE, val: msr & ~X2APIC_ENABLE);
1714	printk_once(KERN_INFO "x2apic disabled\n");
1715	}
1716
1717	static void __x2apic_enable(void)
1718	{
1719	u64 msr;
1720
1721	rdmsrl(MSR_IA32_APICBASE, msr);
1722	if (msr & X2APIC_ENABLE)
1723	return;
1724	wrmsrl(MSR_IA32_APICBASE, val: msr | X2APIC_ENABLE);
1725	printk_once(KERN_INFO "x2apic enabled\n");
1726	}
1727
1728	static int __init setup_nox2apic(char *str)
1729	{
1730	if (x2apic_enabled()) {
1731	u32 apicid = native_apic_msr_read(APIC_ID);
1732
1733	if (apicid >= 255) {
1734	pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1735	apicid);
1736	return 0;
1737	}
1738	if (x2apic_hw_locked()) {
1739	pr_warn("APIC locked in x2apic mode, can't disable\n");
1740	return 0;
1741	}
1742	pr_warn("x2apic already enabled.\n");
1743	__x2apic_disable();
1744	}
1745	setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1746	x2apic_state = X2APIC_DISABLED;
1747	x2apic_mode = 0;
1748	return 0;
1749	}
1750	early_param("nox2apic", setup_nox2apic);
1751
1752	/* Called from cpu_init() to enable x2apic on (secondary) cpus */
1753	void x2apic_setup(void)
1754	{
1755	/*
1756	* Try to make the AP's APIC state match that of the BSP, but if the
1757	* BSP is unlocked and the AP is locked then there is a state mismatch.
1758	* Warn about the mismatch in case a GP fault occurs due to a locked AP
1759	* trying to be turned off.
1760	*/
1761	if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
1762	pr_warn("x2apic lock mismatch between BSP and AP.\n");
1763	/*
1764	* If x2apic is not in ON or LOCKED state, disable it if already enabled
1765	* from BIOS.
1766	*/
1767	if (x2apic_state < X2APIC_ON) {
1768	__x2apic_disable();
1769	return;
1770	}
1771	__x2apic_enable();
1772	}
1773
1774	static __init void apic_set_fixmap(void);
1775
1776	static __init void x2apic_disable(void)
1777	{
1778	u32 x2apic_id, state = x2apic_state;
1779
1780	x2apic_mode = 0;
1781	x2apic_state = X2APIC_DISABLED;
1782
1783	if (state != X2APIC_ON)
1784	return;
1785
1786	x2apic_id = read_apic_id();
1787	if (x2apic_id >= 255)
1788	panic(fmt: "Cannot disable x2apic, id: %08x\n", x2apic_id);
1789
1790	if (x2apic_hw_locked()) {
1791	pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
1792	return;
1793	}
1794
1795	__x2apic_disable();
1796	apic_set_fixmap();
1797	}
1798
1799	static __init void x2apic_enable(void)
1800	{
1801	if (x2apic_state != X2APIC_OFF)
1802	return;
1803
1804	x2apic_mode = 1;
1805	x2apic_state = X2APIC_ON;
1806	__x2apic_enable();
1807	}
1808
1809	static __init void try_to_enable_x2apic(int remap_mode)
1810	{
1811	if (x2apic_state == X2APIC_DISABLED)
1812	return;
1813
1814	if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1815	u32 apic_limit = 255;
1816
1817	/*
1818	* Using X2APIC without IR is not architecturally supported
1819	* on bare metal but may be supported in guests.
1820	*/
1821	if (!x86_init.hyper.x2apic_available()) {
1822	pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1823	x2apic_disable();
1824	return;
1825	}
1826
1827	/*
1828	* If the hypervisor supports extended destination ID in
1829	* MSI, that increases the maximum APIC ID that can be
1830	* used for non-remapped IRQ domains.
1831	*/
1832	if (x86_init.hyper.msi_ext_dest_id()) {
1833	virt_ext_dest_id = 1;
1834	apic_limit = 32767;
1835	}
1836
1837	/*
1838	* Without IR, all CPUs can be addressed by IOAPIC/MSI only
1839	* in physical mode, and CPUs with an APIC ID that cannot
1840	* be addressed must not be brought online.
1841	*/
1842	x2apic_set_max_apicid(apicid: apic_limit);
1843	x2apic_phys = 1;
1844	}
1845	x2apic_enable();
1846	}
1847
1848	void __init check_x2apic(void)
1849	{
1850	if (x2apic_enabled()) {
1851	pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1852	x2apic_mode = 1;
1853	if (x2apic_hw_locked())
1854	x2apic_state = X2APIC_ON_LOCKED;
1855	else
1856	x2apic_state = X2APIC_ON;
1857	apic_read_boot_cpu_id(x2apic: true);
1858	} else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1859	x2apic_state = X2APIC_DISABLED;
1860	}
1861	}
1862	#else /* CONFIG_X86_X2APIC */
1863	void __init check_x2apic(void)
1864	{
1865	if (!apic_is_x2apic_enabled())
1866	return;
1867	/*
1868	* Checkme: Can we simply turn off x2APIC here instead of disabling the APIC?
1869	*/
1870	pr_err("Kernel does not support x2APIC, please recompile with CONFIG_X86_X2APIC.\n");
1871	pr_err("Disabling APIC, expect reduced performance and functionality.\n");
1872
1873	apic_is_disabled = true;
1874	setup_clear_cpu_cap(X86_FEATURE_APIC);
1875	}
1876
1877	static inline void try_to_enable_x2apic(int remap_mode) { }
1878	static inline void __x2apic_enable(void) { }
1879	#endif /* !CONFIG_X86_X2APIC */
1880
1881	void __init enable_IR_x2apic(void)
1882	{
1883	unsigned long flags;
1884	int ret, ir_stat;
1885
1886	if (ioapic_is_disabled) {
1887	pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1888	return;
1889	}
1890
1891	ir_stat = irq_remapping_prepare();
1892	if (ir_stat < 0 && !x2apic_supported())
1893	return;
1894
1895	ret = save_ioapic_entries();
1896	if (ret) {
1897	pr_info("Saving IO-APIC state failed: %d\n", ret);
1898	return;
1899	}
1900
1901	local_irq_save(flags);
1902	legacy_pic->mask_all();
1903	mask_ioapic_entries();
1904
1905	/* If irq_remapping_prepare() succeeded, try to enable it */
1906	if (ir_stat >= 0)
1907	ir_stat = irq_remapping_enable();
1908	/* ir_stat contains the remap mode or an error code */
1909	try_to_enable_x2apic(remap_mode: ir_stat);
1910
1911	if (ir_stat < 0)
1912	restore_ioapic_entries();
1913	legacy_pic->restore_mask();
1914	local_irq_restore(flags);
1915	}
1916
1917	#ifdef CONFIG_X86_64
1918	/*
1919	* Detect and enable local APICs on non-SMP boards.
1920	* Original code written by Keir Fraser.
1921	* On AMD64 we trust the BIOS - if it says no APIC it is likely
1922	* not correctly set up (usually the APIC timer won't work etc.)
1923	*/
1924	static bool __init detect_init_APIC(void)
1925	{
1926	if (!boot_cpu_has(X86_FEATURE_APIC)) {
1927	pr_info("No local APIC present\n");
1928	return false;
1929	}
1930
1931	register_lapic_address(APIC_DEFAULT_PHYS_BASE);
1932	return true;
1933	}
1934	#else
1935
1936	static bool __init apic_verify(unsigned long addr)
1937	{
1938	u32 features, h, l;
1939
1940	/*
1941	* The APIC feature bit should now be enabled
1942	* in `cpuid'
1943	*/
1944	features = cpuid_edx(1);
1945	if (!(features & (1 << X86_FEATURE_APIC))) {
1946	pr_warn("Could not enable APIC!\n");
1947	return false;
1948	}
1949	set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1950
1951	/* The BIOS may have set up the APIC at some other address */
1952	if (boot_cpu_data.x86 >= 6) {
1953	rdmsr(MSR_IA32_APICBASE, l, h);
1954	if (l & MSR_IA32_APICBASE_ENABLE)
1955	addr = l & MSR_IA32_APICBASE_BASE;
1956	}
1957
1958	register_lapic_address(addr);
1959	pr_info("Found and enabled local APIC!\n");
1960	return true;
1961	}
1962
1963	bool __init apic_force_enable(unsigned long addr)
1964	{
1965	u32 h, l;
1966
1967	if (apic_is_disabled)
1968	return false;
1969
1970	/*
1971	* Some BIOSes disable the local APIC in the APIC_BASE
1972	* MSR. This can only be done in software for Intel P6 or later
1973	* and AMD K7 (Model > 1) or later.
1974	*/
1975	if (boot_cpu_data.x86 >= 6) {
1976	rdmsr(MSR_IA32_APICBASE, l, h);
1977	if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1978	pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1979	l &= ~MSR_IA32_APICBASE_BASE;
1980	l |= MSR_IA32_APICBASE_ENABLE | addr;
1981	wrmsr(MSR_IA32_APICBASE, l, h);
1982	enabled_via_apicbase = 1;
1983	}
1984	}
1985	return apic_verify(addr);
1986	}
1987
1988	/*
1989	* Detect and initialize APIC
1990	*/
1991	static bool __init detect_init_APIC(void)
1992	{
1993	/* Disabled by kernel option? */
1994	if (apic_is_disabled)
1995	return false;
1996
1997	switch (boot_cpu_data.x86_vendor) {
1998	case X86_VENDOR_AMD:
1999	if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2000	(boot_cpu_data.x86 >= 15))
2001	break;
2002	goto no_apic;
2003	case X86_VENDOR_HYGON:
2004	break;
2005	case X86_VENDOR_INTEL:
2006	if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
2007	(boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)))
2008	break;
2009	goto no_apic;
2010	default:
2011	goto no_apic;
2012	}
2013
2014	if (!boot_cpu_has(X86_FEATURE_APIC)) {
2015	/*
2016	* Over-ride BIOS and try to enable the local APIC only if
2017	* "lapic" specified.
2018	*/
2019	if (!force_enable_local_apic) {
2020	pr_info("Local APIC disabled by BIOS -- "
2021	"you can enable it with \"lapic\"\n");
2022	return false;
2023	}
2024	if (!apic_force_enable(APIC_DEFAULT_PHYS_BASE))
2025	return false;
2026	} else {
2027	if (!apic_verify(APIC_DEFAULT_PHYS_BASE))
2028	return false;
2029	}
2030
2031	apic_pm_activate();
2032
2033	return true;
2034
2035	no_apic:
2036	pr_info("No local APIC present or hardware disabled\n");
2037	return false;
2038	}
2039	#endif
2040
2041	/**
2042	* init_apic_mappings - initialize APIC mappings
2043	*/
2044	void __init init_apic_mappings(void)
2045	{
2046	if (apic_validate_deadline_timer())
2047	pr_info("TSC deadline timer available\n");
2048
2049	if (x2apic_mode)
2050	return;
2051
2052	if (!smp_found_config) {
2053	if (!detect_init_APIC()) {
2054	pr_info("APIC: disable apic facility\n");
2055	apic_disable();
2056	}
2057	}
2058	}
2059
2060	static __init void apic_set_fixmap(void)
2061	{
2062	set_fixmap_nocache(FIX_APIC_BASE, mp_lapic_addr);
2063	apic_mmio_base = APIC_BASE;
2064	apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
2065	apic_mmio_base, mp_lapic_addr);
2066	apic_read_boot_cpu_id(x2apic: false);
2067	}
2068
2069	void __init register_lapic_address(unsigned long address)
2070	{
2071	/* This should only happen once */
2072	WARN_ON_ONCE(mp_lapic_addr);
2073	mp_lapic_addr = address;
2074
2075	if (!x2apic_mode)
2076	apic_set_fixmap();
2077	}
2078
2079	/*
2080	* Local APIC interrupts
2081	*/
2082
2083	/*
2084	* Common handling code for spurious_interrupt and spurious_vector entry
2085	* points below. No point in allowing the compiler to inline it twice.
2086	*/
2087	static noinline void handle_spurious_interrupt(u8 vector)
2088	{
2089	u32 v;
2090
2091	trace_spurious_apic_entry(vector);
2092
2093	inc_irq_stat(irq_spurious_count);
2094
2095	/*
2096	* If this is a spurious interrupt then do not acknowledge
2097	*/
2098	if (vector == SPURIOUS_APIC_VECTOR) {
2099	/* See SDM vol 3 */
2100	pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2101	smp_processor_id());
2102	goto out;
2103	}
2104
2105	/*
2106	* If it is a vectored one, verify it's set in the ISR. If set,
2107	* acknowledge it.
2108	*/
2109	v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2110	if (v & (1 << (vector & 0x1f))) {
2111	pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2112	vector, smp_processor_id());
2113	apic_eoi();
2114	} else {
2115	pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2116	vector, smp_processor_id());
2117	}
2118	out:
2119	trace_spurious_apic_exit(vector);
2120	}
2121
2122	/**
2123	* spurious_interrupt - Catch all for interrupts raised on unused vectors
2124	* @regs: Pointer to pt_regs on stack
2125	* @vector: The vector number
2126	*
2127	* This is invoked from ASM entry code to catch all interrupts which
2128	* trigger on an entry which is routed to the common_spurious idtentry
2129	* point.
2130	*/
2131	DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2132	{
2133	handle_spurious_interrupt(vector);
2134	}
2135
2136	DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2137	{
2138	handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2139	}
2140
2141	/*
2142	* This interrupt should never happen with our APIC/SMP architecture
2143	*/
2144	DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2145	{
2146	static const char * const error_interrupt_reason[] = {
2147	"Send CS error", /* APIC Error Bit 0 */
2148	"Receive CS error", /* APIC Error Bit 1 */
2149	"Send accept error", /* APIC Error Bit 2 */
2150	"Receive accept error", /* APIC Error Bit 3 */
2151	"Redirectable IPI", /* APIC Error Bit 4 */
2152	"Send illegal vector", /* APIC Error Bit 5 */
2153	"Received illegal vector", /* APIC Error Bit 6 */
2154	"Illegal register address", /* APIC Error Bit 7 */
2155	};
2156	u32 v, i = 0;
2157
2158	trace_error_apic_entry(ERROR_APIC_VECTOR);
2159
2160	/* First tickle the hardware, only then report what went on. -- REW */
2161	if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2162	apic_write(APIC_ESR, val: 0);
2163	v = apic_read(APIC_ESR);
2164	apic_eoi();
2165	atomic_inc(v: &irq_err_count);
2166
2167	apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
2168	smp_processor_id(), v);
2169
2170	v &= 0xff;
2171	while (v) {
2172	if (v & 0x1)
2173	apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
2174	i++;
2175	v >>= 1;
2176	}
2177
2178	apic_printk(APIC_DEBUG, KERN_CONT "\n");
2179
2180	trace_error_apic_exit(ERROR_APIC_VECTOR);
2181	}
2182
2183	/**
2184	* connect_bsp_APIC - attach the APIC to the interrupt system
2185	*/
2186	static void __init connect_bsp_APIC(void)
2187	{
2188	#ifdef CONFIG_X86_32
2189	if (pic_mode) {
2190	/*
2191	* Do not trust the local APIC being empty at bootup.
2192	*/
2193	clear_local_APIC();
2194	/*
2195	* PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2196	* local APIC to INT and NMI lines.
2197	*/
2198	apic_printk(APIC_VERBOSE, "leaving PIC mode, "
2199	"enabling APIC mode.\n");
2200	imcr_pic_to_apic();
2201	}
2202	#endif
2203	}
2204
2205	/**
2206	* disconnect_bsp_APIC - detach the APIC from the interrupt system
2207	* @virt_wire_setup: indicates, whether virtual wire mode is selected
2208	*
2209	* Virtual wire mode is necessary to deliver legacy interrupts even when the
2210	* APIC is disabled.
2211	*/
2212	void disconnect_bsp_APIC(int virt_wire_setup)
2213	{
2214	unsigned int value;
2215
2216	#ifdef CONFIG_X86_32
2217	if (pic_mode) {
2218	/*
2219	* Put the board back into PIC mode (has an effect only on
2220	* certain older boards). Note that APIC interrupts, including
2221	* IPIs, won't work beyond this point! The only exception are
2222	* INIT IPIs.
2223	*/
2224	apic_printk(APIC_VERBOSE, "disabling APIC mode, "
2225	"entering PIC mode.\n");
2226	imcr_apic_to_pic();
2227	return;
2228	}
2229	#endif
2230
2231	/* Go back to Virtual Wire compatibility mode */
2232
2233	/* For the spurious interrupt use vector F, and enable it */
2234	value = apic_read(APIC_SPIV);
2235	value &= ~APIC_VECTOR_MASK;
2236	value |= APIC_SPIV_APIC_ENABLED;
2237	value |= 0xf;
2238	apic_write(APIC_SPIV, val: value);
2239
2240	if (!virt_wire_setup) {
2241	/*
2242	* For LVT0 make it edge triggered, active high,
2243	* external and enabled
2244	*/
2245	value = apic_read(APIC_LVT0);
2246	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2247	APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2248	APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2249	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2250	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2251	apic_write(APIC_LVT0, val: value);
2252	} else {
2253	/* Disable LVT0 */
2254	apic_write(APIC_LVT0, APIC_LVT_MASKED);
2255	}
2256
2257	/*
2258	* For LVT1 make it edge triggered, active high,
2259	* nmi and enabled
2260	*/
2261	value = apic_read(APIC_LVT1);
2262	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2263	APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2264	APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2265	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2266	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2267	apic_write(APIC_LVT1, val: value);
2268	}
2269
2270	void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2271	bool dmar)
2272	{
2273	memset(msg, 0, sizeof(*msg));
2274
2275	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
2276	msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
2277	msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2278
2279	msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
2280	msg->arch_data.vector = cfg->vector;
2281
2282	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2283	/*
2284	* Only the IOMMU itself can use the trick of putting destination
2285	* APIC ID into the high bits of the address. Anything else would
2286	* just be writing to memory if it tried that, and needs IR to
2287	* address APICs which can't be addressed in the normal 32-bit
2288	* address range at 0xFFExxxxx. That is typically just 8 bits, but
2289	* some hypervisors allow the extended destination ID field in bits
2290	* 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2291	*/
2292	if (dmar)
2293	msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2294	else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2295	msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2296	else
2297	WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2298	}
2299
2300	u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
2301	{
2302	u32 dest = msg->arch_addr_lo.destid_0_7;
2303
2304	if (extid)
2305	dest |= msg->arch_addr_hi.destid_8_31 << 8;
2306	return dest;
2307	}
2308	EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
2309
2310	static void __init apic_bsp_up_setup(void)
2311	{
2312	reset_phys_cpu_present_map(apicid: boot_cpu_physical_apicid);
2313	}
2314
2315	/**
2316	* apic_bsp_setup - Setup function for local apic and io-apic
2317	* @upmode: Force UP mode (for APIC_init_uniprocessor)
2318	*/
2319	static void __init apic_bsp_setup(bool upmode)
2320	{
2321	connect_bsp_APIC();
2322	if (upmode)
2323	apic_bsp_up_setup();
2324	setup_local_APIC();
2325
2326	enable_IO_APIC();
2327	end_local_APIC_setup();
2328	irq_remap_enable_fault_handling();
2329	setup_IO_APIC();
2330	lapic_update_legacy_vectors();
2331	}
2332
2333	#ifdef CONFIG_UP_LATE_INIT
2334	void __init up_late_init(void)
2335	{
2336	if (apic_intr_mode == APIC_PIC)
2337	return;
2338
2339	/* Setup local timer */
2340	x86_init.timers.setup_percpu_clockev();
2341	}
2342	#endif
2343
2344	/*
2345	* Power management
2346	*/
2347	#ifdef CONFIG_PM
2348
2349	static struct {
2350	/*
2351	* 'active' is true if the local APIC was enabled by us and
2352	* not the BIOS; this signifies that we are also responsible
2353	* for disabling it before entering apm/acpi suspend
2354	*/
2355	int active;
2356	/* r/w apic fields */
2357	u32 apic_id;
2358	unsigned int apic_taskpri;
2359	unsigned int apic_ldr;
2360	unsigned int apic_dfr;
2361	unsigned int apic_spiv;
2362	unsigned int apic_lvtt;
2363	unsigned int apic_lvtpc;
2364	unsigned int apic_lvt0;
2365	unsigned int apic_lvt1;
2366	unsigned int apic_lvterr;
2367	unsigned int apic_tmict;
2368	unsigned int apic_tdcr;
2369	unsigned int apic_thmr;
2370	unsigned int apic_cmci;
2371	} apic_pm_state;
2372
2373	static int lapic_suspend(void)
2374	{
2375	unsigned long flags;
2376	int maxlvt;
2377
2378	if (!apic_pm_state.active)
2379	return 0;
2380
2381	maxlvt = lapic_get_maxlvt();
2382
2383	apic_pm_state.apic_id = apic_read(APIC_ID);
2384	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2385	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2386	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2387	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2388	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2389	if (maxlvt >= 4)
2390	apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2391	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2392	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2393	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2394	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2395	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2396	#ifdef CONFIG_X86_THERMAL_VECTOR
2397	if (maxlvt >= 5)
2398	apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2399	#endif
2400	#ifdef CONFIG_X86_MCE_INTEL
2401	if (maxlvt >= 6)
2402	apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2403	#endif
2404
2405	local_irq_save(flags);
2406
2407	/*
2408	* Mask IOAPIC before disabling the local APIC to prevent stale IRR
2409	* entries on some implementations.
2410	*/
2411	mask_ioapic_entries();
2412
2413	disable_local_APIC();
2414
2415	irq_remapping_disable();
2416
2417	local_irq_restore(flags);
2418	return 0;
2419	}
2420
2421	static void lapic_resume(void)
2422	{
2423	unsigned int l, h;
2424	unsigned long flags;
2425	int maxlvt;
2426
2427	if (!apic_pm_state.active)
2428	return;
2429
2430	local_irq_save(flags);
2431
2432	/*
2433	* IO-APIC and PIC have their own resume routines.
2434	* We just mask them here to make sure the interrupt
2435	* subsystem is completely quiet while we enable x2apic
2436	* and interrupt-remapping.
2437	*/
2438	mask_ioapic_entries();
2439	legacy_pic->mask_all();
2440
2441	if (x2apic_mode) {
2442	__x2apic_enable();
2443	} else {
2444	/*
2445	* Make sure the APICBASE points to the right address
2446	*
2447	* FIXME! This will be wrong if we ever support suspend on
2448	* SMP! We'll need to do this as part of the CPU restore!
2449	*/
2450	if (boot_cpu_data.x86 >= 6) {
2451	rdmsr(MSR_IA32_APICBASE, l, h);
2452	l &= ~MSR_IA32_APICBASE_BASE;
2453	l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2454	wrmsr(MSR_IA32_APICBASE, l, h);
2455	}
2456	}
2457
2458	maxlvt = lapic_get_maxlvt();
2459	apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2460	apic_write(APIC_ID, val: apic_pm_state.apic_id);
2461	apic_write(APIC_DFR, val: apic_pm_state.apic_dfr);
2462	apic_write(APIC_LDR, val: apic_pm_state.apic_ldr);
2463	apic_write(APIC_TASKPRI, val: apic_pm_state.apic_taskpri);
2464	apic_write(APIC_SPIV, val: apic_pm_state.apic_spiv);
2465	apic_write(APIC_LVT0, val: apic_pm_state.apic_lvt0);
2466	apic_write(APIC_LVT1, val: apic_pm_state.apic_lvt1);
2467	#ifdef CONFIG_X86_THERMAL_VECTOR
2468	if (maxlvt >= 5)
2469	apic_write(APIC_LVTTHMR, val: apic_pm_state.apic_thmr);
2470	#endif
2471	#ifdef CONFIG_X86_MCE_INTEL
2472	if (maxlvt >= 6)
2473	apic_write(APIC_LVTCMCI, val: apic_pm_state.apic_cmci);
2474	#endif
2475	if (maxlvt >= 4)
2476	apic_write(APIC_LVTPC, val: apic_pm_state.apic_lvtpc);
2477	apic_write(APIC_LVTT, val: apic_pm_state.apic_lvtt);
2478	apic_write(APIC_TDCR, val: apic_pm_state.apic_tdcr);
2479	apic_write(APIC_TMICT, val: apic_pm_state.apic_tmict);
2480	apic_write(APIC_ESR, val: 0);
2481	apic_read(APIC_ESR);
2482	apic_write(APIC_LVTERR, val: apic_pm_state.apic_lvterr);
2483	apic_write(APIC_ESR, val: 0);
2484	apic_read(APIC_ESR);
2485
2486	irq_remapping_reenable(x2apic_mode);
2487
2488	local_irq_restore(flags);
2489	}
2490
2491	/*
2492	* This device has no shutdown method - fully functioning local APICs
2493	* are needed on every CPU up until machine_halt/restart/poweroff.
2494	*/
2495
2496	static struct syscore_ops lapic_syscore_ops = {
2497	.resume = lapic_resume,
2498	.suspend = lapic_suspend,
2499	};
2500
2501	static void apic_pm_activate(void)
2502	{
2503	apic_pm_state.active = 1;
2504	}
2505
2506	static int __init init_lapic_sysfs(void)
2507	{
2508	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2509	if (boot_cpu_has(X86_FEATURE_APIC))
2510	register_syscore_ops(ops: &lapic_syscore_ops);
2511
2512	return 0;
2513	}
2514
2515	/* local apic needs to resume before other devices access its registers. */
2516	core_initcall(init_lapic_sysfs);
2517
2518	#else /* CONFIG_PM */
2519
2520	static void apic_pm_activate(void) { }
2521
2522	#endif /* CONFIG_PM */
2523
2524	#ifdef CONFIG_X86_64
2525
2526	static int multi_checked;
2527	static int multi;
2528
2529	static int set_multi(const struct dmi_system_id *d)
2530	{
2531	if (multi)
2532	return 0;
2533	pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2534	multi = 1;
2535	return 0;
2536	}
2537
2538	static const struct dmi_system_id multi_dmi_table[] = {
2539	{
2540	.callback = set_multi,
2541	.ident = "IBM System Summit2",
2542	.matches = {
2543	DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2544	DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2545	},
2546	},
2547	{}
2548	};
2549
2550	static void dmi_check_multi(void)
2551	{
2552	if (multi_checked)
2553	return;
2554
2555	dmi_check_system(list: multi_dmi_table);
2556	multi_checked = 1;
2557	}
2558
2559	/*
2560	* apic_is_clustered_box() -- Check if we can expect good TSC
2561	*
2562	* Thus far, the major user of this is IBM's Summit2 series:
2563	* Clustered boxes may have unsynced TSC problems if they are
2564	* multi-chassis.
2565	* Use DMI to check them
2566	*/
2567	int apic_is_clustered_box(void)
2568	{
2569	dmi_check_multi();
2570	return multi;
2571	}
2572	#endif
2573
2574	/*
2575	* APIC command line parameters
2576	*/
2577	static int __init setup_disableapic(char *arg)
2578	{
2579	apic_is_disabled = true;
2580	setup_clear_cpu_cap(X86_FEATURE_APIC);
2581	return 0;
2582	}
2583	early_param("disableapic", setup_disableapic);
2584
2585	/* same as disableapic, for compatibility */
2586	static int __init setup_nolapic(char *arg)
2587	{
2588	return setup_disableapic(arg);
2589	}
2590	early_param("nolapic", setup_nolapic);
2591
2592	static int __init parse_lapic_timer_c2_ok(char *arg)
2593	{
2594	local_apic_timer_c2_ok = 1;
2595	return 0;
2596	}
2597	early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2598
2599	static int __init parse_disable_apic_timer(char *arg)
2600	{
2601	disable_apic_timer = 1;
2602	return 0;
2603	}
2604	early_param("noapictimer", parse_disable_apic_timer);
2605
2606	static int __init parse_nolapic_timer(char *arg)
2607	{
2608	disable_apic_timer = 1;
2609	return 0;
2610	}
2611	early_param("nolapic_timer", parse_nolapic_timer);
2612
2613	static int __init apic_set_verbosity(char *arg)
2614	{
2615	if (!arg) {
2616	if (IS_ENABLED(CONFIG_X86_32))
2617	return -EINVAL;
2618
2619	ioapic_is_disabled = false;
2620	return 0;
2621	}
2622
2623	if (strcmp("debug", arg) == 0)
2624	apic_verbosity = APIC_DEBUG;
2625	else if (strcmp("verbose", arg) == 0)
2626	apic_verbosity = APIC_VERBOSE;
2627	#ifdef CONFIG_X86_64
2628	else {
2629	pr_warn("APIC Verbosity level %s not recognised"
2630	" use apic=verbose or apic=debug\n", arg);
2631	return -EINVAL;
2632	}
2633	#endif
2634
2635	return 0;
2636	}
2637	early_param("apic", apic_set_verbosity);
2638
2639	static int __init lapic_insert_resource(void)
2640	{
2641	if (!apic_mmio_base)
2642	return -1;
2643
2644	/* Put local APIC into the resource map. */
2645	lapic_resource.start = apic_mmio_base;
2646	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2647	insert_resource(parent: &iomem_resource, new: &lapic_resource);
2648
2649	return 0;
2650	}
2651
2652	/*
2653	* need call insert after e820__reserve_resources()
2654	* that is using request_resource
2655	*/
2656	late_initcall(lapic_insert_resource);
2657
2658	static int __init apic_set_extnmi(char *arg)
2659	{
2660	if (!arg)
2661	return -EINVAL;
2662
2663	if (!strncmp("all", arg, 3))
2664	apic_extnmi = APIC_EXTNMI_ALL;
2665	else if (!strncmp("none", arg, 4))
2666	apic_extnmi = APIC_EXTNMI_NONE;
2667	else if (!strncmp("bsp", arg, 3))
2668	apic_extnmi = APIC_EXTNMI_BSP;
2669	else {
2670	pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2671	return -EINVAL;
2672	}
2673
2674	return 0;
2675	}
2676	early_param("apic_extnmi", apic_set_extnmi);
2677