1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Common interrupt code for 32 and 64 bit |
4 | */ |
5 | #include <linux/cpu.h> |
6 | #include <linux/interrupt.h> |
7 | #include <linux/kernel_stat.h> |
8 | #include <linux/of.h> |
9 | #include <linux/seq_file.h> |
10 | #include <linux/smp.h> |
11 | #include <linux/ftrace.h> |
12 | #include <linux/delay.h> |
13 | #include <linux/export.h> |
14 | #include <linux/irq.h> |
15 | |
16 | #include <asm/irq_stack.h> |
17 | #include <asm/apic.h> |
18 | #include <asm/io_apic.h> |
19 | #include <asm/irq.h> |
20 | #include <asm/mce.h> |
21 | #include <asm/hw_irq.h> |
22 | #include <asm/desc.h> |
23 | #include <asm/traps.h> |
24 | #include <asm/thermal.h> |
25 | |
26 | #define CREATE_TRACE_POINTS |
27 | #include <asm/trace/irq_vectors.h> |
28 | |
29 | DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); |
30 | EXPORT_PER_CPU_SYMBOL(irq_stat); |
31 | |
32 | atomic_t irq_err_count; |
33 | |
34 | /* |
35 | * 'what should we do if we get a hw irq event on an illegal vector'. |
36 | * each architecture has to answer this themselves. |
37 | */ |
38 | void ack_bad_irq(unsigned int irq) |
39 | { |
40 | if (printk_ratelimit()) |
41 | pr_err("unexpected IRQ trap at vector %02x\n" , irq); |
42 | |
43 | /* |
44 | * Currently unexpected vectors happen only on SMP and APIC. |
45 | * We _must_ ack these because every local APIC has only N |
46 | * irq slots per priority level, and a 'hanging, unacked' IRQ |
47 | * holds up an irq slot - in excessive cases (when multiple |
48 | * unexpected vectors occur) that might lock up the APIC |
49 | * completely. |
50 | * But only ack when the APIC is enabled -AK |
51 | */ |
52 | apic_eoi(); |
53 | } |
54 | |
55 | #define irq_stats(x) (&per_cpu(irq_stat, x)) |
56 | /* |
57 | * /proc/interrupts printing for arch specific interrupts |
58 | */ |
59 | int arch_show_interrupts(struct seq_file *p, int prec) |
60 | { |
61 | int j; |
62 | |
63 | seq_printf(m: p, fmt: "%*s: " , prec, "NMI" ); |
64 | for_each_online_cpu(j) |
65 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->__nmi_count); |
66 | seq_puts(m: p, s: " Non-maskable interrupts\n" ); |
67 | #ifdef CONFIG_X86_LOCAL_APIC |
68 | seq_printf(m: p, fmt: "%*s: " , prec, "LOC" ); |
69 | for_each_online_cpu(j) |
70 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->apic_timer_irqs); |
71 | seq_puts(m: p, s: " Local timer interrupts\n" ); |
72 | |
73 | seq_printf(m: p, fmt: "%*s: " , prec, "SPU" ); |
74 | for_each_online_cpu(j) |
75 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_spurious_count); |
76 | seq_puts(m: p, s: " Spurious interrupts\n" ); |
77 | seq_printf(m: p, fmt: "%*s: " , prec, "PMI" ); |
78 | for_each_online_cpu(j) |
79 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->apic_perf_irqs); |
80 | seq_puts(m: p, s: " Performance monitoring interrupts\n" ); |
81 | seq_printf(m: p, fmt: "%*s: " , prec, "IWI" ); |
82 | for_each_online_cpu(j) |
83 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->apic_irq_work_irqs); |
84 | seq_puts(m: p, s: " IRQ work interrupts\n" ); |
85 | seq_printf(m: p, fmt: "%*s: " , prec, "RTR" ); |
86 | for_each_online_cpu(j) |
87 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->icr_read_retry_count); |
88 | seq_puts(m: p, s: " APIC ICR read retries\n" ); |
89 | if (x86_platform_ipi_callback) { |
90 | seq_printf(m: p, fmt: "%*s: " , prec, "PLT" ); |
91 | for_each_online_cpu(j) |
92 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->x86_platform_ipis); |
93 | seq_puts(m: p, s: " Platform interrupts\n" ); |
94 | } |
95 | #endif |
96 | #ifdef CONFIG_SMP |
97 | seq_printf(m: p, fmt: "%*s: " , prec, "RES" ); |
98 | for_each_online_cpu(j) |
99 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_resched_count); |
100 | seq_puts(m: p, s: " Rescheduling interrupts\n" ); |
101 | seq_printf(m: p, fmt: "%*s: " , prec, "CAL" ); |
102 | for_each_online_cpu(j) |
103 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_call_count); |
104 | seq_puts(m: p, s: " Function call interrupts\n" ); |
105 | seq_printf(m: p, fmt: "%*s: " , prec, "TLB" ); |
106 | for_each_online_cpu(j) |
107 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_tlb_count); |
108 | seq_puts(m: p, s: " TLB shootdowns\n" ); |
109 | #endif |
110 | #ifdef CONFIG_X86_THERMAL_VECTOR |
111 | seq_printf(m: p, fmt: "%*s: " , prec, "TRM" ); |
112 | for_each_online_cpu(j) |
113 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_thermal_count); |
114 | seq_puts(m: p, s: " Thermal event interrupts\n" ); |
115 | #endif |
116 | #ifdef CONFIG_X86_MCE_THRESHOLD |
117 | seq_printf(m: p, fmt: "%*s: " , prec, "THR" ); |
118 | for_each_online_cpu(j) |
119 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_threshold_count); |
120 | seq_puts(m: p, s: " Threshold APIC interrupts\n" ); |
121 | #endif |
122 | #ifdef CONFIG_X86_MCE_AMD |
123 | seq_printf(m: p, fmt: "%*s: " , prec, "DFR" ); |
124 | for_each_online_cpu(j) |
125 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->irq_deferred_error_count); |
126 | seq_puts(m: p, s: " Deferred Error APIC interrupts\n" ); |
127 | #endif |
128 | #ifdef CONFIG_X86_MCE |
129 | seq_printf(m: p, fmt: "%*s: " , prec, "MCE" ); |
130 | for_each_online_cpu(j) |
131 | seq_printf(m: p, fmt: "%10u " , per_cpu(mce_exception_count, j)); |
132 | seq_puts(m: p, s: " Machine check exceptions\n" ); |
133 | seq_printf(m: p, fmt: "%*s: " , prec, "MCP" ); |
134 | for_each_online_cpu(j) |
135 | seq_printf(m: p, fmt: "%10u " , per_cpu(mce_poll_count, j)); |
136 | seq_puts(m: p, s: " Machine check polls\n" ); |
137 | #endif |
138 | #ifdef CONFIG_X86_HV_CALLBACK_VECTOR |
139 | if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) { |
140 | seq_printf(m: p, fmt: "%*s: " , prec, "HYP" ); |
141 | for_each_online_cpu(j) |
142 | seq_printf(m: p, fmt: "%10u " , |
143 | irq_stats(j)->irq_hv_callback_count); |
144 | seq_puts(m: p, s: " Hypervisor callback interrupts\n" ); |
145 | } |
146 | #endif |
147 | #if IS_ENABLED(CONFIG_HYPERV) |
148 | if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) { |
149 | seq_printf(m: p, fmt: "%*s: " , prec, "HRE" ); |
150 | for_each_online_cpu(j) |
151 | seq_printf(m: p, fmt: "%10u " , |
152 | irq_stats(j)->irq_hv_reenlightenment_count); |
153 | seq_puts(m: p, s: " Hyper-V reenlightenment interrupts\n" ); |
154 | } |
155 | if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) { |
156 | seq_printf(m: p, fmt: "%*s: " , prec, "HVS" ); |
157 | for_each_online_cpu(j) |
158 | seq_printf(m: p, fmt: "%10u " , |
159 | irq_stats(j)->hyperv_stimer0_count); |
160 | seq_puts(m: p, s: " Hyper-V stimer0 interrupts\n" ); |
161 | } |
162 | #endif |
163 | seq_printf(m: p, fmt: "%*s: %10u\n" , prec, "ERR" , atomic_read(v: &irq_err_count)); |
164 | #if defined(CONFIG_X86_IO_APIC) |
165 | seq_printf(m: p, fmt: "%*s: %10u\n" , prec, "MIS" , atomic_read(v: &irq_mis_count)); |
166 | #endif |
167 | #ifdef CONFIG_HAVE_KVM |
168 | seq_printf(m: p, fmt: "%*s: " , prec, "PIN" ); |
169 | for_each_online_cpu(j) |
170 | seq_printf(m: p, fmt: "%10u " , irq_stats(j)->kvm_posted_intr_ipis); |
171 | seq_puts(m: p, s: " Posted-interrupt notification event\n" ); |
172 | |
173 | seq_printf(m: p, fmt: "%*s: " , prec, "NPI" ); |
174 | for_each_online_cpu(j) |
175 | seq_printf(m: p, fmt: "%10u " , |
176 | irq_stats(j)->kvm_posted_intr_nested_ipis); |
177 | seq_puts(m: p, s: " Nested posted-interrupt event\n" ); |
178 | |
179 | seq_printf(m: p, fmt: "%*s: " , prec, "PIW" ); |
180 | for_each_online_cpu(j) |
181 | seq_printf(m: p, fmt: "%10u " , |
182 | irq_stats(j)->kvm_posted_intr_wakeup_ipis); |
183 | seq_puts(m: p, s: " Posted-interrupt wakeup event\n" ); |
184 | #endif |
185 | return 0; |
186 | } |
187 | |
188 | /* |
189 | * /proc/stat helpers |
190 | */ |
191 | u64 arch_irq_stat_cpu(unsigned int cpu) |
192 | { |
193 | u64 sum = irq_stats(cpu)->__nmi_count; |
194 | |
195 | #ifdef CONFIG_X86_LOCAL_APIC |
196 | sum += irq_stats(cpu)->apic_timer_irqs; |
197 | sum += irq_stats(cpu)->irq_spurious_count; |
198 | sum += irq_stats(cpu)->apic_perf_irqs; |
199 | sum += irq_stats(cpu)->apic_irq_work_irqs; |
200 | sum += irq_stats(cpu)->icr_read_retry_count; |
201 | if (x86_platform_ipi_callback) |
202 | sum += irq_stats(cpu)->x86_platform_ipis; |
203 | #endif |
204 | #ifdef CONFIG_SMP |
205 | sum += irq_stats(cpu)->irq_resched_count; |
206 | sum += irq_stats(cpu)->irq_call_count; |
207 | #endif |
208 | #ifdef CONFIG_X86_THERMAL_VECTOR |
209 | sum += irq_stats(cpu)->irq_thermal_count; |
210 | #endif |
211 | #ifdef CONFIG_X86_MCE_THRESHOLD |
212 | sum += irq_stats(cpu)->irq_threshold_count; |
213 | #endif |
214 | #ifdef CONFIG_X86_HV_CALLBACK_VECTOR |
215 | sum += irq_stats(cpu)->irq_hv_callback_count; |
216 | #endif |
217 | #if IS_ENABLED(CONFIG_HYPERV) |
218 | sum += irq_stats(cpu)->irq_hv_reenlightenment_count; |
219 | sum += irq_stats(cpu)->hyperv_stimer0_count; |
220 | #endif |
221 | #ifdef CONFIG_X86_MCE |
222 | sum += per_cpu(mce_exception_count, cpu); |
223 | sum += per_cpu(mce_poll_count, cpu); |
224 | #endif |
225 | return sum; |
226 | } |
227 | |
228 | u64 arch_irq_stat(void) |
229 | { |
230 | u64 sum = atomic_read(v: &irq_err_count); |
231 | return sum; |
232 | } |
233 | |
234 | static __always_inline void handle_irq(struct irq_desc *desc, |
235 | struct pt_regs *regs) |
236 | { |
237 | if (IS_ENABLED(CONFIG_X86_64)) |
238 | generic_handle_irq_desc(desc); |
239 | else |
240 | __handle_irq(desc, regs); |
241 | } |
242 | |
243 | /* |
244 | * common_interrupt() handles all normal device IRQ's (the special SMP |
245 | * cross-CPU interrupts have their own entry points). |
246 | */ |
247 | DEFINE_IDTENTRY_IRQ(common_interrupt) |
248 | { |
249 | struct pt_regs *old_regs = set_irq_regs(regs); |
250 | struct irq_desc *desc; |
251 | |
252 | /* entry code tells RCU that we're not quiescent. Check it. */ |
253 | RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU" ); |
254 | |
255 | desc = __this_cpu_read(vector_irq[vector]); |
256 | if (likely(!IS_ERR_OR_NULL(desc))) { |
257 | handle_irq(desc, regs); |
258 | } else { |
259 | apic_eoi(); |
260 | |
261 | if (desc == VECTOR_UNUSED) { |
262 | pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n" , |
263 | __func__, smp_processor_id(), |
264 | vector); |
265 | } else { |
266 | __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); |
267 | } |
268 | } |
269 | |
270 | set_irq_regs(old_regs); |
271 | } |
272 | |
273 | #ifdef CONFIG_X86_LOCAL_APIC |
274 | /* Function pointer for generic interrupt vector handling */ |
275 | void (*x86_platform_ipi_callback)(void) = NULL; |
276 | /* |
277 | * Handler for X86_PLATFORM_IPI_VECTOR. |
278 | */ |
279 | DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) |
280 | { |
281 | struct pt_regs *old_regs = set_irq_regs(regs); |
282 | |
283 | apic_eoi(); |
284 | trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); |
285 | inc_irq_stat(x86_platform_ipis); |
286 | if (x86_platform_ipi_callback) |
287 | x86_platform_ipi_callback(); |
288 | trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR); |
289 | set_irq_regs(old_regs); |
290 | } |
291 | #endif |
292 | |
293 | #ifdef CONFIG_HAVE_KVM |
294 | static void dummy_handler(void) {} |
295 | static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler; |
296 | |
297 | void kvm_set_posted_intr_wakeup_handler(void (*handler)(void)) |
298 | { |
299 | if (handler) |
300 | kvm_posted_intr_wakeup_handler = handler; |
301 | else { |
302 | kvm_posted_intr_wakeup_handler = dummy_handler; |
303 | synchronize_rcu(); |
304 | } |
305 | } |
306 | EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler); |
307 | |
308 | /* |
309 | * Handler for POSTED_INTERRUPT_VECTOR. |
310 | */ |
311 | DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) |
312 | { |
313 | apic_eoi(); |
314 | inc_irq_stat(kvm_posted_intr_ipis); |
315 | } |
316 | |
317 | /* |
318 | * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. |
319 | */ |
320 | DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) |
321 | { |
322 | apic_eoi(); |
323 | inc_irq_stat(kvm_posted_intr_wakeup_ipis); |
324 | kvm_posted_intr_wakeup_handler(); |
325 | } |
326 | |
327 | /* |
328 | * Handler for POSTED_INTERRUPT_NESTED_VECTOR. |
329 | */ |
330 | DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) |
331 | { |
332 | apic_eoi(); |
333 | inc_irq_stat(kvm_posted_intr_nested_ipis); |
334 | } |
335 | #endif |
336 | |
337 | |
338 | #ifdef CONFIG_HOTPLUG_CPU |
339 | /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ |
340 | void fixup_irqs(void) |
341 | { |
342 | unsigned int irr, vector; |
343 | struct irq_desc *desc; |
344 | struct irq_data *data; |
345 | struct irq_chip *chip; |
346 | |
347 | irq_migrate_all_off_this_cpu(); |
348 | |
349 | /* |
350 | * We can remove mdelay() and then send spurious interrupts to |
351 | * new cpu targets for all the irqs that were handled previously by |
352 | * this cpu. While it works, I have seen spurious interrupt messages |
353 | * (nothing wrong but still...). |
354 | * |
355 | * So for now, retain mdelay(1) and check the IRR and then send those |
356 | * interrupts to new targets as this cpu is already offlined... |
357 | */ |
358 | mdelay(1); |
359 | |
360 | /* |
361 | * We can walk the vector array of this cpu without holding |
362 | * vector_lock because the cpu is already marked !online, so |
363 | * nothing else will touch it. |
364 | */ |
365 | for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { |
366 | if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector]))) |
367 | continue; |
368 | |
369 | irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); |
370 | if (irr & (1 << (vector % 32))) { |
371 | desc = __this_cpu_read(vector_irq[vector]); |
372 | |
373 | raw_spin_lock(&desc->lock); |
374 | data = irq_desc_get_irq_data(desc); |
375 | chip = irq_data_get_irq_chip(d: data); |
376 | if (chip->irq_retrigger) { |
377 | chip->irq_retrigger(data); |
378 | __this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED); |
379 | } |
380 | raw_spin_unlock(&desc->lock); |
381 | } |
382 | if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED) |
383 | __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); |
384 | } |
385 | } |
386 | #endif |
387 | |
388 | #ifdef CONFIG_X86_THERMAL_VECTOR |
389 | static void smp_thermal_vector(void) |
390 | { |
391 | if (x86_thermal_enabled()) |
392 | intel_thermal_interrupt(); |
393 | else |
394 | pr_err("CPU%d: Unexpected LVT thermal interrupt!\n" , |
395 | smp_processor_id()); |
396 | } |
397 | |
398 | DEFINE_IDTENTRY_SYSVEC(sysvec_thermal) |
399 | { |
400 | trace_thermal_apic_entry(THERMAL_APIC_VECTOR); |
401 | inc_irq_stat(irq_thermal_count); |
402 | smp_thermal_vector(); |
403 | trace_thermal_apic_exit(THERMAL_APIC_VECTOR); |
404 | apic_eoi(); |
405 | } |
406 | #endif |
407 | |