1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright IBM Corp. 2004, 2011 |
4 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, |
5 | * Holger Smolinski <Holger.Smolinski@de.ibm.com>, |
6 | * Thomas Spatzier <tspat@de.ibm.com>, |
7 | * |
8 | * This file contains interrupt related functions. |
9 | */ |
10 | |
11 | #include <linux/kernel_stat.h> |
12 | #include <linux/interrupt.h> |
13 | #include <linux/seq_file.h> |
14 | #include <linux/proc_fs.h> |
15 | #include <linux/profile.h> |
16 | #include <linux/export.h> |
17 | #include <linux/kernel.h> |
18 | #include <linux/ftrace.h> |
19 | #include <linux/errno.h> |
20 | #include <linux/slab.h> |
21 | #include <linux/init.h> |
22 | #include <linux/cpu.h> |
23 | #include <linux/irq.h> |
24 | #include <linux/entry-common.h> |
25 | #include <asm/irq_regs.h> |
26 | #include <asm/cputime.h> |
27 | #include <asm/lowcore.h> |
28 | #include <asm/irq.h> |
29 | #include <asm/hw_irq.h> |
30 | #include <asm/stacktrace.h> |
31 | #include <asm/softirq_stack.h> |
32 | #include "entry.h" |
33 | |
34 | DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); |
35 | EXPORT_PER_CPU_SYMBOL_GPL(irq_stat); |
36 | |
37 | struct irq_class { |
38 | int irq; |
39 | char *name; |
40 | char *desc; |
41 | }; |
42 | |
43 | /* |
44 | * The list of "main" irq classes on s390. This is the list of interrupts |
45 | * that appear both in /proc/stat ("intr" line) and /proc/interrupts. |
46 | * Historically only external and I/O interrupts have been part of /proc/stat. |
47 | * We can't add the split external and I/O sub classes since the first field |
48 | * in the "intr" line in /proc/stat is supposed to be the sum of all other |
49 | * fields. |
50 | * Since the external and I/O interrupt fields are already sums we would end |
51 | * up with having a sum which accounts each interrupt twice. |
52 | */ |
53 | static const struct irq_class irqclass_main_desc[NR_IRQS_BASE] = { |
54 | {.irq = EXT_INTERRUPT, .name = "EXT" }, |
55 | {.irq = IO_INTERRUPT, .name = "I/O" }, |
56 | {.irq = THIN_INTERRUPT, .name = "AIO" }, |
57 | }; |
58 | |
59 | /* |
60 | * The list of split external and I/O interrupts that appear only in |
61 | * /proc/interrupts. |
62 | * In addition this list contains non external / I/O events like NMIs. |
63 | */ |
64 | static const struct irq_class irqclass_sub_desc[] = { |
65 | {.irq = IRQEXT_CLK, .name = "CLK" , .desc = "[EXT] Clock Comparator" }, |
66 | {.irq = IRQEXT_EXC, .name = "EXC" , .desc = "[EXT] External Call" }, |
67 | {.irq = IRQEXT_EMS, .name = "EMS" , .desc = "[EXT] Emergency Signal" }, |
68 | {.irq = IRQEXT_TMR, .name = "TMR" , .desc = "[EXT] CPU Timer" }, |
69 | {.irq = IRQEXT_TLA, .name = "TAL" , .desc = "[EXT] Timing Alert" }, |
70 | {.irq = IRQEXT_PFL, .name = "PFL" , .desc = "[EXT] Pseudo Page Fault" }, |
71 | {.irq = IRQEXT_DSD, .name = "DSD" , .desc = "[EXT] DASD Diag" }, |
72 | {.irq = IRQEXT_VRT, .name = "VRT" , .desc = "[EXT] Virtio" }, |
73 | {.irq = IRQEXT_SCP, .name = "SCP" , .desc = "[EXT] Service Call" }, |
74 | {.irq = IRQEXT_IUC, .name = "IUC" , .desc = "[EXT] IUCV" }, |
75 | {.irq = IRQEXT_CMS, .name = "CMS" , .desc = "[EXT] CPU-Measurement: Sampling" }, |
76 | {.irq = IRQEXT_CMC, .name = "CMC" , .desc = "[EXT] CPU-Measurement: Counter" }, |
77 | {.irq = IRQEXT_FTP, .name = "FTP" , .desc = "[EXT] HMC FTP Service" }, |
78 | {.irq = IRQIO_CIO, .name = "CIO" , .desc = "[I/O] Common I/O Layer Interrupt" }, |
79 | {.irq = IRQIO_DAS, .name = "DAS" , .desc = "[I/O] DASD" }, |
80 | {.irq = IRQIO_C15, .name = "C15" , .desc = "[I/O] 3215" }, |
81 | {.irq = IRQIO_C70, .name = "C70" , .desc = "[I/O] 3270" }, |
82 | {.irq = IRQIO_TAP, .name = "TAP" , .desc = "[I/O] Tape" }, |
83 | {.irq = IRQIO_VMR, .name = "VMR" , .desc = "[I/O] Unit Record Devices" }, |
84 | {.irq = IRQIO_LCS, .name = "LCS" , .desc = "[I/O] LCS" }, |
85 | {.irq = IRQIO_CTC, .name = "CTC" , .desc = "[I/O] CTC" }, |
86 | {.irq = IRQIO_ADM, .name = "ADM" , .desc = "[I/O] EADM Subchannel" }, |
87 | {.irq = IRQIO_CSC, .name = "CSC" , .desc = "[I/O] CHSC Subchannel" }, |
88 | {.irq = IRQIO_VIR, .name = "VIR" , .desc = "[I/O] Virtual I/O Devices" }, |
89 | {.irq = IRQIO_QAI, .name = "QAI" , .desc = "[AIO] QDIO Adapter Interrupt" }, |
90 | {.irq = IRQIO_APB, .name = "APB" , .desc = "[AIO] AP Bus" }, |
91 | {.irq = IRQIO_PCF, .name = "PCF" , .desc = "[AIO] PCI Floating Interrupt" }, |
92 | {.irq = IRQIO_PCD, .name = "PCD" , .desc = "[AIO] PCI Directed Interrupt" }, |
93 | {.irq = IRQIO_MSI, .name = "MSI" , .desc = "[AIO] MSI Interrupt" }, |
94 | {.irq = IRQIO_VAI, .name = "VAI" , .desc = "[AIO] Virtual I/O Devices AI" }, |
95 | {.irq = IRQIO_GAL, .name = "GAL" , .desc = "[AIO] GIB Alert" }, |
96 | {.irq = NMI_NMI, .name = "NMI" , .desc = "[NMI] Machine Check" }, |
97 | {.irq = CPU_RST, .name = "RST" , .desc = "[CPU] CPU Restart" }, |
98 | }; |
99 | |
100 | static void do_IRQ(struct pt_regs *regs, int irq) |
101 | { |
102 | if (tod_after_eq(S390_lowcore.int_clock, |
103 | S390_lowcore.clock_comparator)) |
104 | /* Serve timer interrupts first. */ |
105 | clock_comparator_work(); |
106 | generic_handle_irq(irq); |
107 | } |
108 | |
109 | static int on_async_stack(void) |
110 | { |
111 | unsigned long frame = current_frame_address(); |
112 | |
113 | return ((S390_lowcore.async_stack ^ frame) & ~(THREAD_SIZE - 1)) == 0; |
114 | } |
115 | |
116 | static void do_irq_async(struct pt_regs *regs, int irq) |
117 | { |
118 | if (on_async_stack()) { |
119 | do_IRQ(regs, irq); |
120 | } else { |
121 | call_on_stack(2, S390_lowcore.async_stack, void, do_IRQ, |
122 | struct pt_regs *, regs, int, irq); |
123 | } |
124 | } |
125 | |
126 | static int irq_pending(struct pt_regs *regs) |
127 | { |
128 | int cc; |
129 | |
130 | asm volatile("tpi 0\n" |
131 | "ipm %0" : "=d" (cc) : : "cc" ); |
132 | return cc >> 28; |
133 | } |
134 | |
135 | void noinstr do_io_irq(struct pt_regs *regs) |
136 | { |
137 | irqentry_state_t state = irqentry_enter(regs); |
138 | struct pt_regs *old_regs = set_irq_regs(regs); |
139 | bool from_idle; |
140 | |
141 | irq_enter_rcu(); |
142 | |
143 | if (user_mode(regs)) { |
144 | update_timer_sys(); |
145 | if (static_branch_likely(&cpu_has_bear)) |
146 | current->thread.last_break = regs->last_break; |
147 | } |
148 | |
149 | from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); |
150 | if (from_idle) |
151 | account_idle_time_irq(); |
152 | |
153 | do { |
154 | regs->tpi_info = S390_lowcore.tpi_info; |
155 | if (S390_lowcore.tpi_info.adapter_IO) |
156 | do_irq_async(regs, THIN_INTERRUPT); |
157 | else |
158 | do_irq_async(regs, IO_INTERRUPT); |
159 | } while (MACHINE_IS_LPAR && irq_pending(regs)); |
160 | |
161 | irq_exit_rcu(); |
162 | |
163 | set_irq_regs(old_regs); |
164 | irqentry_exit(regs, state); |
165 | |
166 | if (from_idle) |
167 | regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); |
168 | } |
169 | |
170 | void noinstr do_ext_irq(struct pt_regs *regs) |
171 | { |
172 | irqentry_state_t state = irqentry_enter(regs); |
173 | struct pt_regs *old_regs = set_irq_regs(regs); |
174 | bool from_idle; |
175 | |
176 | irq_enter_rcu(); |
177 | |
178 | if (user_mode(regs)) { |
179 | update_timer_sys(); |
180 | if (static_branch_likely(&cpu_has_bear)) |
181 | current->thread.last_break = regs->last_break; |
182 | } |
183 | |
184 | regs->int_code = S390_lowcore.ext_int_code_addr; |
185 | regs->int_parm = S390_lowcore.ext_params; |
186 | regs->int_parm_long = S390_lowcore.ext_params2; |
187 | |
188 | from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); |
189 | if (from_idle) |
190 | account_idle_time_irq(); |
191 | |
192 | do_irq_async(regs, EXT_INTERRUPT); |
193 | |
194 | irq_exit_rcu(); |
195 | set_irq_regs(old_regs); |
196 | irqentry_exit(regs, state); |
197 | |
198 | if (from_idle) |
199 | regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); |
200 | } |
201 | |
202 | static void show_msi_interrupt(struct seq_file *p, int irq) |
203 | { |
204 | struct irq_desc *desc; |
205 | unsigned long flags; |
206 | int cpu; |
207 | |
208 | rcu_read_lock(); |
209 | desc = irq_to_desc(irq); |
210 | if (!desc) |
211 | goto out; |
212 | |
213 | raw_spin_lock_irqsave(&desc->lock, flags); |
214 | seq_printf(m: p, fmt: "%3d: " , irq); |
215 | for_each_online_cpu(cpu) |
216 | seq_printf(m: p, fmt: "%10u " , irq_desc_kstat_cpu(desc, cpu)); |
217 | |
218 | if (desc->irq_data.chip) |
219 | seq_printf(m: p, fmt: " %8s" , desc->irq_data.chip->name); |
220 | |
221 | if (desc->action) |
222 | seq_printf(m: p, fmt: " %s" , desc->action->name); |
223 | |
224 | seq_putc(m: p, c: '\n'); |
225 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
226 | out: |
227 | rcu_read_unlock(); |
228 | } |
229 | |
230 | /* |
231 | * show_interrupts is needed by /proc/interrupts. |
232 | */ |
233 | int show_interrupts(struct seq_file *p, void *v) |
234 | { |
235 | int index = *(loff_t *) v; |
236 | int cpu, irq; |
237 | |
238 | cpus_read_lock(); |
239 | if (index == 0) { |
240 | seq_puts(m: p, s: " " ); |
241 | for_each_online_cpu(cpu) |
242 | seq_printf(m: p, fmt: "CPU%-8d" , cpu); |
243 | seq_putc(m: p, c: '\n'); |
244 | } |
245 | if (index < NR_IRQS_BASE) { |
246 | seq_printf(m: p, fmt: "%s: " , irqclass_main_desc[index].name); |
247 | irq = irqclass_main_desc[index].irq; |
248 | for_each_online_cpu(cpu) |
249 | seq_printf(m: p, fmt: "%10u " , kstat_irqs_cpu(irq, cpu)); |
250 | seq_putc(m: p, c: '\n'); |
251 | goto out; |
252 | } |
253 | if (index < nr_irqs) { |
254 | show_msi_interrupt(p, irq: index); |
255 | goto out; |
256 | } |
257 | for (index = 0; index < NR_ARCH_IRQS; index++) { |
258 | seq_printf(p, "%s: " , irqclass_sub_desc[index].name); |
259 | irq = irqclass_sub_desc[index].irq; |
260 | for_each_online_cpu(cpu) |
261 | seq_printf(p, "%10u " , |
262 | per_cpu(irq_stat, cpu).irqs[irq]); |
263 | if (irqclass_sub_desc[index].desc) |
264 | seq_printf(p, " %s" , irqclass_sub_desc[index].desc); |
265 | seq_putc(p, '\n'); |
266 | } |
267 | out: |
268 | cpus_read_unlock(); |
269 | return 0; |
270 | } |
271 | |
272 | unsigned int arch_dynirq_lower_bound(unsigned int from) |
273 | { |
274 | return from < NR_IRQS_BASE ? NR_IRQS_BASE : from; |
275 | } |
276 | |
277 | /* |
278 | * ext_int_hash[index] is the list head for all external interrupts that hash |
279 | * to this index. |
280 | */ |
281 | static struct hlist_head ext_int_hash[32] ____cacheline_aligned; |
282 | |
283 | struct ext_int_info { |
284 | ext_int_handler_t handler; |
285 | struct hlist_node entry; |
286 | struct rcu_head rcu; |
287 | u16 code; |
288 | }; |
289 | |
290 | /* ext_int_hash_lock protects the handler lists for external interrupts */ |
291 | static DEFINE_SPINLOCK(ext_int_hash_lock); |
292 | |
293 | static inline int ext_hash(u16 code) |
294 | { |
295 | BUILD_BUG_ON(!is_power_of_2(ARRAY_SIZE(ext_int_hash))); |
296 | |
297 | return (code + (code >> 9)) & (ARRAY_SIZE(ext_int_hash) - 1); |
298 | } |
299 | |
300 | int register_external_irq(u16 code, ext_int_handler_t handler) |
301 | { |
302 | struct ext_int_info *p; |
303 | unsigned long flags; |
304 | int index; |
305 | |
306 | p = kmalloc(size: sizeof(*p), GFP_ATOMIC); |
307 | if (!p) |
308 | return -ENOMEM; |
309 | p->code = code; |
310 | p->handler = handler; |
311 | index = ext_hash(code); |
312 | |
313 | spin_lock_irqsave(&ext_int_hash_lock, flags); |
314 | hlist_add_head_rcu(n: &p->entry, h: &ext_int_hash[index]); |
315 | spin_unlock_irqrestore(lock: &ext_int_hash_lock, flags); |
316 | return 0; |
317 | } |
318 | EXPORT_SYMBOL(register_external_irq); |
319 | |
320 | int unregister_external_irq(u16 code, ext_int_handler_t handler) |
321 | { |
322 | struct ext_int_info *p; |
323 | unsigned long flags; |
324 | int index = ext_hash(code); |
325 | |
326 | spin_lock_irqsave(&ext_int_hash_lock, flags); |
327 | hlist_for_each_entry_rcu(p, &ext_int_hash[index], entry) { |
328 | if (p->code == code && p->handler == handler) { |
329 | hlist_del_rcu(n: &p->entry); |
330 | kfree_rcu(p, rcu); |
331 | } |
332 | } |
333 | spin_unlock_irqrestore(lock: &ext_int_hash_lock, flags); |
334 | return 0; |
335 | } |
336 | EXPORT_SYMBOL(unregister_external_irq); |
337 | |
338 | static irqreturn_t do_ext_interrupt(int irq, void *dummy) |
339 | { |
340 | struct pt_regs *regs = get_irq_regs(); |
341 | struct ext_code ext_code; |
342 | struct ext_int_info *p; |
343 | int index; |
344 | |
345 | ext_code.int_code = regs->int_code; |
346 | if (ext_code.code != EXT_IRQ_CLK_COMP) |
347 | set_cpu_flag(CIF_NOHZ_DELAY); |
348 | |
349 | index = ext_hash(code: ext_code.code); |
350 | rcu_read_lock(); |
351 | hlist_for_each_entry_rcu(p, &ext_int_hash[index], entry) { |
352 | if (unlikely(p->code != ext_code.code)) |
353 | continue; |
354 | p->handler(ext_code, regs->int_parm, regs->int_parm_long); |
355 | } |
356 | rcu_read_unlock(); |
357 | return IRQ_HANDLED; |
358 | } |
359 | |
360 | static void __init init_ext_interrupts(void) |
361 | { |
362 | int idx; |
363 | |
364 | for (idx = 0; idx < ARRAY_SIZE(ext_int_hash); idx++) |
365 | INIT_HLIST_HEAD(&ext_int_hash[idx]); |
366 | |
367 | irq_set_chip_and_handler(EXT_INTERRUPT, |
368 | &dummy_irq_chip, handle_percpu_irq); |
369 | if (request_irq(EXT_INTERRUPT, do_ext_interrupt, 0, "EXT" , NULL)) |
370 | panic(fmt: "Failed to register EXT interrupt\n" ); |
371 | } |
372 | |
373 | void __init init_IRQ(void) |
374 | { |
375 | BUILD_BUG_ON(ARRAY_SIZE(irqclass_sub_desc) != NR_ARCH_IRQS); |
376 | init_cio_interrupts(); |
377 | init_airq_interrupts(); |
378 | init_ext_interrupts(); |
379 | } |
380 | |
381 | static DEFINE_SPINLOCK(irq_subclass_lock); |
382 | static unsigned char irq_subclass_refcount[64]; |
383 | |
384 | void irq_subclass_register(enum irq_subclass subclass) |
385 | { |
386 | spin_lock(lock: &irq_subclass_lock); |
387 | if (!irq_subclass_refcount[subclass]) |
388 | system_ctl_set_bit(0, subclass); |
389 | irq_subclass_refcount[subclass]++; |
390 | spin_unlock(lock: &irq_subclass_lock); |
391 | } |
392 | EXPORT_SYMBOL(irq_subclass_register); |
393 | |
394 | void irq_subclass_unregister(enum irq_subclass subclass) |
395 | { |
396 | spin_lock(lock: &irq_subclass_lock); |
397 | irq_subclass_refcount[subclass]--; |
398 | if (!irq_subclass_refcount[subclass]) |
399 | system_ctl_clear_bit(0, subclass); |
400 | spin_unlock(lock: &irq_subclass_lock); |
401 | } |
402 | EXPORT_SYMBOL(irq_subclass_unregister); |
403 | |