1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Cell Broadband Engine Performance Monitor |
4 | * |
5 | * (C) Copyright IBM Corporation 2001,2006 |
6 | * |
7 | * Author: |
8 | * David Erb (djerb@us.ibm.com) |
9 | * Kevin Corry (kevcorry@us.ibm.com) |
10 | */ |
11 | |
12 | #include <linux/interrupt.h> |
13 | #include <linux/irqdomain.h> |
14 | #include <linux/types.h> |
15 | #include <linux/export.h> |
16 | #include <asm/io.h> |
17 | #include <asm/irq_regs.h> |
18 | #include <asm/machdep.h> |
19 | #include <asm/pmc.h> |
20 | #include <asm/reg.h> |
21 | #include <asm/spu.h> |
22 | #include <asm/cell-regs.h> |
23 | |
24 | #include "interrupt.h" |
25 | |
26 | /* |
27 | * When writing to write-only mmio addresses, save a shadow copy. All of the |
28 | * registers are 32-bit, but stored in the upper-half of a 64-bit field in |
29 | * pmd_regs. |
30 | */ |
31 | |
32 | #define WRITE_WO_MMIO(reg, x) \ |
33 | do { \ |
34 | u32 _x = (x); \ |
35 | struct cbe_pmd_regs __iomem *pmd_regs; \ |
36 | struct cbe_pmd_shadow_regs *shadow_regs; \ |
37 | pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ |
38 | shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ |
39 | out_be64(&(pmd_regs->reg), (((u64)_x) << 32)); \ |
40 | shadow_regs->reg = _x; \ |
41 | } while (0) |
42 | |
43 | #define READ_SHADOW_REG(val, reg) \ |
44 | do { \ |
45 | struct cbe_pmd_shadow_regs *shadow_regs; \ |
46 | shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ |
47 | (val) = shadow_regs->reg; \ |
48 | } while (0) |
49 | |
50 | #define READ_MMIO_UPPER32(val, reg) \ |
51 | do { \ |
52 | struct cbe_pmd_regs __iomem *pmd_regs; \ |
53 | pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ |
54 | (val) = (u32)(in_be64(&pmd_regs->reg) >> 32); \ |
55 | } while (0) |
56 | |
57 | /* |
58 | * Physical counter registers. |
59 | * Each physical counter can act as one 32-bit counter or two 16-bit counters. |
60 | */ |
61 | |
62 | u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr) |
63 | { |
64 | u32 val_in_latch, val = 0; |
65 | |
66 | if (phys_ctr < NR_PHYS_CTRS) { |
67 | READ_SHADOW_REG(val_in_latch, counter_value_in_latch); |
68 | |
69 | /* Read the latch or the actual counter, whichever is newer. */ |
70 | if (val_in_latch & (1 << phys_ctr)) { |
71 | READ_SHADOW_REG(val, pm_ctr[phys_ctr]); |
72 | } else { |
73 | READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]); |
74 | } |
75 | } |
76 | |
77 | return val; |
78 | } |
79 | EXPORT_SYMBOL_GPL(cbe_read_phys_ctr); |
80 | |
81 | void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val) |
82 | { |
83 | struct cbe_pmd_shadow_regs *shadow_regs; |
84 | u32 pm_ctrl; |
85 | |
86 | if (phys_ctr < NR_PHYS_CTRS) { |
87 | /* Writing to a counter only writes to a hardware latch. |
88 | * The new value is not propagated to the actual counter |
89 | * until the performance monitor is enabled. |
90 | */ |
91 | WRITE_WO_MMIO(pm_ctr[phys_ctr], val); |
92 | |
93 | pm_ctrl = cbe_read_pm(cpu, pm_control); |
94 | if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) { |
95 | /* The counters are already active, so we need to |
96 | * rewrite the pm_control register to "re-enable" |
97 | * the PMU. |
98 | */ |
99 | cbe_write_pm(cpu, pm_control, pm_ctrl); |
100 | } else { |
101 | shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); |
102 | shadow_regs->counter_value_in_latch |= (1 << phys_ctr); |
103 | } |
104 | } |
105 | } |
106 | EXPORT_SYMBOL_GPL(cbe_write_phys_ctr); |
107 | |
108 | /* |
109 | * "Logical" counter registers. |
110 | * These will read/write 16-bits or 32-bits depending on the |
111 | * current size of the counter. Counters 4 - 7 are always 16-bit. |
112 | */ |
113 | |
114 | u32 cbe_read_ctr(u32 cpu, u32 ctr) |
115 | { |
116 | u32 val; |
117 | u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1); |
118 | |
119 | val = cbe_read_phys_ctr(cpu, phys_ctr); |
120 | |
121 | if (cbe_get_ctr_size(cpu, phys_ctr) == 16) |
122 | val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff); |
123 | |
124 | return val; |
125 | } |
126 | EXPORT_SYMBOL_GPL(cbe_read_ctr); |
127 | |
128 | void cbe_write_ctr(u32 cpu, u32 ctr, u32 val) |
129 | { |
130 | u32 phys_ctr; |
131 | u32 phys_val; |
132 | |
133 | phys_ctr = ctr & (NR_PHYS_CTRS - 1); |
134 | |
135 | if (cbe_get_ctr_size(cpu, phys_ctr) == 16) { |
136 | phys_val = cbe_read_phys_ctr(cpu, phys_ctr); |
137 | |
138 | if (ctr < NR_PHYS_CTRS) |
139 | val = (val << 16) | (phys_val & 0xffff); |
140 | else |
141 | val = (val & 0xffff) | (phys_val & 0xffff0000); |
142 | } |
143 | |
144 | cbe_write_phys_ctr(cpu, phys_ctr, val); |
145 | } |
146 | EXPORT_SYMBOL_GPL(cbe_write_ctr); |
147 | |
148 | /* |
149 | * Counter-control registers. |
150 | * Each "logical" counter has a corresponding control register. |
151 | */ |
152 | |
153 | u32 cbe_read_pm07_control(u32 cpu, u32 ctr) |
154 | { |
155 | u32 pm07_control = 0; |
156 | |
157 | if (ctr < NR_CTRS) |
158 | READ_SHADOW_REG(pm07_control, pm07_control[ctr]); |
159 | |
160 | return pm07_control; |
161 | } |
162 | EXPORT_SYMBOL_GPL(cbe_read_pm07_control); |
163 | |
164 | void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val) |
165 | { |
166 | if (ctr < NR_CTRS) |
167 | WRITE_WO_MMIO(pm07_control[ctr], val); |
168 | } |
169 | EXPORT_SYMBOL_GPL(cbe_write_pm07_control); |
170 | |
171 | /* |
172 | * Other PMU control registers. Most of these are write-only. |
173 | */ |
174 | |
175 | u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg) |
176 | { |
177 | u32 val = 0; |
178 | |
179 | switch (reg) { |
180 | case group_control: |
181 | READ_SHADOW_REG(val, group_control); |
182 | break; |
183 | |
184 | case debug_bus_control: |
185 | READ_SHADOW_REG(val, debug_bus_control); |
186 | break; |
187 | |
188 | case trace_address: |
189 | READ_MMIO_UPPER32(val, trace_address); |
190 | break; |
191 | |
192 | case ext_tr_timer: |
193 | READ_SHADOW_REG(val, ext_tr_timer); |
194 | break; |
195 | |
196 | case pm_status: |
197 | READ_MMIO_UPPER32(val, pm_status); |
198 | break; |
199 | |
200 | case pm_control: |
201 | READ_SHADOW_REG(val, pm_control); |
202 | break; |
203 | |
204 | case pm_interval: |
205 | READ_MMIO_UPPER32(val, pm_interval); |
206 | break; |
207 | |
208 | case pm_start_stop: |
209 | READ_SHADOW_REG(val, pm_start_stop); |
210 | break; |
211 | } |
212 | |
213 | return val; |
214 | } |
215 | EXPORT_SYMBOL_GPL(cbe_read_pm); |
216 | |
217 | void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val) |
218 | { |
219 | switch (reg) { |
220 | case group_control: |
221 | WRITE_WO_MMIO(group_control, val); |
222 | break; |
223 | |
224 | case debug_bus_control: |
225 | WRITE_WO_MMIO(debug_bus_control, val); |
226 | break; |
227 | |
228 | case trace_address: |
229 | WRITE_WO_MMIO(trace_address, val); |
230 | break; |
231 | |
232 | case ext_tr_timer: |
233 | WRITE_WO_MMIO(ext_tr_timer, val); |
234 | break; |
235 | |
236 | case pm_status: |
237 | WRITE_WO_MMIO(pm_status, val); |
238 | break; |
239 | |
240 | case pm_control: |
241 | WRITE_WO_MMIO(pm_control, val); |
242 | break; |
243 | |
244 | case pm_interval: |
245 | WRITE_WO_MMIO(pm_interval, val); |
246 | break; |
247 | |
248 | case pm_start_stop: |
249 | WRITE_WO_MMIO(pm_start_stop, val); |
250 | break; |
251 | } |
252 | } |
253 | EXPORT_SYMBOL_GPL(cbe_write_pm); |
254 | |
255 | /* |
256 | * Get/set the size of a physical counter to either 16 or 32 bits. |
257 | */ |
258 | |
259 | u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr) |
260 | { |
261 | u32 pm_ctrl, size = 0; |
262 | |
263 | if (phys_ctr < NR_PHYS_CTRS) { |
264 | pm_ctrl = cbe_read_pm(cpu, pm_control); |
265 | size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32; |
266 | } |
267 | |
268 | return size; |
269 | } |
270 | EXPORT_SYMBOL_GPL(cbe_get_ctr_size); |
271 | |
272 | void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size) |
273 | { |
274 | u32 pm_ctrl; |
275 | |
276 | if (phys_ctr < NR_PHYS_CTRS) { |
277 | pm_ctrl = cbe_read_pm(cpu, pm_control); |
278 | switch (ctr_size) { |
279 | case 16: |
280 | pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr); |
281 | break; |
282 | |
283 | case 32: |
284 | pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr); |
285 | break; |
286 | } |
287 | cbe_write_pm(cpu, pm_control, pm_ctrl); |
288 | } |
289 | } |
290 | EXPORT_SYMBOL_GPL(cbe_set_ctr_size); |
291 | |
292 | /* |
293 | * Enable/disable the entire performance monitoring unit. |
294 | * When we enable the PMU, all pending writes to counters get committed. |
295 | */ |
296 | |
297 | void cbe_enable_pm(u32 cpu) |
298 | { |
299 | struct cbe_pmd_shadow_regs *shadow_regs; |
300 | u32 pm_ctrl; |
301 | |
302 | shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); |
303 | shadow_regs->counter_value_in_latch = 0; |
304 | |
305 | pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON; |
306 | cbe_write_pm(cpu, pm_control, pm_ctrl); |
307 | } |
308 | EXPORT_SYMBOL_GPL(cbe_enable_pm); |
309 | |
310 | void cbe_disable_pm(u32 cpu) |
311 | { |
312 | u32 pm_ctrl; |
313 | pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON; |
314 | cbe_write_pm(cpu, pm_control, pm_ctrl); |
315 | } |
316 | EXPORT_SYMBOL_GPL(cbe_disable_pm); |
317 | |
318 | /* |
319 | * Reading from the trace_buffer. |
320 | * The trace buffer is two 64-bit registers. Reading from |
321 | * the second half automatically increments the trace_address. |
322 | */ |
323 | |
324 | void cbe_read_trace_buffer(u32 cpu, u64 *buf) |
325 | { |
326 | struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu); |
327 | |
328 | *buf++ = in_be64(&pmd_regs->trace_buffer_0_63); |
329 | *buf++ = in_be64(&pmd_regs->trace_buffer_64_127); |
330 | } |
331 | EXPORT_SYMBOL_GPL(cbe_read_trace_buffer); |
332 | |
333 | /* |
334 | * Enabling/disabling interrupts for the entire performance monitoring unit. |
335 | */ |
336 | |
337 | u32 cbe_get_and_clear_pm_interrupts(u32 cpu) |
338 | { |
339 | /* Reading pm_status clears the interrupt bits. */ |
340 | return cbe_read_pm(cpu, pm_status); |
341 | } |
342 | EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts); |
343 | |
344 | void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask) |
345 | { |
346 | /* Set which node and thread will handle the next interrupt. */ |
347 | iic_set_interrupt_routing(cpu, thread, priority: 0); |
348 | |
349 | /* Enable the interrupt bits in the pm_status register. */ |
350 | if (mask) |
351 | cbe_write_pm(cpu, pm_status, mask); |
352 | } |
353 | EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts); |
354 | |
355 | void cbe_disable_pm_interrupts(u32 cpu) |
356 | { |
357 | cbe_get_and_clear_pm_interrupts(cpu); |
358 | cbe_write_pm(cpu, pm_status, 0); |
359 | } |
360 | EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts); |
361 | |
362 | static irqreturn_t cbe_pm_irq(int irq, void *dev_id) |
363 | { |
364 | perf_irq(get_irq_regs()); |
365 | return IRQ_HANDLED; |
366 | } |
367 | |
368 | static int __init cbe_init_pm_irq(void) |
369 | { |
370 | unsigned int irq; |
371 | int rc, node; |
372 | |
373 | for_each_online_node(node) { |
374 | irq = irq_create_mapping(NULL, hwirq: IIC_IRQ_IOEX_PMI | |
375 | (node << IIC_IRQ_NODE_SHIFT)); |
376 | if (!irq) { |
377 | printk("ERROR: Unable to allocate irq for node %d\n" , |
378 | node); |
379 | return -EINVAL; |
380 | } |
381 | |
382 | rc = request_irq(irq, handler: cbe_pm_irq, |
383 | flags: 0, name: "cbe-pmu-0" , NULL); |
384 | if (rc) { |
385 | printk("ERROR: Request for irq on node %d failed\n" , |
386 | node); |
387 | return rc; |
388 | } |
389 | } |
390 | |
391 | return 0; |
392 | } |
393 | machine_arch_initcall(cell, cbe_init_pm_irq); |
394 | |
395 | void cbe_sync_irq(int node) |
396 | { |
397 | unsigned int irq; |
398 | |
399 | irq = irq_find_mapping(NULL, |
400 | hwirq: IIC_IRQ_IOEX_PMI |
401 | | (node << IIC_IRQ_NODE_SHIFT)); |
402 | |
403 | if (!irq) { |
404 | printk(KERN_WARNING "ERROR, unable to get existing irq %d " \ |
405 | "for node %d\n" , irq, node); |
406 | return; |
407 | } |
408 | |
409 | synchronize_irq(irq); |
410 | } |
411 | EXPORT_SYMBOL_GPL(cbe_sync_irq); |
412 | |
413 | |