1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* pcr.c: Generic sparc64 performance counter infrastructure. |
3 | * |
4 | * Copyright (C) 2009 David S. Miller (davem@davemloft.net) |
5 | */ |
6 | #include <linux/kernel.h> |
7 | #include <linux/export.h> |
8 | #include <linux/init.h> |
9 | #include <linux/irq.h> |
10 | |
11 | #include <linux/irq_work.h> |
12 | #include <linux/ftrace.h> |
13 | |
14 | #include <asm/pil.h> |
15 | #include <asm/pcr.h> |
16 | #include <asm/nmi.h> |
17 | #include <asm/asi.h> |
18 | #include <asm/spitfire.h> |
19 | |
20 | /* This code is shared between various users of the performance |
21 | * counters. Users will be oprofile, pseudo-NMI watchdog, and the |
22 | * perf_event support layer. |
23 | */ |
24 | |
25 | /* Performance counter interrupts run unmasked at PIL level 15. |
26 | * Therefore we can't do things like wakeups and other work |
27 | * that expects IRQ disabling to be adhered to in locking etc. |
28 | * |
29 | * Therefore in such situations we defer the work by signalling |
30 | * a lower level cpu IRQ. |
31 | */ |
32 | void __irq_entry deferred_pcr_work_irq(int irq, struct pt_regs *regs) |
33 | { |
34 | struct pt_regs *old_regs; |
35 | |
36 | clear_softint(1 << PIL_DEFERRED_PCR_WORK); |
37 | |
38 | old_regs = set_irq_regs(regs); |
39 | irq_enter(); |
40 | #ifdef CONFIG_IRQ_WORK |
41 | irq_work_run(); |
42 | #endif |
43 | irq_exit(); |
44 | set_irq_regs(old_regs); |
45 | } |
46 | |
47 | void arch_irq_work_raise(void) |
48 | { |
49 | set_softint(1 << PIL_DEFERRED_PCR_WORK); |
50 | } |
51 | |
52 | const struct pcr_ops *pcr_ops; |
53 | EXPORT_SYMBOL_GPL(pcr_ops); |
54 | |
55 | static u64 direct_pcr_read(unsigned long reg_num) |
56 | { |
57 | u64 val; |
58 | |
59 | WARN_ON_ONCE(reg_num != 0); |
60 | __asm__ __volatile__("rd %%pcr, %0" : "=r" (val)); |
61 | return val; |
62 | } |
63 | |
64 | static void direct_pcr_write(unsigned long reg_num, u64 val) |
65 | { |
66 | WARN_ON_ONCE(reg_num != 0); |
67 | __asm__ __volatile__("wr %0, 0x0, %%pcr" : : "r" (val)); |
68 | } |
69 | |
70 | static u64 direct_pic_read(unsigned long reg_num) |
71 | { |
72 | u64 val; |
73 | |
74 | WARN_ON_ONCE(reg_num != 0); |
75 | __asm__ __volatile__("rd %%pic, %0" : "=r" (val)); |
76 | return val; |
77 | } |
78 | |
79 | static void direct_pic_write(unsigned long reg_num, u64 val) |
80 | { |
81 | WARN_ON_ONCE(reg_num != 0); |
82 | |
83 | /* Blackbird errata workaround. See commentary in |
84 | * arch/sparc64/kernel/smp.c:smp_percpu_timer_interrupt() |
85 | * for more information. |
86 | */ |
87 | __asm__ __volatile__("ba,pt %%xcc, 99f\n\t" |
88 | " nop\n\t" |
89 | ".align 64\n" |
90 | "99:wr %0, 0x0, %%pic\n\t" |
91 | "rd %%pic, %%g0" : : "r" (val)); |
92 | } |
93 | |
94 | static u64 direct_picl_value(unsigned int nmi_hz) |
95 | { |
96 | u32 delta = local_cpu_data().clock_tick / nmi_hz; |
97 | |
98 | return ((u64)((0 - delta) & 0xffffffff)) << 32; |
99 | } |
100 | |
101 | static const struct pcr_ops direct_pcr_ops = { |
102 | .read_pcr = direct_pcr_read, |
103 | .write_pcr = direct_pcr_write, |
104 | .read_pic = direct_pic_read, |
105 | .write_pic = direct_pic_write, |
106 | .nmi_picl_value = direct_picl_value, |
107 | .pcr_nmi_enable = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE), |
108 | .pcr_nmi_disable = PCR_PIC_PRIV, |
109 | }; |
110 | |
111 | static void n2_pcr_write(unsigned long reg_num, u64 val) |
112 | { |
113 | unsigned long ret; |
114 | |
115 | WARN_ON_ONCE(reg_num != 0); |
116 | if (val & PCR_N2_HTRACE) { |
117 | ret = sun4v_niagara2_setperf(HV_N2_PERF_SPARC_CTL, val); |
118 | if (ret != HV_EOK) |
119 | direct_pcr_write(reg_num, val); |
120 | } else |
121 | direct_pcr_write(reg_num, val); |
122 | } |
123 | |
124 | static u64 n2_picl_value(unsigned int nmi_hz) |
125 | { |
126 | u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2); |
127 | |
128 | return ((u64)((0 - delta) & 0xffffffff)) << 32; |
129 | } |
130 | |
131 | static const struct pcr_ops n2_pcr_ops = { |
132 | .read_pcr = direct_pcr_read, |
133 | .write_pcr = n2_pcr_write, |
134 | .read_pic = direct_pic_read, |
135 | .write_pic = direct_pic_write, |
136 | .nmi_picl_value = n2_picl_value, |
137 | .pcr_nmi_enable = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE | |
138 | PCR_N2_TOE_OV1 | |
139 | (2 << PCR_N2_SL1_SHIFT) | |
140 | (0xff << PCR_N2_MASK1_SHIFT)), |
141 | .pcr_nmi_disable = PCR_PIC_PRIV, |
142 | }; |
143 | |
144 | static u64 n4_pcr_read(unsigned long reg_num) |
145 | { |
146 | unsigned long val; |
147 | |
148 | (void) sun4v_vt_get_perfreg(reg_num, &val); |
149 | |
150 | return val; |
151 | } |
152 | |
153 | static void n4_pcr_write(unsigned long reg_num, u64 val) |
154 | { |
155 | (void) sun4v_vt_set_perfreg(reg_num, val); |
156 | } |
157 | |
158 | static u64 n4_pic_read(unsigned long reg_num) |
159 | { |
160 | unsigned long val; |
161 | |
162 | __asm__ __volatile__("ldxa [%1] %2, %0" |
163 | : "=r" (val) |
164 | : "r" (reg_num * 0x8UL), "i" (ASI_PIC)); |
165 | |
166 | return val; |
167 | } |
168 | |
169 | static void n4_pic_write(unsigned long reg_num, u64 val) |
170 | { |
171 | __asm__ __volatile__("stxa %0, [%1] %2" |
172 | : /* no outputs */ |
173 | : "r" (val), "r" (reg_num * 0x8UL), "i" (ASI_PIC)); |
174 | } |
175 | |
176 | static u64 n4_picl_value(unsigned int nmi_hz) |
177 | { |
178 | u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2); |
179 | |
180 | return ((u64)((0 - delta) & 0xffffffff)); |
181 | } |
182 | |
183 | static const struct pcr_ops n4_pcr_ops = { |
184 | .read_pcr = n4_pcr_read, |
185 | .write_pcr = n4_pcr_write, |
186 | .read_pic = n4_pic_read, |
187 | .write_pic = n4_pic_write, |
188 | .nmi_picl_value = n4_picl_value, |
189 | .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | |
190 | PCR_N4_UTRACE | PCR_N4_TOE | |
191 | (26 << PCR_N4_SL_SHIFT)), |
192 | .pcr_nmi_disable = PCR_N4_PICNPT, |
193 | }; |
194 | |
195 | static u64 n5_pcr_read(unsigned long reg_num) |
196 | { |
197 | unsigned long val; |
198 | |
199 | (void) sun4v_t5_get_perfreg(reg_num, &val); |
200 | |
201 | return val; |
202 | } |
203 | |
204 | static void n5_pcr_write(unsigned long reg_num, u64 val) |
205 | { |
206 | (void) sun4v_t5_set_perfreg(reg_num, val); |
207 | } |
208 | |
209 | static const struct pcr_ops n5_pcr_ops = { |
210 | .read_pcr = n5_pcr_read, |
211 | .write_pcr = n5_pcr_write, |
212 | .read_pic = n4_pic_read, |
213 | .write_pic = n4_pic_write, |
214 | .nmi_picl_value = n4_picl_value, |
215 | .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | |
216 | PCR_N4_UTRACE | PCR_N4_TOE | |
217 | (26 << PCR_N4_SL_SHIFT)), |
218 | .pcr_nmi_disable = PCR_N4_PICNPT, |
219 | }; |
220 | |
221 | static u64 m7_pcr_read(unsigned long reg_num) |
222 | { |
223 | unsigned long val; |
224 | |
225 | (void) sun4v_m7_get_perfreg(reg_num, &val); |
226 | |
227 | return val; |
228 | } |
229 | |
230 | static void m7_pcr_write(unsigned long reg_num, u64 val) |
231 | { |
232 | (void) sun4v_m7_set_perfreg(reg_num, val); |
233 | } |
234 | |
235 | static const struct pcr_ops m7_pcr_ops = { |
236 | .read_pcr = m7_pcr_read, |
237 | .write_pcr = m7_pcr_write, |
238 | .read_pic = n4_pic_read, |
239 | .write_pic = n4_pic_write, |
240 | .nmi_picl_value = n4_picl_value, |
241 | .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | |
242 | PCR_N4_UTRACE | PCR_N4_TOE | |
243 | (26 << PCR_N4_SL_SHIFT)), |
244 | .pcr_nmi_disable = PCR_N4_PICNPT, |
245 | }; |
246 | |
247 | static unsigned long perf_hsvc_group; |
248 | static unsigned long perf_hsvc_major; |
249 | static unsigned long perf_hsvc_minor; |
250 | |
251 | static int __init register_perf_hsvc(void) |
252 | { |
253 | unsigned long hverror; |
254 | |
255 | if (tlb_type == hypervisor) { |
256 | switch (sun4v_chip_type) { |
257 | case SUN4V_CHIP_NIAGARA1: |
258 | perf_hsvc_group = HV_GRP_NIAG_PERF; |
259 | break; |
260 | |
261 | case SUN4V_CHIP_NIAGARA2: |
262 | perf_hsvc_group = HV_GRP_N2_CPU; |
263 | break; |
264 | |
265 | case SUN4V_CHIP_NIAGARA3: |
266 | perf_hsvc_group = HV_GRP_KT_CPU; |
267 | break; |
268 | |
269 | case SUN4V_CHIP_NIAGARA4: |
270 | perf_hsvc_group = HV_GRP_VT_CPU; |
271 | break; |
272 | |
273 | case SUN4V_CHIP_NIAGARA5: |
274 | perf_hsvc_group = HV_GRP_T5_CPU; |
275 | break; |
276 | |
277 | case SUN4V_CHIP_SPARC_M7: |
278 | perf_hsvc_group = HV_GRP_M7_PERF; |
279 | break; |
280 | |
281 | default: |
282 | return -ENODEV; |
283 | } |
284 | |
285 | |
286 | perf_hsvc_major = 1; |
287 | perf_hsvc_minor = 0; |
288 | hverror = sun4v_hvapi_register(perf_hsvc_group, |
289 | perf_hsvc_major, |
290 | &perf_hsvc_minor); |
291 | if (hverror) { |
292 | pr_err("perfmon: Could not register hvapi(0x%lx).\n" , |
293 | hverror); |
294 | return -ENODEV; |
295 | } |
296 | } |
297 | return 0; |
298 | } |
299 | |
300 | static void __init unregister_perf_hsvc(void) |
301 | { |
302 | if (tlb_type != hypervisor) |
303 | return; |
304 | sun4v_hvapi_unregister(perf_hsvc_group); |
305 | } |
306 | |
307 | static int __init setup_sun4v_pcr_ops(void) |
308 | { |
309 | int ret = 0; |
310 | |
311 | switch (sun4v_chip_type) { |
312 | case SUN4V_CHIP_NIAGARA1: |
313 | case SUN4V_CHIP_NIAGARA2: |
314 | case SUN4V_CHIP_NIAGARA3: |
315 | pcr_ops = &n2_pcr_ops; |
316 | break; |
317 | |
318 | case SUN4V_CHIP_NIAGARA4: |
319 | pcr_ops = &n4_pcr_ops; |
320 | break; |
321 | |
322 | case SUN4V_CHIP_NIAGARA5: |
323 | pcr_ops = &n5_pcr_ops; |
324 | break; |
325 | |
326 | case SUN4V_CHIP_SPARC_M7: |
327 | pcr_ops = &m7_pcr_ops; |
328 | break; |
329 | |
330 | default: |
331 | ret = -ENODEV; |
332 | break; |
333 | } |
334 | |
335 | return ret; |
336 | } |
337 | |
338 | int __init pcr_arch_init(void) |
339 | { |
340 | int err = register_perf_hsvc(); |
341 | |
342 | if (err) |
343 | return err; |
344 | |
345 | switch (tlb_type) { |
346 | case hypervisor: |
347 | err = setup_sun4v_pcr_ops(); |
348 | if (err) |
349 | goto out_unregister; |
350 | break; |
351 | |
352 | case cheetah: |
353 | case cheetah_plus: |
354 | pcr_ops = &direct_pcr_ops; |
355 | break; |
356 | |
357 | case spitfire: |
358 | /* UltraSPARC-I/II and derivatives lack a profile |
359 | * counter overflow interrupt so we can't make use of |
360 | * their hardware currently. |
361 | */ |
362 | fallthrough; |
363 | default: |
364 | err = -ENODEV; |
365 | goto out_unregister; |
366 | } |
367 | |
368 | return nmi_init(); |
369 | |
370 | out_unregister: |
371 | unregister_perf_hsvc(); |
372 | return err; |
373 | } |
374 | |