1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/perf_event.h> |
3 | #include <linux/jump_label.h> |
4 | #include <linux/export.h> |
5 | #include <linux/types.h> |
6 | #include <linux/init.h> |
7 | #include <linux/slab.h> |
8 | #include <linux/delay.h> |
9 | #include <linux/jiffies.h> |
10 | #include <asm/apicdef.h> |
11 | #include <asm/apic.h> |
12 | #include <asm/nmi.h> |
13 | |
14 | #include "../perf_event.h" |
15 | |
16 | static DEFINE_PER_CPU(unsigned long, perf_nmi_tstamp); |
17 | static unsigned long perf_nmi_window; |
18 | |
19 | /* AMD Event 0xFFF: Merge. Used with Large Increment per Cycle events */ |
20 | #define AMD_MERGE_EVENT ((0xFULL << 32) | 0xFFULL) |
21 | #define AMD_MERGE_EVENT_ENABLE (AMD_MERGE_EVENT | ARCH_PERFMON_EVENTSEL_ENABLE) |
22 | |
23 | /* PMC Enable and Overflow bits for PerfCntrGlobal* registers */ |
24 | static u64 amd_pmu_global_cntr_mask __read_mostly; |
25 | |
26 | static __initconst const u64 amd_hw_cache_event_ids |
27 | [PERF_COUNT_HW_CACHE_MAX] |
28 | [PERF_COUNT_HW_CACHE_OP_MAX] |
29 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = |
30 | { |
31 | [ C(L1D) ] = { |
32 | [ C(OP_READ) ] = { |
33 | [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */ |
34 | [ C(RESULT_MISS) ] = 0x0141, /* Data Cache Misses */ |
35 | }, |
36 | [ C(OP_WRITE) ] = { |
37 | [ C(RESULT_ACCESS) ] = 0, |
38 | [ C(RESULT_MISS) ] = 0, |
39 | }, |
40 | [ C(OP_PREFETCH) ] = { |
41 | [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */ |
42 | [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */ |
43 | }, |
44 | }, |
45 | [ C(L1I ) ] = { |
46 | [ C(OP_READ) ] = { |
47 | [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */ |
48 | [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */ |
49 | }, |
50 | [ C(OP_WRITE) ] = { |
51 | [ C(RESULT_ACCESS) ] = -1, |
52 | [ C(RESULT_MISS) ] = -1, |
53 | }, |
54 | [ C(OP_PREFETCH) ] = { |
55 | [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */ |
56 | [ C(RESULT_MISS) ] = 0, |
57 | }, |
58 | }, |
59 | [ C(LL ) ] = { |
60 | [ C(OP_READ) ] = { |
61 | [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */ |
62 | [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */ |
63 | }, |
64 | [ C(OP_WRITE) ] = { |
65 | [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */ |
66 | [ C(RESULT_MISS) ] = 0, |
67 | }, |
68 | [ C(OP_PREFETCH) ] = { |
69 | [ C(RESULT_ACCESS) ] = 0, |
70 | [ C(RESULT_MISS) ] = 0, |
71 | }, |
72 | }, |
73 | [ C(DTLB) ] = { |
74 | [ C(OP_READ) ] = { |
75 | [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */ |
76 | [ C(RESULT_MISS) ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */ |
77 | }, |
78 | [ C(OP_WRITE) ] = { |
79 | [ C(RESULT_ACCESS) ] = 0, |
80 | [ C(RESULT_MISS) ] = 0, |
81 | }, |
82 | [ C(OP_PREFETCH) ] = { |
83 | [ C(RESULT_ACCESS) ] = 0, |
84 | [ C(RESULT_MISS) ] = 0, |
85 | }, |
86 | }, |
87 | [ C(ITLB) ] = { |
88 | [ C(OP_READ) ] = { |
89 | [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */ |
90 | [ C(RESULT_MISS) ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */ |
91 | }, |
92 | [ C(OP_WRITE) ] = { |
93 | [ C(RESULT_ACCESS) ] = -1, |
94 | [ C(RESULT_MISS) ] = -1, |
95 | }, |
96 | [ C(OP_PREFETCH) ] = { |
97 | [ C(RESULT_ACCESS) ] = -1, |
98 | [ C(RESULT_MISS) ] = -1, |
99 | }, |
100 | }, |
101 | [ C(BPU ) ] = { |
102 | [ C(OP_READ) ] = { |
103 | [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */ |
104 | [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */ |
105 | }, |
106 | [ C(OP_WRITE) ] = { |
107 | [ C(RESULT_ACCESS) ] = -1, |
108 | [ C(RESULT_MISS) ] = -1, |
109 | }, |
110 | [ C(OP_PREFETCH) ] = { |
111 | [ C(RESULT_ACCESS) ] = -1, |
112 | [ C(RESULT_MISS) ] = -1, |
113 | }, |
114 | }, |
115 | [ C(NODE) ] = { |
116 | [ C(OP_READ) ] = { |
117 | [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */ |
118 | [ C(RESULT_MISS) ] = 0x98e9, /* CPU Request to Memory, r */ |
119 | }, |
120 | [ C(OP_WRITE) ] = { |
121 | [ C(RESULT_ACCESS) ] = -1, |
122 | [ C(RESULT_MISS) ] = -1, |
123 | }, |
124 | [ C(OP_PREFETCH) ] = { |
125 | [ C(RESULT_ACCESS) ] = -1, |
126 | [ C(RESULT_MISS) ] = -1, |
127 | }, |
128 | }, |
129 | }; |
130 | |
131 | static __initconst const u64 amd_hw_cache_event_ids_f17h |
132 | [PERF_COUNT_HW_CACHE_MAX] |
133 | [PERF_COUNT_HW_CACHE_OP_MAX] |
134 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
135 | [C(L1D)] = { |
136 | [C(OP_READ)] = { |
137 | [C(RESULT_ACCESS)] = 0x0040, /* Data Cache Accesses */ |
138 | [C(RESULT_MISS)] = 0xc860, /* L2$ access from DC Miss */ |
139 | }, |
140 | [C(OP_WRITE)] = { |
141 | [C(RESULT_ACCESS)] = 0, |
142 | [C(RESULT_MISS)] = 0, |
143 | }, |
144 | [C(OP_PREFETCH)] = { |
145 | [C(RESULT_ACCESS)] = 0xff5a, /* h/w prefetch DC Fills */ |
146 | [C(RESULT_MISS)] = 0, |
147 | }, |
148 | }, |
149 | [C(L1I)] = { |
150 | [C(OP_READ)] = { |
151 | [C(RESULT_ACCESS)] = 0x0080, /* Instruction cache fetches */ |
152 | [C(RESULT_MISS)] = 0x0081, /* Instruction cache misses */ |
153 | }, |
154 | [C(OP_WRITE)] = { |
155 | [C(RESULT_ACCESS)] = -1, |
156 | [C(RESULT_MISS)] = -1, |
157 | }, |
158 | [C(OP_PREFETCH)] = { |
159 | [C(RESULT_ACCESS)] = 0, |
160 | [C(RESULT_MISS)] = 0, |
161 | }, |
162 | }, |
163 | [C(LL)] = { |
164 | [C(OP_READ)] = { |
165 | [C(RESULT_ACCESS)] = 0, |
166 | [C(RESULT_MISS)] = 0, |
167 | }, |
168 | [C(OP_WRITE)] = { |
169 | [C(RESULT_ACCESS)] = 0, |
170 | [C(RESULT_MISS)] = 0, |
171 | }, |
172 | [C(OP_PREFETCH)] = { |
173 | [C(RESULT_ACCESS)] = 0, |
174 | [C(RESULT_MISS)] = 0, |
175 | }, |
176 | }, |
177 | [C(DTLB)] = { |
178 | [C(OP_READ)] = { |
179 | [C(RESULT_ACCESS)] = 0xff45, /* All L2 DTLB accesses */ |
180 | [C(RESULT_MISS)] = 0xf045, /* L2 DTLB misses (PT walks) */ |
181 | }, |
182 | [C(OP_WRITE)] = { |
183 | [C(RESULT_ACCESS)] = 0, |
184 | [C(RESULT_MISS)] = 0, |
185 | }, |
186 | [C(OP_PREFETCH)] = { |
187 | [C(RESULT_ACCESS)] = 0, |
188 | [C(RESULT_MISS)] = 0, |
189 | }, |
190 | }, |
191 | [C(ITLB)] = { |
192 | [C(OP_READ)] = { |
193 | [C(RESULT_ACCESS)] = 0x0084, /* L1 ITLB misses, L2 ITLB hits */ |
194 | [C(RESULT_MISS)] = 0xff85, /* L1 ITLB misses, L2 misses */ |
195 | }, |
196 | [C(OP_WRITE)] = { |
197 | [C(RESULT_ACCESS)] = -1, |
198 | [C(RESULT_MISS)] = -1, |
199 | }, |
200 | [C(OP_PREFETCH)] = { |
201 | [C(RESULT_ACCESS)] = -1, |
202 | [C(RESULT_MISS)] = -1, |
203 | }, |
204 | }, |
205 | [C(BPU)] = { |
206 | [C(OP_READ)] = { |
207 | [C(RESULT_ACCESS)] = 0x00c2, /* Retired Branch Instr. */ |
208 | [C(RESULT_MISS)] = 0x00c3, /* Retired Mispredicted BI */ |
209 | }, |
210 | [C(OP_WRITE)] = { |
211 | [C(RESULT_ACCESS)] = -1, |
212 | [C(RESULT_MISS)] = -1, |
213 | }, |
214 | [C(OP_PREFETCH)] = { |
215 | [C(RESULT_ACCESS)] = -1, |
216 | [C(RESULT_MISS)] = -1, |
217 | }, |
218 | }, |
219 | [C(NODE)] = { |
220 | [C(OP_READ)] = { |
221 | [C(RESULT_ACCESS)] = 0, |
222 | [C(RESULT_MISS)] = 0, |
223 | }, |
224 | [C(OP_WRITE)] = { |
225 | [C(RESULT_ACCESS)] = -1, |
226 | [C(RESULT_MISS)] = -1, |
227 | }, |
228 | [C(OP_PREFETCH)] = { |
229 | [C(RESULT_ACCESS)] = -1, |
230 | [C(RESULT_MISS)] = -1, |
231 | }, |
232 | }, |
233 | }; |
234 | |
235 | /* |
236 | * AMD Performance Monitor K7 and later, up to and including Family 16h: |
237 | */ |
238 | static const u64 amd_perfmon_event_map[PERF_COUNT_HW_MAX] = |
239 | { |
240 | [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, |
241 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, |
242 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0x077d, |
243 | [PERF_COUNT_HW_CACHE_MISSES] = 0x077e, |
244 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, |
245 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, |
246 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */ |
247 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */ |
248 | }; |
249 | |
250 | /* |
251 | * AMD Performance Monitor Family 17h and later: |
252 | */ |
253 | static const u64 amd_zen1_perfmon_event_map[PERF_COUNT_HW_MAX] = |
254 | { |
255 | [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, |
256 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, |
257 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0xff60, |
258 | [PERF_COUNT_HW_CACHE_MISSES] = 0x0964, |
259 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, |
260 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, |
261 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x0287, |
262 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x0187, |
263 | }; |
264 | |
265 | static const u64 amd_zen2_perfmon_event_map[PERF_COUNT_HW_MAX] = |
266 | { |
267 | [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, |
268 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, |
269 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0xff60, |
270 | [PERF_COUNT_HW_CACHE_MISSES] = 0x0964, |
271 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, |
272 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, |
273 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00a9, |
274 | }; |
275 | |
276 | static const u64 amd_zen4_perfmon_event_map[PERF_COUNT_HW_MAX] = |
277 | { |
278 | [PERF_COUNT_HW_CPU_CYCLES] = 0x0076, |
279 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, |
280 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0xff60, |
281 | [PERF_COUNT_HW_CACHE_MISSES] = 0x0964, |
282 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2, |
283 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3, |
284 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00a9, |
285 | [PERF_COUNT_HW_REF_CPU_CYCLES] = 0x100000120, |
286 | }; |
287 | |
288 | static u64 amd_pmu_event_map(int hw_event) |
289 | { |
290 | if (cpu_feature_enabled(X86_FEATURE_ZEN4) || boot_cpu_data.x86 >= 0x1a) |
291 | return amd_zen4_perfmon_event_map[hw_event]; |
292 | |
293 | if (cpu_feature_enabled(X86_FEATURE_ZEN2) || boot_cpu_data.x86 >= 0x19) |
294 | return amd_zen2_perfmon_event_map[hw_event]; |
295 | |
296 | if (cpu_feature_enabled(X86_FEATURE_ZEN1)) |
297 | return amd_zen1_perfmon_event_map[hw_event]; |
298 | |
299 | return amd_perfmon_event_map[hw_event]; |
300 | } |
301 | |
302 | /* |
303 | * Previously calculated offsets |
304 | */ |
305 | static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly; |
306 | static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly; |
307 | |
308 | /* |
309 | * Legacy CPUs: |
310 | * 4 counters starting at 0xc0010000 each offset by 1 |
311 | * |
312 | * CPUs with core performance counter extensions: |
313 | * 6 counters starting at 0xc0010200 each offset by 2 |
314 | */ |
315 | static inline int amd_pmu_addr_offset(int index, bool eventsel) |
316 | { |
317 | int offset; |
318 | |
319 | if (!index) |
320 | return index; |
321 | |
322 | if (eventsel) |
323 | offset = event_offsets[index]; |
324 | else |
325 | offset = count_offsets[index]; |
326 | |
327 | if (offset) |
328 | return offset; |
329 | |
330 | if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) |
331 | offset = index; |
332 | else |
333 | offset = index << 1; |
334 | |
335 | if (eventsel) |
336 | event_offsets[index] = offset; |
337 | else |
338 | count_offsets[index] = offset; |
339 | |
340 | return offset; |
341 | } |
342 | |
343 | /* |
344 | * AMD64 events are detected based on their event codes. |
345 | */ |
346 | static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc) |
347 | { |
348 | return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff); |
349 | } |
350 | |
351 | static inline bool amd_is_pair_event_code(struct hw_perf_event *hwc) |
352 | { |
353 | if (!(x86_pmu.flags & PMU_FL_PAIR)) |
354 | return false; |
355 | |
356 | switch (amd_get_event_code(hwc)) { |
357 | case 0x003: return true; /* Retired SSE/AVX FLOPs */ |
358 | default: return false; |
359 | } |
360 | } |
361 | |
362 | DEFINE_STATIC_CALL_RET0(amd_pmu_branch_hw_config, *x86_pmu.hw_config); |
363 | |
364 | static int amd_core_hw_config(struct perf_event *event) |
365 | { |
366 | if (event->attr.exclude_host && event->attr.exclude_guest) |
367 | /* |
368 | * When HO == GO == 1 the hardware treats that as GO == HO == 0 |
369 | * and will count in both modes. We don't want to count in that |
370 | * case so we emulate no-counting by setting US = OS = 0. |
371 | */ |
372 | event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR | |
373 | ARCH_PERFMON_EVENTSEL_OS); |
374 | else if (event->attr.exclude_host) |
375 | event->hw.config |= AMD64_EVENTSEL_GUESTONLY; |
376 | else if (event->attr.exclude_guest) |
377 | event->hw.config |= AMD64_EVENTSEL_HOSTONLY; |
378 | |
379 | if ((x86_pmu.flags & PMU_FL_PAIR) && amd_is_pair_event_code(hwc: &event->hw)) |
380 | event->hw.flags |= PERF_X86_EVENT_PAIR; |
381 | |
382 | if (has_branch_stack(event)) |
383 | return static_call(amd_pmu_branch_hw_config)(event); |
384 | |
385 | return 0; |
386 | } |
387 | |
388 | static inline int amd_is_nb_event(struct hw_perf_event *hwc) |
389 | { |
390 | return (hwc->config & 0xe0) == 0xe0; |
391 | } |
392 | |
393 | static inline int amd_has_nb(struct cpu_hw_events *cpuc) |
394 | { |
395 | struct amd_nb *nb = cpuc->amd_nb; |
396 | |
397 | return nb && nb->nb_id != -1; |
398 | } |
399 | |
400 | static int amd_pmu_hw_config(struct perf_event *event) |
401 | { |
402 | int ret; |
403 | |
404 | /* pass precise event sampling to ibs: */ |
405 | if (event->attr.precise_ip && get_ibs_caps()) |
406 | return forward_event_to_ibs(event); |
407 | |
408 | if (has_branch_stack(event) && !x86_pmu.lbr_nr) |
409 | return -EOPNOTSUPP; |
410 | |
411 | ret = x86_pmu_hw_config(event); |
412 | if (ret) |
413 | return ret; |
414 | |
415 | if (event->attr.type == PERF_TYPE_RAW) |
416 | event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK; |
417 | |
418 | return amd_core_hw_config(event); |
419 | } |
420 | |
421 | static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc, |
422 | struct perf_event *event) |
423 | { |
424 | struct amd_nb *nb = cpuc->amd_nb; |
425 | int i; |
426 | |
427 | /* |
428 | * need to scan whole list because event may not have |
429 | * been assigned during scheduling |
430 | * |
431 | * no race condition possible because event can only |
432 | * be removed on one CPU at a time AND PMU is disabled |
433 | * when we come here |
434 | */ |
435 | for (i = 0; i < x86_pmu.num_counters; i++) { |
436 | if (cmpxchg(nb->owners + i, event, NULL) == event) |
437 | break; |
438 | } |
439 | } |
440 | |
441 | /* |
442 | * AMD64 NorthBridge events need special treatment because |
443 | * counter access needs to be synchronized across all cores |
444 | * of a package. Refer to BKDG section 3.12 |
445 | * |
446 | * NB events are events measuring L3 cache, Hypertransport |
447 | * traffic. They are identified by an event code >= 0xe00. |
448 | * They measure events on the NorthBride which is shared |
449 | * by all cores on a package. NB events are counted on a |
450 | * shared set of counters. When a NB event is programmed |
451 | * in a counter, the data actually comes from a shared |
452 | * counter. Thus, access to those counters needs to be |
453 | * synchronized. |
454 | * |
455 | * We implement the synchronization such that no two cores |
456 | * can be measuring NB events using the same counters. Thus, |
457 | * we maintain a per-NB allocation table. The available slot |
458 | * is propagated using the event_constraint structure. |
459 | * |
460 | * We provide only one choice for each NB event based on |
461 | * the fact that only NB events have restrictions. Consequently, |
462 | * if a counter is available, there is a guarantee the NB event |
463 | * will be assigned to it. If no slot is available, an empty |
464 | * constraint is returned and scheduling will eventually fail |
465 | * for this event. |
466 | * |
467 | * Note that all cores attached the same NB compete for the same |
468 | * counters to host NB events, this is why we use atomic ops. Some |
469 | * multi-chip CPUs may have more than one NB. |
470 | * |
471 | * Given that resources are allocated (cmpxchg), they must be |
472 | * eventually freed for others to use. This is accomplished by |
473 | * calling __amd_put_nb_event_constraints() |
474 | * |
475 | * Non NB events are not impacted by this restriction. |
476 | */ |
477 | static struct event_constraint * |
478 | __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, |
479 | struct event_constraint *c) |
480 | { |
481 | struct hw_perf_event *hwc = &event->hw; |
482 | struct amd_nb *nb = cpuc->amd_nb; |
483 | struct perf_event *old; |
484 | int idx, new = -1; |
485 | |
486 | if (!c) |
487 | c = &unconstrained; |
488 | |
489 | if (cpuc->is_fake) |
490 | return c; |
491 | |
492 | /* |
493 | * detect if already present, if so reuse |
494 | * |
495 | * cannot merge with actual allocation |
496 | * because of possible holes |
497 | * |
498 | * event can already be present yet not assigned (in hwc->idx) |
499 | * because of successive calls to x86_schedule_events() from |
500 | * hw_perf_group_sched_in() without hw_perf_enable() |
501 | */ |
502 | for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) { |
503 | if (new == -1 || hwc->idx == idx) |
504 | /* assign free slot, prefer hwc->idx */ |
505 | old = cmpxchg(nb->owners + idx, NULL, event); |
506 | else if (nb->owners[idx] == event) |
507 | /* event already present */ |
508 | old = event; |
509 | else |
510 | continue; |
511 | |
512 | if (old && old != event) |
513 | continue; |
514 | |
515 | /* reassign to this slot */ |
516 | if (new != -1) |
517 | cmpxchg(nb->owners + new, event, NULL); |
518 | new = idx; |
519 | |
520 | /* already present, reuse */ |
521 | if (old == event) |
522 | break; |
523 | } |
524 | |
525 | if (new == -1) |
526 | return &emptyconstraint; |
527 | |
528 | return &nb->event_constraints[new]; |
529 | } |
530 | |
531 | static struct amd_nb *amd_alloc_nb(int cpu) |
532 | { |
533 | struct amd_nb *nb; |
534 | int i; |
535 | |
536 | nb = kzalloc_node(size: sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu)); |
537 | if (!nb) |
538 | return NULL; |
539 | |
540 | nb->nb_id = -1; |
541 | |
542 | /* |
543 | * initialize all possible NB constraints |
544 | */ |
545 | for (i = 0; i < x86_pmu.num_counters; i++) { |
546 | __set_bit(i, nb->event_constraints[i].idxmsk); |
547 | nb->event_constraints[i].weight = 1; |
548 | } |
549 | return nb; |
550 | } |
551 | |
552 | typedef void (amd_pmu_branch_reset_t)(void); |
553 | DEFINE_STATIC_CALL_NULL(amd_pmu_branch_reset, amd_pmu_branch_reset_t); |
554 | |
555 | static void amd_pmu_cpu_reset(int cpu) |
556 | { |
557 | if (x86_pmu.lbr_nr) |
558 | static_call(amd_pmu_branch_reset)(); |
559 | |
560 | if (x86_pmu.version < 2) |
561 | return; |
562 | |
563 | /* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */ |
564 | wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, val: 0); |
565 | |
566 | /* |
567 | * Clear freeze and overflow bits i.e. PerfCntrGLobalStatus.LbrFreeze |
568 | * and PerfCntrGLobalStatus.PerfCntrOvfl |
569 | */ |
570 | wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, |
571 | GLOBAL_STATUS_LBRS_FROZEN | amd_pmu_global_cntr_mask); |
572 | } |
573 | |
574 | static int amd_pmu_cpu_prepare(int cpu) |
575 | { |
576 | struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); |
577 | |
578 | cpuc->lbr_sel = kzalloc_node(size: sizeof(struct er_account), GFP_KERNEL, |
579 | cpu_to_node(cpu)); |
580 | if (!cpuc->lbr_sel) |
581 | return -ENOMEM; |
582 | |
583 | WARN_ON_ONCE(cpuc->amd_nb); |
584 | |
585 | if (!x86_pmu.amd_nb_constraints) |
586 | return 0; |
587 | |
588 | cpuc->amd_nb = amd_alloc_nb(cpu); |
589 | if (cpuc->amd_nb) |
590 | return 0; |
591 | |
592 | kfree(objp: cpuc->lbr_sel); |
593 | cpuc->lbr_sel = NULL; |
594 | |
595 | return -ENOMEM; |
596 | } |
597 | |
598 | static void amd_pmu_cpu_starting(int cpu) |
599 | { |
600 | struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); |
601 | void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED]; |
602 | struct amd_nb *nb; |
603 | int i, nb_id; |
604 | |
605 | cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY; |
606 | amd_pmu_cpu_reset(cpu); |
607 | |
608 | if (!x86_pmu.amd_nb_constraints) |
609 | return; |
610 | |
611 | nb_id = topology_amd_node_id(cpu); |
612 | WARN_ON_ONCE(nb_id == BAD_APICID); |
613 | |
614 | for_each_online_cpu(i) { |
615 | nb = per_cpu(cpu_hw_events, i).amd_nb; |
616 | if (WARN_ON_ONCE(!nb)) |
617 | continue; |
618 | |
619 | if (nb->nb_id == nb_id) { |
620 | *onln = cpuc->amd_nb; |
621 | cpuc->amd_nb = nb; |
622 | break; |
623 | } |
624 | } |
625 | |
626 | cpuc->amd_nb->nb_id = nb_id; |
627 | cpuc->amd_nb->refcnt++; |
628 | } |
629 | |
630 | static void amd_pmu_cpu_dead(int cpu) |
631 | { |
632 | struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); |
633 | |
634 | kfree(objp: cpuhw->lbr_sel); |
635 | cpuhw->lbr_sel = NULL; |
636 | |
637 | if (!x86_pmu.amd_nb_constraints) |
638 | return; |
639 | |
640 | if (cpuhw->amd_nb) { |
641 | struct amd_nb *nb = cpuhw->amd_nb; |
642 | |
643 | if (nb->nb_id == -1 || --nb->refcnt == 0) |
644 | kfree(objp: nb); |
645 | |
646 | cpuhw->amd_nb = NULL; |
647 | } |
648 | } |
649 | |
650 | static inline void amd_pmu_set_global_ctl(u64 ctl) |
651 | { |
652 | wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, val: ctl); |
653 | } |
654 | |
655 | static inline u64 amd_pmu_get_global_status(void) |
656 | { |
657 | u64 status; |
658 | |
659 | /* PerfCntrGlobalStatus is read-only */ |
660 | rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status); |
661 | |
662 | return status; |
663 | } |
664 | |
665 | static inline void amd_pmu_ack_global_status(u64 status) |
666 | { |
667 | /* |
668 | * PerfCntrGlobalStatus is read-only but an overflow acknowledgment |
669 | * mechanism exists; writing 1 to a bit in PerfCntrGlobalStatusClr |
670 | * clears the same bit in PerfCntrGlobalStatus |
671 | */ |
672 | |
673 | wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, val: status); |
674 | } |
675 | |
676 | static bool amd_pmu_test_overflow_topbit(int idx) |
677 | { |
678 | u64 counter; |
679 | |
680 | rdmsrl(x86_pmu_event_addr(idx), counter); |
681 | |
682 | return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1)); |
683 | } |
684 | |
685 | static bool amd_pmu_test_overflow_status(int idx) |
686 | { |
687 | return amd_pmu_get_global_status() & BIT_ULL(idx); |
688 | } |
689 | |
690 | DEFINE_STATIC_CALL(amd_pmu_test_overflow, amd_pmu_test_overflow_topbit); |
691 | |
692 | /* |
693 | * When a PMC counter overflows, an NMI is used to process the event and |
694 | * reset the counter. NMI latency can result in the counter being updated |
695 | * before the NMI can run, which can result in what appear to be spurious |
696 | * NMIs. This function is intended to wait for the NMI to run and reset |
697 | * the counter to avoid possible unhandled NMI messages. |
698 | */ |
699 | #define OVERFLOW_WAIT_COUNT 50 |
700 | |
701 | static void amd_pmu_wait_on_overflow(int idx) |
702 | { |
703 | unsigned int i; |
704 | |
705 | /* |
706 | * Wait for the counter to be reset if it has overflowed. This loop |
707 | * should exit very, very quickly, but just in case, don't wait |
708 | * forever... |
709 | */ |
710 | for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) { |
711 | if (!static_call(amd_pmu_test_overflow)(idx)) |
712 | break; |
713 | |
714 | /* Might be in IRQ context, so can't sleep */ |
715 | udelay(1); |
716 | } |
717 | } |
718 | |
719 | static void amd_pmu_check_overflow(void) |
720 | { |
721 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
722 | int idx; |
723 | |
724 | /* |
725 | * This shouldn't be called from NMI context, but add a safeguard here |
726 | * to return, since if we're in NMI context we can't wait for an NMI |
727 | * to reset an overflowed counter value. |
728 | */ |
729 | if (in_nmi()) |
730 | return; |
731 | |
732 | /* |
733 | * Check each counter for overflow and wait for it to be reset by the |
734 | * NMI if it has overflowed. This relies on the fact that all active |
735 | * counters are always enabled when this function is called and |
736 | * ARCH_PERFMON_EVENTSEL_INT is always set. |
737 | */ |
738 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
739 | if (!test_bit(idx, cpuc->active_mask)) |
740 | continue; |
741 | |
742 | amd_pmu_wait_on_overflow(idx); |
743 | } |
744 | } |
745 | |
746 | static void amd_pmu_enable_event(struct perf_event *event) |
747 | { |
748 | x86_pmu_enable_event(event); |
749 | } |
750 | |
751 | static void amd_pmu_enable_all(int added) |
752 | { |
753 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
754 | int idx; |
755 | |
756 | amd_brs_enable_all(); |
757 | |
758 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
759 | /* only activate events which are marked as active */ |
760 | if (!test_bit(idx, cpuc->active_mask)) |
761 | continue; |
762 | |
763 | amd_pmu_enable_event(event: cpuc->events[idx]); |
764 | } |
765 | } |
766 | |
767 | static void amd_pmu_v2_enable_event(struct perf_event *event) |
768 | { |
769 | struct hw_perf_event *hwc = &event->hw; |
770 | |
771 | /* |
772 | * Testing cpu_hw_events.enabled should be skipped in this case unlike |
773 | * in x86_pmu_enable_event(). |
774 | * |
775 | * Since cpu_hw_events.enabled is set only after returning from |
776 | * x86_pmu_start(), the PMCs must be programmed and kept ready. |
777 | * Counting starts only after x86_pmu_enable_all() is called. |
778 | */ |
779 | __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); |
780 | } |
781 | |
782 | static __always_inline void amd_pmu_core_enable_all(void) |
783 | { |
784 | amd_pmu_set_global_ctl(ctl: amd_pmu_global_cntr_mask); |
785 | } |
786 | |
787 | static void amd_pmu_v2_enable_all(int added) |
788 | { |
789 | amd_pmu_lbr_enable_all(); |
790 | amd_pmu_core_enable_all(); |
791 | } |
792 | |
793 | static void amd_pmu_disable_event(struct perf_event *event) |
794 | { |
795 | x86_pmu_disable_event(event); |
796 | |
797 | /* |
798 | * This can be called from NMI context (via x86_pmu_stop). The counter |
799 | * may have overflowed, but either way, we'll never see it get reset |
800 | * by the NMI if we're already in the NMI. And the NMI latency support |
801 | * below will take care of any pending NMI that might have been |
802 | * generated by the overflow. |
803 | */ |
804 | if (in_nmi()) |
805 | return; |
806 | |
807 | amd_pmu_wait_on_overflow(idx: event->hw.idx); |
808 | } |
809 | |
810 | static void amd_pmu_disable_all(void) |
811 | { |
812 | amd_brs_disable_all(); |
813 | x86_pmu_disable_all(); |
814 | amd_pmu_check_overflow(); |
815 | } |
816 | |
817 | static __always_inline void amd_pmu_core_disable_all(void) |
818 | { |
819 | amd_pmu_set_global_ctl(ctl: 0); |
820 | } |
821 | |
822 | static void amd_pmu_v2_disable_all(void) |
823 | { |
824 | amd_pmu_core_disable_all(); |
825 | amd_pmu_lbr_disable_all(); |
826 | amd_pmu_check_overflow(); |
827 | } |
828 | |
829 | DEFINE_STATIC_CALL_NULL(amd_pmu_branch_add, *x86_pmu.add); |
830 | |
831 | static void amd_pmu_add_event(struct perf_event *event) |
832 | { |
833 | if (needs_branch_stack(event)) |
834 | static_call(amd_pmu_branch_add)(event); |
835 | } |
836 | |
837 | DEFINE_STATIC_CALL_NULL(amd_pmu_branch_del, *x86_pmu.del); |
838 | |
839 | static void amd_pmu_del_event(struct perf_event *event) |
840 | { |
841 | if (needs_branch_stack(event)) |
842 | static_call(amd_pmu_branch_del)(event); |
843 | } |
844 | |
845 | /* |
846 | * Because of NMI latency, if multiple PMC counters are active or other sources |
847 | * of NMIs are received, the perf NMI handler can handle one or more overflowed |
848 | * PMC counters outside of the NMI associated with the PMC overflow. If the NMI |
849 | * doesn't arrive at the LAPIC in time to become a pending NMI, then the kernel |
850 | * back-to-back NMI support won't be active. This PMC handler needs to take into |
851 | * account that this can occur, otherwise this could result in unknown NMI |
852 | * messages being issued. Examples of this is PMC overflow while in the NMI |
853 | * handler when multiple PMCs are active or PMC overflow while handling some |
854 | * other source of an NMI. |
855 | * |
856 | * Attempt to mitigate this by creating an NMI window in which un-handled NMIs |
857 | * received during this window will be claimed. This prevents extending the |
858 | * window past when it is possible that latent NMIs should be received. The |
859 | * per-CPU perf_nmi_tstamp will be set to the window end time whenever perf has |
860 | * handled a counter. When an un-handled NMI is received, it will be claimed |
861 | * only if arriving within that window. |
862 | */ |
863 | static inline int amd_pmu_adjust_nmi_window(int handled) |
864 | { |
865 | /* |
866 | * If a counter was handled, record a timestamp such that un-handled |
867 | * NMIs will be claimed if arriving within that window. |
868 | */ |
869 | if (handled) { |
870 | this_cpu_write(perf_nmi_tstamp, jiffies + perf_nmi_window); |
871 | |
872 | return handled; |
873 | } |
874 | |
875 | if (time_after(jiffies, this_cpu_read(perf_nmi_tstamp))) |
876 | return NMI_DONE; |
877 | |
878 | return NMI_HANDLED; |
879 | } |
880 | |
881 | static int amd_pmu_handle_irq(struct pt_regs *regs) |
882 | { |
883 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
884 | int handled; |
885 | int pmu_enabled; |
886 | |
887 | /* |
888 | * Save the PMU state. |
889 | * It needs to be restored when leaving the handler. |
890 | */ |
891 | pmu_enabled = cpuc->enabled; |
892 | cpuc->enabled = 0; |
893 | |
894 | amd_brs_disable_all(); |
895 | |
896 | /* Drain BRS is in use (could be inactive) */ |
897 | if (cpuc->lbr_users) |
898 | amd_brs_drain(); |
899 | |
900 | /* Process any counter overflows */ |
901 | handled = x86_pmu_handle_irq(regs); |
902 | |
903 | cpuc->enabled = pmu_enabled; |
904 | if (pmu_enabled) |
905 | amd_brs_enable_all(); |
906 | |
907 | return amd_pmu_adjust_nmi_window(handled); |
908 | } |
909 | |
910 | static int amd_pmu_v2_handle_irq(struct pt_regs *regs) |
911 | { |
912 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
913 | struct perf_sample_data data; |
914 | struct hw_perf_event *hwc; |
915 | struct perf_event *event; |
916 | int handled = 0, idx; |
917 | u64 reserved, status, mask; |
918 | bool pmu_enabled; |
919 | |
920 | /* |
921 | * Save the PMU state as it needs to be restored when leaving the |
922 | * handler |
923 | */ |
924 | pmu_enabled = cpuc->enabled; |
925 | cpuc->enabled = 0; |
926 | |
927 | /* Stop counting but do not disable LBR */ |
928 | amd_pmu_core_disable_all(); |
929 | |
930 | status = amd_pmu_get_global_status(); |
931 | |
932 | /* Check if any overflows are pending */ |
933 | if (!status) |
934 | goto done; |
935 | |
936 | /* Read branch records */ |
937 | if (x86_pmu.lbr_nr) { |
938 | amd_pmu_lbr_read(); |
939 | status &= ~GLOBAL_STATUS_LBRS_FROZEN; |
940 | } |
941 | |
942 | reserved = status & ~amd_pmu_global_cntr_mask; |
943 | if (reserved) |
944 | pr_warn_once("Reserved PerfCntrGlobalStatus bits are set (0x%llx), please consider updating microcode\n" , |
945 | reserved); |
946 | |
947 | /* Clear any reserved bits set by buggy microcode */ |
948 | status &= amd_pmu_global_cntr_mask; |
949 | |
950 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
951 | if (!test_bit(idx, cpuc->active_mask)) |
952 | continue; |
953 | |
954 | event = cpuc->events[idx]; |
955 | hwc = &event->hw; |
956 | x86_perf_event_update(event); |
957 | mask = BIT_ULL(idx); |
958 | |
959 | if (!(status & mask)) |
960 | continue; |
961 | |
962 | /* Event overflow */ |
963 | handled++; |
964 | status &= ~mask; |
965 | perf_sample_data_init(data: &data, addr: 0, period: hwc->last_period); |
966 | |
967 | if (!x86_perf_event_set_period(event)) |
968 | continue; |
969 | |
970 | if (has_branch_stack(event)) |
971 | perf_sample_save_brstack(data: &data, event, brs: &cpuc->lbr_stack, NULL); |
972 | |
973 | if (perf_event_overflow(event, data: &data, regs)) |
974 | x86_pmu_stop(event, flags: 0); |
975 | } |
976 | |
977 | /* |
978 | * It should never be the case that some overflows are not handled as |
979 | * the corresponding PMCs are expected to be inactive according to the |
980 | * active_mask |
981 | */ |
982 | WARN_ON(status > 0); |
983 | |
984 | /* Clear overflow and freeze bits */ |
985 | amd_pmu_ack_global_status(status: ~status); |
986 | |
987 | /* |
988 | * Unmasking the LVTPC is not required as the Mask (M) bit of the LVT |
989 | * PMI entry is not set by the local APIC when a PMC overflow occurs |
990 | */ |
991 | inc_irq_stat(apic_perf_irqs); |
992 | |
993 | done: |
994 | cpuc->enabled = pmu_enabled; |
995 | |
996 | /* Resume counting only if PMU is active */ |
997 | if (pmu_enabled) |
998 | amd_pmu_core_enable_all(); |
999 | |
1000 | return amd_pmu_adjust_nmi_window(handled); |
1001 | } |
1002 | |
1003 | static struct event_constraint * |
1004 | amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx, |
1005 | struct perf_event *event) |
1006 | { |
1007 | /* |
1008 | * if not NB event or no NB, then no constraints |
1009 | */ |
1010 | if (!(amd_has_nb(cpuc) && amd_is_nb_event(hwc: &event->hw))) |
1011 | return &unconstrained; |
1012 | |
1013 | return __amd_get_nb_event_constraints(cpuc, event, NULL); |
1014 | } |
1015 | |
1016 | static void amd_put_event_constraints(struct cpu_hw_events *cpuc, |
1017 | struct perf_event *event) |
1018 | { |
1019 | if (amd_has_nb(cpuc) && amd_is_nb_event(hwc: &event->hw)) |
1020 | __amd_put_nb_event_constraints(cpuc, event); |
1021 | } |
1022 | |
1023 | PMU_FORMAT_ATTR(event, "config:0-7,32-35" ); |
1024 | PMU_FORMAT_ATTR(umask, "config:8-15" ); |
1025 | PMU_FORMAT_ATTR(edge, "config:18" ); |
1026 | PMU_FORMAT_ATTR(inv, "config:23" ); |
1027 | PMU_FORMAT_ATTR(cmask, "config:24-31" ); |
1028 | |
1029 | static struct attribute *amd_format_attr[] = { |
1030 | &format_attr_event.attr, |
1031 | &format_attr_umask.attr, |
1032 | &format_attr_edge.attr, |
1033 | &format_attr_inv.attr, |
1034 | &format_attr_cmask.attr, |
1035 | NULL, |
1036 | }; |
1037 | |
1038 | /* AMD Family 15h */ |
1039 | |
1040 | #define AMD_EVENT_TYPE_MASK 0x000000F0ULL |
1041 | |
1042 | #define AMD_EVENT_FP 0x00000000ULL ... 0x00000010ULL |
1043 | #define AMD_EVENT_LS 0x00000020ULL ... 0x00000030ULL |
1044 | #define AMD_EVENT_DC 0x00000040ULL ... 0x00000050ULL |
1045 | #define AMD_EVENT_CU 0x00000060ULL ... 0x00000070ULL |
1046 | #define AMD_EVENT_IC_DE 0x00000080ULL ... 0x00000090ULL |
1047 | #define AMD_EVENT_EX_LS 0x000000C0ULL |
1048 | #define AMD_EVENT_DE 0x000000D0ULL |
1049 | #define AMD_EVENT_NB 0x000000E0ULL ... 0x000000F0ULL |
1050 | |
1051 | /* |
1052 | * AMD family 15h event code/PMC mappings: |
1053 | * |
1054 | * type = event_code & 0x0F0: |
1055 | * |
1056 | * 0x000 FP PERF_CTL[5:3] |
1057 | * 0x010 FP PERF_CTL[5:3] |
1058 | * 0x020 LS PERF_CTL[5:0] |
1059 | * 0x030 LS PERF_CTL[5:0] |
1060 | * 0x040 DC PERF_CTL[5:0] |
1061 | * 0x050 DC PERF_CTL[5:0] |
1062 | * 0x060 CU PERF_CTL[2:0] |
1063 | * 0x070 CU PERF_CTL[2:0] |
1064 | * 0x080 IC/DE PERF_CTL[2:0] |
1065 | * 0x090 IC/DE PERF_CTL[2:0] |
1066 | * 0x0A0 --- |
1067 | * 0x0B0 --- |
1068 | * 0x0C0 EX/LS PERF_CTL[5:0] |
1069 | * 0x0D0 DE PERF_CTL[2:0] |
1070 | * 0x0E0 NB NB_PERF_CTL[3:0] |
1071 | * 0x0F0 NB NB_PERF_CTL[3:0] |
1072 | * |
1073 | * Exceptions: |
1074 | * |
1075 | * 0x000 FP PERF_CTL[3], PERF_CTL[5:3] (*) |
1076 | * 0x003 FP PERF_CTL[3] |
1077 | * 0x004 FP PERF_CTL[3], PERF_CTL[5:3] (*) |
1078 | * 0x00B FP PERF_CTL[3] |
1079 | * 0x00D FP PERF_CTL[3] |
1080 | * 0x023 DE PERF_CTL[2:0] |
1081 | * 0x02D LS PERF_CTL[3] |
1082 | * 0x02E LS PERF_CTL[3,0] |
1083 | * 0x031 LS PERF_CTL[2:0] (**) |
1084 | * 0x043 CU PERF_CTL[2:0] |
1085 | * 0x045 CU PERF_CTL[2:0] |
1086 | * 0x046 CU PERF_CTL[2:0] |
1087 | * 0x054 CU PERF_CTL[2:0] |
1088 | * 0x055 CU PERF_CTL[2:0] |
1089 | * 0x08F IC PERF_CTL[0] |
1090 | * 0x187 DE PERF_CTL[0] |
1091 | * 0x188 DE PERF_CTL[0] |
1092 | * 0x0DB EX PERF_CTL[5:0] |
1093 | * 0x0DC LS PERF_CTL[5:0] |
1094 | * 0x0DD LS PERF_CTL[5:0] |
1095 | * 0x0DE LS PERF_CTL[5:0] |
1096 | * 0x0DF LS PERF_CTL[5:0] |
1097 | * 0x1C0 EX PERF_CTL[5:3] |
1098 | * 0x1D6 EX PERF_CTL[5:0] |
1099 | * 0x1D8 EX PERF_CTL[5:0] |
1100 | * |
1101 | * (*) depending on the umask all FPU counters may be used |
1102 | * (**) only one unitmask enabled at a time |
1103 | */ |
1104 | |
1105 | static struct event_constraint amd_f15_PMC0 = EVENT_CONSTRAINT(0, 0x01, 0); |
1106 | static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0); |
1107 | static struct event_constraint amd_f15_PMC3 = EVENT_CONSTRAINT(0, 0x08, 0); |
1108 | static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0); |
1109 | static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0); |
1110 | static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0); |
1111 | |
1112 | static struct event_constraint * |
1113 | amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, int idx, |
1114 | struct perf_event *event) |
1115 | { |
1116 | struct hw_perf_event *hwc = &event->hw; |
1117 | unsigned int event_code = amd_get_event_code(hwc); |
1118 | |
1119 | switch (event_code & AMD_EVENT_TYPE_MASK) { |
1120 | case AMD_EVENT_FP: |
1121 | switch (event_code) { |
1122 | case 0x000: |
1123 | if (!(hwc->config & 0x0000F000ULL)) |
1124 | break; |
1125 | if (!(hwc->config & 0x00000F00ULL)) |
1126 | break; |
1127 | return &amd_f15_PMC3; |
1128 | case 0x004: |
1129 | if (hweight_long(w: hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1) |
1130 | break; |
1131 | return &amd_f15_PMC3; |
1132 | case 0x003: |
1133 | case 0x00B: |
1134 | case 0x00D: |
1135 | return &amd_f15_PMC3; |
1136 | } |
1137 | return &amd_f15_PMC53; |
1138 | case AMD_EVENT_LS: |
1139 | case AMD_EVENT_DC: |
1140 | case AMD_EVENT_EX_LS: |
1141 | switch (event_code) { |
1142 | case 0x023: |
1143 | case 0x043: |
1144 | case 0x045: |
1145 | case 0x046: |
1146 | case 0x054: |
1147 | case 0x055: |
1148 | return &amd_f15_PMC20; |
1149 | case 0x02D: |
1150 | return &amd_f15_PMC3; |
1151 | case 0x02E: |
1152 | return &amd_f15_PMC30; |
1153 | case 0x031: |
1154 | if (hweight_long(w: hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1) |
1155 | return &amd_f15_PMC20; |
1156 | return &emptyconstraint; |
1157 | case 0x1C0: |
1158 | return &amd_f15_PMC53; |
1159 | default: |
1160 | return &amd_f15_PMC50; |
1161 | } |
1162 | case AMD_EVENT_CU: |
1163 | case AMD_EVENT_IC_DE: |
1164 | case AMD_EVENT_DE: |
1165 | switch (event_code) { |
1166 | case 0x08F: |
1167 | case 0x187: |
1168 | case 0x188: |
1169 | return &amd_f15_PMC0; |
1170 | case 0x0DB ... 0x0DF: |
1171 | case 0x1D6: |
1172 | case 0x1D8: |
1173 | return &amd_f15_PMC50; |
1174 | default: |
1175 | return &amd_f15_PMC20; |
1176 | } |
1177 | case AMD_EVENT_NB: |
1178 | /* moved to uncore.c */ |
1179 | return &emptyconstraint; |
1180 | default: |
1181 | return &emptyconstraint; |
1182 | } |
1183 | } |
1184 | |
1185 | static struct event_constraint pair_constraint; |
1186 | |
1187 | static struct event_constraint * |
1188 | amd_get_event_constraints_f17h(struct cpu_hw_events *cpuc, int idx, |
1189 | struct perf_event *event) |
1190 | { |
1191 | struct hw_perf_event *hwc = &event->hw; |
1192 | |
1193 | if (amd_is_pair_event_code(hwc)) |
1194 | return &pair_constraint; |
1195 | |
1196 | return &unconstrained; |
1197 | } |
1198 | |
1199 | static void amd_put_event_constraints_f17h(struct cpu_hw_events *cpuc, |
1200 | struct perf_event *event) |
1201 | { |
1202 | struct hw_perf_event *hwc = &event->hw; |
1203 | |
1204 | if (is_counter_pair(hwc)) |
1205 | --cpuc->n_pair; |
1206 | } |
1207 | |
1208 | /* |
1209 | * Because of the way BRS operates with an inactive and active phases, and |
1210 | * the link to one counter, it is not possible to have two events using BRS |
1211 | * scheduled at the same time. There would be an issue with enforcing the |
1212 | * period of each one and given that the BRS saturates, it would not be possible |
1213 | * to guarantee correlated content for all events. Therefore, in situations |
1214 | * where multiple events want to use BRS, the kernel enforces mutual exclusion. |
1215 | * Exclusion is enforced by choosing only one counter for events using BRS. |
1216 | * The event scheduling logic will then automatically multiplex the |
1217 | * events and ensure that at most one event is actively using BRS. |
1218 | * |
1219 | * The BRS counter could be any counter, but there is no constraint on Fam19h, |
1220 | * therefore all counters are equal and thus we pick the first one: PMC0 |
1221 | */ |
1222 | static struct event_constraint amd_fam19h_brs_cntr0_constraint = |
1223 | EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK); |
1224 | |
1225 | static struct event_constraint amd_fam19h_brs_pair_cntr0_constraint = |
1226 | __EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK, 1, 0, PERF_X86_EVENT_PAIR); |
1227 | |
1228 | static struct event_constraint * |
1229 | amd_get_event_constraints_f19h(struct cpu_hw_events *cpuc, int idx, |
1230 | struct perf_event *event) |
1231 | { |
1232 | struct hw_perf_event *hwc = &event->hw; |
1233 | bool has_brs = has_amd_brs(hwc); |
1234 | |
1235 | /* |
1236 | * In case BRS is used with an event requiring a counter pair, |
1237 | * the kernel allows it but only on counter 0 & 1 to enforce |
1238 | * multiplexing requiring to protect BRS in case of multiple |
1239 | * BRS users |
1240 | */ |
1241 | if (amd_is_pair_event_code(hwc)) { |
1242 | return has_brs ? &amd_fam19h_brs_pair_cntr0_constraint |
1243 | : &pair_constraint; |
1244 | } |
1245 | |
1246 | if (has_brs) |
1247 | return &amd_fam19h_brs_cntr0_constraint; |
1248 | |
1249 | return &unconstrained; |
1250 | } |
1251 | |
1252 | |
1253 | static ssize_t amd_event_sysfs_show(char *page, u64 config) |
1254 | { |
1255 | u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) | |
1256 | (config & AMD64_EVENTSEL_EVENT) >> 24; |
1257 | |
1258 | return x86_event_sysfs_show(page, config, event); |
1259 | } |
1260 | |
1261 | static void amd_pmu_limit_period(struct perf_event *event, s64 *left) |
1262 | { |
1263 | /* |
1264 | * Decrease period by the depth of the BRS feature to get the last N |
1265 | * taken branches and approximate the desired period |
1266 | */ |
1267 | if (has_branch_stack(event) && *left > x86_pmu.lbr_nr) |
1268 | *left -= x86_pmu.lbr_nr; |
1269 | } |
1270 | |
1271 | static __initconst const struct x86_pmu amd_pmu = { |
1272 | .name = "AMD" , |
1273 | .handle_irq = amd_pmu_handle_irq, |
1274 | .disable_all = amd_pmu_disable_all, |
1275 | .enable_all = amd_pmu_enable_all, |
1276 | .enable = amd_pmu_enable_event, |
1277 | .disable = amd_pmu_disable_event, |
1278 | .hw_config = amd_pmu_hw_config, |
1279 | .schedule_events = x86_schedule_events, |
1280 | .eventsel = MSR_K7_EVNTSEL0, |
1281 | .perfctr = MSR_K7_PERFCTR0, |
1282 | .addr_offset = amd_pmu_addr_offset, |
1283 | .event_map = amd_pmu_event_map, |
1284 | .max_events = ARRAY_SIZE(amd_perfmon_event_map), |
1285 | .num_counters = AMD64_NUM_COUNTERS, |
1286 | .add = amd_pmu_add_event, |
1287 | .del = amd_pmu_del_event, |
1288 | .cntval_bits = 48, |
1289 | .cntval_mask = (1ULL << 48) - 1, |
1290 | .apic = 1, |
1291 | /* use highest bit to detect overflow */ |
1292 | .max_period = (1ULL << 47) - 1, |
1293 | .get_event_constraints = amd_get_event_constraints, |
1294 | .put_event_constraints = amd_put_event_constraints, |
1295 | |
1296 | .format_attrs = amd_format_attr, |
1297 | .events_sysfs_show = amd_event_sysfs_show, |
1298 | |
1299 | .cpu_prepare = amd_pmu_cpu_prepare, |
1300 | .cpu_starting = amd_pmu_cpu_starting, |
1301 | .cpu_dead = amd_pmu_cpu_dead, |
1302 | |
1303 | .amd_nb_constraints = 1, |
1304 | }; |
1305 | |
1306 | static ssize_t branches_show(struct device *cdev, |
1307 | struct device_attribute *attr, |
1308 | char *buf) |
1309 | { |
1310 | return snprintf(buf, PAGE_SIZE, fmt: "%d\n" , x86_pmu.lbr_nr); |
1311 | } |
1312 | |
1313 | static DEVICE_ATTR_RO(branches); |
1314 | |
1315 | static struct attribute *amd_pmu_branches_attrs[] = { |
1316 | &dev_attr_branches.attr, |
1317 | NULL, |
1318 | }; |
1319 | |
1320 | static umode_t |
1321 | amd_branches_is_visible(struct kobject *kobj, struct attribute *attr, int i) |
1322 | { |
1323 | return x86_pmu.lbr_nr ? attr->mode : 0; |
1324 | } |
1325 | |
1326 | static struct attribute_group group_caps_amd_branches = { |
1327 | .name = "caps" , |
1328 | .attrs = amd_pmu_branches_attrs, |
1329 | .is_visible = amd_branches_is_visible, |
1330 | }; |
1331 | |
1332 | #ifdef CONFIG_PERF_EVENTS_AMD_BRS |
1333 | |
1334 | EVENT_ATTR_STR(branch-brs, amd_branch_brs, |
1335 | "event=" __stringify(AMD_FAM19H_BRS_EVENT)"\n" ); |
1336 | |
1337 | static struct attribute *amd_brs_events_attrs[] = { |
1338 | EVENT_PTR(amd_branch_brs), |
1339 | NULL, |
1340 | }; |
1341 | |
1342 | static umode_t |
1343 | amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i) |
1344 | { |
1345 | return static_cpu_has(X86_FEATURE_BRS) && x86_pmu.lbr_nr ? |
1346 | attr->mode : 0; |
1347 | } |
1348 | |
1349 | static struct attribute_group group_events_amd_brs = { |
1350 | .name = "events" , |
1351 | .attrs = amd_brs_events_attrs, |
1352 | .is_visible = amd_brs_is_visible, |
1353 | }; |
1354 | |
1355 | #endif /* CONFIG_PERF_EVENTS_AMD_BRS */ |
1356 | |
1357 | static const struct attribute_group *amd_attr_update[] = { |
1358 | &group_caps_amd_branches, |
1359 | #ifdef CONFIG_PERF_EVENTS_AMD_BRS |
1360 | &group_events_amd_brs, |
1361 | #endif |
1362 | NULL, |
1363 | }; |
1364 | |
1365 | static int __init amd_core_pmu_init(void) |
1366 | { |
1367 | union cpuid_0x80000022_ebx ebx; |
1368 | u64 even_ctr_mask = 0ULL; |
1369 | int i; |
1370 | |
1371 | if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE)) |
1372 | return 0; |
1373 | |
1374 | /* Avoid calculating the value each time in the NMI handler */ |
1375 | perf_nmi_window = msecs_to_jiffies(m: 100); |
1376 | |
1377 | /* |
1378 | * If core performance counter extensions exists, we must use |
1379 | * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also |
1380 | * amd_pmu_addr_offset(). |
1381 | */ |
1382 | x86_pmu.eventsel = MSR_F15H_PERF_CTL; |
1383 | x86_pmu.perfctr = MSR_F15H_PERF_CTR; |
1384 | x86_pmu.num_counters = AMD64_NUM_COUNTERS_CORE; |
1385 | |
1386 | /* Check for Performance Monitoring v2 support */ |
1387 | if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) { |
1388 | ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); |
1389 | |
1390 | /* Update PMU version for later usage */ |
1391 | x86_pmu.version = 2; |
1392 | |
1393 | /* Find the number of available Core PMCs */ |
1394 | x86_pmu.num_counters = ebx.split.num_core_pmc; |
1395 | |
1396 | amd_pmu_global_cntr_mask = (1ULL << x86_pmu.num_counters) - 1; |
1397 | |
1398 | /* Update PMC handling functions */ |
1399 | x86_pmu.enable_all = amd_pmu_v2_enable_all; |
1400 | x86_pmu.disable_all = amd_pmu_v2_disable_all; |
1401 | x86_pmu.enable = amd_pmu_v2_enable_event; |
1402 | x86_pmu.handle_irq = amd_pmu_v2_handle_irq; |
1403 | static_call_update(amd_pmu_test_overflow, amd_pmu_test_overflow_status); |
1404 | } |
1405 | |
1406 | /* |
1407 | * AMD Core perfctr has separate MSRs for the NB events, see |
1408 | * the amd/uncore.c driver. |
1409 | */ |
1410 | x86_pmu.amd_nb_constraints = 0; |
1411 | |
1412 | if (boot_cpu_data.x86 == 0x15) { |
1413 | pr_cont("Fam15h " ); |
1414 | x86_pmu.get_event_constraints = amd_get_event_constraints_f15h; |
1415 | } |
1416 | if (boot_cpu_data.x86 >= 0x17) { |
1417 | pr_cont("Fam17h+ " ); |
1418 | /* |
1419 | * Family 17h and compatibles have constraints for Large |
1420 | * Increment per Cycle events: they may only be assigned an |
1421 | * even numbered counter that has a consecutive adjacent odd |
1422 | * numbered counter following it. |
1423 | */ |
1424 | for (i = 0; i < x86_pmu.num_counters - 1; i += 2) |
1425 | even_ctr_mask |= BIT_ULL(i); |
1426 | |
1427 | pair_constraint = (struct event_constraint) |
1428 | __EVENT_CONSTRAINT(0, even_ctr_mask, 0, |
1429 | x86_pmu.num_counters / 2, 0, |
1430 | PERF_X86_EVENT_PAIR); |
1431 | |
1432 | x86_pmu.get_event_constraints = amd_get_event_constraints_f17h; |
1433 | x86_pmu.put_event_constraints = amd_put_event_constraints_f17h; |
1434 | x86_pmu.perf_ctr_pair_en = AMD_MERGE_EVENT_ENABLE; |
1435 | x86_pmu.flags |= PMU_FL_PAIR; |
1436 | } |
1437 | |
1438 | /* LBR and BRS are mutually exclusive features */ |
1439 | if (!amd_pmu_lbr_init()) { |
1440 | /* LBR requires flushing on context switch */ |
1441 | x86_pmu.sched_task = amd_pmu_lbr_sched_task; |
1442 | static_call_update(amd_pmu_branch_hw_config, amd_pmu_lbr_hw_config); |
1443 | static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset); |
1444 | static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add); |
1445 | static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del); |
1446 | } else if (!amd_brs_init()) { |
1447 | /* |
1448 | * BRS requires special event constraints and flushing on ctxsw. |
1449 | */ |
1450 | x86_pmu.get_event_constraints = amd_get_event_constraints_f19h; |
1451 | x86_pmu.sched_task = amd_pmu_brs_sched_task; |
1452 | x86_pmu.limit_period = amd_pmu_limit_period; |
1453 | |
1454 | static_call_update(amd_pmu_branch_hw_config, amd_brs_hw_config); |
1455 | static_call_update(amd_pmu_branch_reset, amd_brs_reset); |
1456 | static_call_update(amd_pmu_branch_add, amd_pmu_brs_add); |
1457 | static_call_update(amd_pmu_branch_del, amd_pmu_brs_del); |
1458 | |
1459 | /* |
1460 | * put_event_constraints callback same as Fam17h, set above |
1461 | */ |
1462 | |
1463 | /* branch sampling must be stopped when entering low power */ |
1464 | amd_brs_lopwr_init(); |
1465 | } |
1466 | |
1467 | x86_pmu.attr_update = amd_attr_update; |
1468 | |
1469 | pr_cont("core perfctr, " ); |
1470 | return 0; |
1471 | } |
1472 | |
1473 | __init int amd_pmu_init(void) |
1474 | { |
1475 | int ret; |
1476 | |
1477 | /* Performance-monitoring supported from K7 and later: */ |
1478 | if (boot_cpu_data.x86 < 6) |
1479 | return -ENODEV; |
1480 | |
1481 | x86_pmu = amd_pmu; |
1482 | |
1483 | ret = amd_core_pmu_init(); |
1484 | if (ret) |
1485 | return ret; |
1486 | |
1487 | if (num_possible_cpus() == 1) { |
1488 | /* |
1489 | * No point in allocating data structures to serialize |
1490 | * against other CPUs, when there is only the one CPU. |
1491 | */ |
1492 | x86_pmu.amd_nb_constraints = 0; |
1493 | } |
1494 | |
1495 | if (boot_cpu_data.x86 >= 0x17) |
1496 | memcpy(hw_cache_event_ids, amd_hw_cache_event_ids_f17h, sizeof(hw_cache_event_ids)); |
1497 | else |
1498 | memcpy(hw_cache_event_ids, amd_hw_cache_event_ids, sizeof(hw_cache_event_ids)); |
1499 | |
1500 | return 0; |
1501 | } |
1502 | |
1503 | static inline void amd_pmu_reload_virt(void) |
1504 | { |
1505 | if (x86_pmu.version >= 2) { |
1506 | /* |
1507 | * Clear global enable bits, reprogram the PERF_CTL |
1508 | * registers with updated perf_ctr_virt_mask and then |
1509 | * set global enable bits once again |
1510 | */ |
1511 | amd_pmu_v2_disable_all(); |
1512 | amd_pmu_enable_all(added: 0); |
1513 | amd_pmu_v2_enable_all(added: 0); |
1514 | return; |
1515 | } |
1516 | |
1517 | amd_pmu_disable_all(); |
1518 | amd_pmu_enable_all(added: 0); |
1519 | } |
1520 | |
1521 | void amd_pmu_enable_virt(void) |
1522 | { |
1523 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
1524 | |
1525 | cpuc->perf_ctr_virt_mask = 0; |
1526 | |
1527 | /* Reload all events */ |
1528 | amd_pmu_reload_virt(); |
1529 | } |
1530 | EXPORT_SYMBOL_GPL(amd_pmu_enable_virt); |
1531 | |
1532 | void amd_pmu_disable_virt(void) |
1533 | { |
1534 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
1535 | |
1536 | /* |
1537 | * We only mask out the Host-only bit so that host-only counting works |
1538 | * when SVM is disabled. If someone sets up a guest-only counter when |
1539 | * SVM is disabled the Guest-only bits still gets set and the counter |
1540 | * will not count anything. |
1541 | */ |
1542 | cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY; |
1543 | |
1544 | /* Reload all events */ |
1545 | amd_pmu_reload_virt(); |
1546 | } |
1547 | EXPORT_SYMBOL_GPL(amd_pmu_disable_virt); |
1548 | |