1 | // SPDX-License-Identifier: GPL-2.0 |
---|---|
2 | /* |
3 | * local apic based NMI watchdog for various CPUs. |
4 | * |
5 | * This file also handles reservation of performance counters for coordination |
6 | * with other users. |
7 | * |
8 | * Note that these events normally don't tick when the CPU idles. This means |
9 | * the frequency varies with CPU load. |
10 | * |
11 | * Original code for K7/P6 written by Keith Owens |
12 | * |
13 | */ |
14 | |
15 | #include <linux/percpu.h> |
16 | #include <linux/export.h> |
17 | #include <linux/kernel.h> |
18 | #include <linux/bitops.h> |
19 | #include <linux/smp.h> |
20 | #include <asm/nmi.h> |
21 | #include <linux/kprobes.h> |
22 | |
23 | #include <asm/apic.h> |
24 | #include <asm/perf_event.h> |
25 | |
26 | /* |
27 | * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's |
28 | * offset from MSR_P4_BSU_ESCR0. |
29 | * |
30 | * It will be the max for all platforms (for now) |
31 | */ |
32 | #define NMI_MAX_COUNTER_BITS 66 |
33 | |
34 | /* |
35 | * perfctr_nmi_owner tracks the ownership of the perfctr registers: |
36 | * evtsel_nmi_owner tracks the ownership of the event selection |
37 | * - different performance counters/ event selection may be reserved for |
38 | * different subsystems this reservation system just tries to coordinate |
39 | * things a little |
40 | */ |
41 | static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS); |
42 | static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS); |
43 | |
44 | /* converts an msr to an appropriate reservation bit */ |
45 | static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr) |
46 | { |
47 | /* returns the bit offset of the performance counter register */ |
48 | switch (boot_cpu_data.x86_vendor) { |
49 | case X86_VENDOR_HYGON: |
50 | case X86_VENDOR_AMD: |
51 | if (msr >= MSR_F15H_PERF_CTR) |
52 | return (msr - MSR_F15H_PERF_CTR) >> 1; |
53 | return msr - MSR_K7_PERFCTR0; |
54 | case X86_VENDOR_INTEL: |
55 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
56 | return msr - MSR_ARCH_PERFMON_PERFCTR0; |
57 | |
58 | switch (boot_cpu_data.x86) { |
59 | case 6: |
60 | return msr - MSR_P6_PERFCTR0; |
61 | case 11: |
62 | return msr - MSR_KNC_PERFCTR0; |
63 | case 15: |
64 | return msr - MSR_P4_BPU_PERFCTR0; |
65 | } |
66 | break; |
67 | case X86_VENDOR_ZHAOXIN: |
68 | case X86_VENDOR_CENTAUR: |
69 | return msr - MSR_ARCH_PERFMON_PERFCTR0; |
70 | } |
71 | return 0; |
72 | } |
73 | |
74 | /* |
75 | * converts an msr to an appropriate reservation bit |
76 | * returns the bit offset of the event selection register |
77 | */ |
78 | static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr) |
79 | { |
80 | /* returns the bit offset of the event selection register */ |
81 | switch (boot_cpu_data.x86_vendor) { |
82 | case X86_VENDOR_HYGON: |
83 | case X86_VENDOR_AMD: |
84 | if (msr >= MSR_F15H_PERF_CTL) |
85 | return (msr - MSR_F15H_PERF_CTL) >> 1; |
86 | return msr - MSR_K7_EVNTSEL0; |
87 | case X86_VENDOR_INTEL: |
88 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
89 | return msr - MSR_ARCH_PERFMON_EVENTSEL0; |
90 | |
91 | switch (boot_cpu_data.x86) { |
92 | case 6: |
93 | return msr - MSR_P6_EVNTSEL0; |
94 | case 11: |
95 | return msr - MSR_KNC_EVNTSEL0; |
96 | case 15: |
97 | return msr - MSR_P4_BSU_ESCR0; |
98 | } |
99 | break; |
100 | case X86_VENDOR_ZHAOXIN: |
101 | case X86_VENDOR_CENTAUR: |
102 | return msr - MSR_ARCH_PERFMON_EVENTSEL0; |
103 | } |
104 | return 0; |
105 | |
106 | } |
107 | |
108 | int reserve_perfctr_nmi(unsigned int msr) |
109 | { |
110 | unsigned int counter; |
111 | |
112 | counter = nmi_perfctr_msr_to_bit(msr); |
113 | /* register not managed by the allocator? */ |
114 | if (counter > NMI_MAX_COUNTER_BITS) |
115 | return 1; |
116 | |
117 | if (!test_and_set_bit(nr: counter, addr: perfctr_nmi_owner)) |
118 | return 1; |
119 | return 0; |
120 | } |
121 | EXPORT_SYMBOL(reserve_perfctr_nmi); |
122 | |
123 | void release_perfctr_nmi(unsigned int msr) |
124 | { |
125 | unsigned int counter; |
126 | |
127 | counter = nmi_perfctr_msr_to_bit(msr); |
128 | /* register not managed by the allocator? */ |
129 | if (counter > NMI_MAX_COUNTER_BITS) |
130 | return; |
131 | |
132 | clear_bit(nr: counter, addr: perfctr_nmi_owner); |
133 | } |
134 | EXPORT_SYMBOL(release_perfctr_nmi); |
135 | |
136 | int reserve_evntsel_nmi(unsigned int msr) |
137 | { |
138 | unsigned int counter; |
139 | |
140 | counter = nmi_evntsel_msr_to_bit(msr); |
141 | /* register not managed by the allocator? */ |
142 | if (counter > NMI_MAX_COUNTER_BITS) |
143 | return 1; |
144 | |
145 | if (!test_and_set_bit(nr: counter, addr: evntsel_nmi_owner)) |
146 | return 1; |
147 | return 0; |
148 | } |
149 | EXPORT_SYMBOL(reserve_evntsel_nmi); |
150 | |
151 | void release_evntsel_nmi(unsigned int msr) |
152 | { |
153 | unsigned int counter; |
154 | |
155 | counter = nmi_evntsel_msr_to_bit(msr); |
156 | /* register not managed by the allocator? */ |
157 | if (counter > NMI_MAX_COUNTER_BITS) |
158 | return; |
159 | |
160 | clear_bit(nr: counter, addr: evntsel_nmi_owner); |
161 | } |
162 | EXPORT_SYMBOL(release_evntsel_nmi); |
163 |