1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Intel Running Average Power Limit (RAPL) Driver via MSR interface |
4 | * Copyright (c) 2019, Intel Corporation. |
5 | */ |
6 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
7 | |
8 | #include <linux/kernel.h> |
9 | #include <linux/module.h> |
10 | #include <linux/list.h> |
11 | #include <linux/types.h> |
12 | #include <linux/device.h> |
13 | #include <linux/slab.h> |
14 | #include <linux/log2.h> |
15 | #include <linux/bitmap.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/sysfs.h> |
18 | #include <linux/cpu.h> |
19 | #include <linux/powercap.h> |
20 | #include <linux/suspend.h> |
21 | #include <linux/intel_rapl.h> |
22 | #include <linux/processor.h> |
23 | #include <linux/platform_device.h> |
24 | |
25 | #include <asm/cpu_device_id.h> |
26 | #include <asm/intel-family.h> |
27 | |
28 | /* Local defines */ |
29 | #define MSR_PLATFORM_POWER_LIMIT 0x0000065C |
30 | #define MSR_VR_CURRENT_CONFIG 0x00000601 |
31 | |
32 | /* private data for RAPL MSR Interface */ |
33 | static struct rapl_if_priv *rapl_msr_priv; |
34 | |
35 | static struct rapl_if_priv rapl_msr_priv_intel = { |
36 | .type = RAPL_IF_MSR, |
37 | .reg_unit.msr = MSR_RAPL_POWER_UNIT, |
38 | .regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_LIMIT].msr = MSR_PKG_POWER_LIMIT, |
39 | .regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_STATUS].msr = MSR_PKG_ENERGY_STATUS, |
40 | .regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_PERF].msr = MSR_PKG_PERF_STATUS, |
41 | .regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_INFO].msr = MSR_PKG_POWER_INFO, |
42 | .regs[RAPL_DOMAIN_PP0][RAPL_DOMAIN_REG_LIMIT].msr = MSR_PP0_POWER_LIMIT, |
43 | .regs[RAPL_DOMAIN_PP0][RAPL_DOMAIN_REG_STATUS].msr = MSR_PP0_ENERGY_STATUS, |
44 | .regs[RAPL_DOMAIN_PP0][RAPL_DOMAIN_REG_POLICY].msr = MSR_PP0_POLICY, |
45 | .regs[RAPL_DOMAIN_PP1][RAPL_DOMAIN_REG_LIMIT].msr = MSR_PP1_POWER_LIMIT, |
46 | .regs[RAPL_DOMAIN_PP1][RAPL_DOMAIN_REG_STATUS].msr = MSR_PP1_ENERGY_STATUS, |
47 | .regs[RAPL_DOMAIN_PP1][RAPL_DOMAIN_REG_POLICY].msr = MSR_PP1_POLICY, |
48 | .regs[RAPL_DOMAIN_DRAM][RAPL_DOMAIN_REG_LIMIT].msr = MSR_DRAM_POWER_LIMIT, |
49 | .regs[RAPL_DOMAIN_DRAM][RAPL_DOMAIN_REG_STATUS].msr = MSR_DRAM_ENERGY_STATUS, |
50 | .regs[RAPL_DOMAIN_DRAM][RAPL_DOMAIN_REG_PERF].msr = MSR_DRAM_PERF_STATUS, |
51 | .regs[RAPL_DOMAIN_DRAM][RAPL_DOMAIN_REG_INFO].msr = MSR_DRAM_POWER_INFO, |
52 | .regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_LIMIT].msr = MSR_PLATFORM_POWER_LIMIT, |
53 | .regs[RAPL_DOMAIN_PLATFORM][RAPL_DOMAIN_REG_STATUS].msr = MSR_PLATFORM_ENERGY_STATUS, |
54 | .limits[RAPL_DOMAIN_PACKAGE] = BIT(POWER_LIMIT2), |
55 | .limits[RAPL_DOMAIN_PLATFORM] = BIT(POWER_LIMIT2), |
56 | }; |
57 | |
58 | static struct rapl_if_priv rapl_msr_priv_amd = { |
59 | .type = RAPL_IF_MSR, |
60 | .reg_unit.msr = MSR_AMD_RAPL_POWER_UNIT, |
61 | .regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_STATUS].msr = MSR_AMD_PKG_ENERGY_STATUS, |
62 | .regs[RAPL_DOMAIN_PP0][RAPL_DOMAIN_REG_STATUS].msr = MSR_AMD_CORE_ENERGY_STATUS, |
63 | }; |
64 | |
65 | /* Handles CPU hotplug on multi-socket systems. |
66 | * If a CPU goes online as the first CPU of the physical package |
67 | * we add the RAPL package to the system. Similarly, when the last |
68 | * CPU of the package is removed, we remove the RAPL package and its |
69 | * associated domains. Cooling devices are handled accordingly at |
70 | * per-domain level. |
71 | */ |
72 | static int rapl_cpu_online(unsigned int cpu) |
73 | { |
74 | struct rapl_package *rp; |
75 | |
76 | rp = rapl_find_package_domain_cpuslocked(id: cpu, priv: rapl_msr_priv, id_is_cpu: true); |
77 | if (!rp) { |
78 | rp = rapl_add_package_cpuslocked(id: cpu, priv: rapl_msr_priv, id_is_cpu: true); |
79 | if (IS_ERR(ptr: rp)) |
80 | return PTR_ERR(ptr: rp); |
81 | } |
82 | cpumask_set_cpu(cpu, dstp: &rp->cpumask); |
83 | return 0; |
84 | } |
85 | |
86 | static int rapl_cpu_down_prep(unsigned int cpu) |
87 | { |
88 | struct rapl_package *rp; |
89 | int lead_cpu; |
90 | |
91 | rp = rapl_find_package_domain_cpuslocked(id: cpu, priv: rapl_msr_priv, id_is_cpu: true); |
92 | if (!rp) |
93 | return 0; |
94 | |
95 | cpumask_clear_cpu(cpu, dstp: &rp->cpumask); |
96 | lead_cpu = cpumask_first(srcp: &rp->cpumask); |
97 | if (lead_cpu >= nr_cpu_ids) |
98 | rapl_remove_package_cpuslocked(rp); |
99 | else if (rp->lead_cpu == cpu) |
100 | rp->lead_cpu = lead_cpu; |
101 | return 0; |
102 | } |
103 | |
104 | static int rapl_msr_read_raw(int cpu, struct reg_action *ra) |
105 | { |
106 | if (rdmsrl_safe_on_cpu(cpu, msr_no: ra->reg.msr, q: &ra->value)) { |
107 | pr_debug("failed to read msr 0x%x on cpu %d\n" , ra->reg.msr, cpu); |
108 | return -EIO; |
109 | } |
110 | ra->value &= ra->mask; |
111 | return 0; |
112 | } |
113 | |
114 | static void rapl_msr_update_func(void *info) |
115 | { |
116 | struct reg_action *ra = info; |
117 | u64 val; |
118 | |
119 | ra->err = rdmsrl_safe(msr: ra->reg.msr, p: &val); |
120 | if (ra->err) |
121 | return; |
122 | |
123 | val &= ~ra->mask; |
124 | val |= ra->value; |
125 | |
126 | ra->err = wrmsrl_safe(msr: ra->reg.msr, val); |
127 | } |
128 | |
129 | static int rapl_msr_write_raw(int cpu, struct reg_action *ra) |
130 | { |
131 | int ret; |
132 | |
133 | ret = smp_call_function_single(cpuid: cpu, func: rapl_msr_update_func, info: ra, wait: 1); |
134 | if (WARN_ON_ONCE(ret)) |
135 | return ret; |
136 | |
137 | return ra->err; |
138 | } |
139 | |
140 | /* List of verified CPUs. */ |
141 | static const struct x86_cpu_id pl4_support_ids[] = { |
142 | X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, NULL), |
143 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, NULL), |
144 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, NULL), |
145 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, NULL), |
146 | X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, NULL), |
147 | X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, NULL), |
148 | X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, NULL), |
149 | X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, NULL), |
150 | {} |
151 | }; |
152 | |
153 | static int rapl_msr_probe(struct platform_device *pdev) |
154 | { |
155 | const struct x86_cpu_id *id = x86_match_cpu(match: pl4_support_ids); |
156 | int ret; |
157 | |
158 | switch (boot_cpu_data.x86_vendor) { |
159 | case X86_VENDOR_INTEL: |
160 | rapl_msr_priv = &rapl_msr_priv_intel; |
161 | break; |
162 | case X86_VENDOR_HYGON: |
163 | case X86_VENDOR_AMD: |
164 | rapl_msr_priv = &rapl_msr_priv_amd; |
165 | break; |
166 | default: |
167 | pr_err("intel-rapl does not support CPU vendor %d\n" , boot_cpu_data.x86_vendor); |
168 | return -ENODEV; |
169 | } |
170 | rapl_msr_priv->read_raw = rapl_msr_read_raw; |
171 | rapl_msr_priv->write_raw = rapl_msr_write_raw; |
172 | |
173 | if (id) { |
174 | rapl_msr_priv->limits[RAPL_DOMAIN_PACKAGE] |= BIT(POWER_LIMIT4); |
175 | rapl_msr_priv->regs[RAPL_DOMAIN_PACKAGE][RAPL_DOMAIN_REG_PL4].msr = |
176 | MSR_VR_CURRENT_CONFIG; |
177 | pr_info("PL4 support detected.\n" ); |
178 | } |
179 | |
180 | rapl_msr_priv->control_type = powercap_register_control_type(NULL, name: "intel-rapl" , NULL); |
181 | if (IS_ERR(ptr: rapl_msr_priv->control_type)) { |
182 | pr_debug("failed to register powercap control_type.\n" ); |
183 | return PTR_ERR(ptr: rapl_msr_priv->control_type); |
184 | } |
185 | |
186 | ret = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "powercap/rapl:online" , |
187 | startup: rapl_cpu_online, teardown: rapl_cpu_down_prep); |
188 | if (ret < 0) |
189 | goto out; |
190 | rapl_msr_priv->pcap_rapl_online = ret; |
191 | |
192 | return 0; |
193 | |
194 | out: |
195 | if (ret) |
196 | powercap_unregister_control_type(instance: rapl_msr_priv->control_type); |
197 | return ret; |
198 | } |
199 | |
200 | static void rapl_msr_remove(struct platform_device *pdev) |
201 | { |
202 | cpuhp_remove_state(state: rapl_msr_priv->pcap_rapl_online); |
203 | powercap_unregister_control_type(instance: rapl_msr_priv->control_type); |
204 | } |
205 | |
206 | static const struct platform_device_id rapl_msr_ids[] = { |
207 | { .name = "intel_rapl_msr" , }, |
208 | {} |
209 | }; |
210 | MODULE_DEVICE_TABLE(platform, rapl_msr_ids); |
211 | |
212 | static struct platform_driver intel_rapl_msr_driver = { |
213 | .probe = rapl_msr_probe, |
214 | .remove_new = rapl_msr_remove, |
215 | .id_table = rapl_msr_ids, |
216 | .driver = { |
217 | .name = "intel_rapl_msr" , |
218 | }, |
219 | }; |
220 | |
221 | module_platform_driver(intel_rapl_msr_driver); |
222 | |
223 | MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit) control via MSR interface" ); |
224 | MODULE_AUTHOR("Zhang Rui <rui.zhang@intel.com>" ); |
225 | MODULE_LICENSE("GPL v2" ); |
226 | |