1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * |
4 | * Copyright (C) 2013 Citrix Systems |
5 | * |
6 | * Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com> |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "arm-pv: " fmt |
10 | |
11 | #include <linux/arm-smccc.h> |
12 | #include <linux/cpuhotplug.h> |
13 | #include <linux/export.h> |
14 | #include <linux/io.h> |
15 | #include <linux/jump_label.h> |
16 | #include <linux/printk.h> |
17 | #include <linux/psci.h> |
18 | #include <linux/reboot.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/types.h> |
21 | #include <linux/static_call.h> |
22 | |
23 | #include <asm/paravirt.h> |
24 | #include <asm/pvclock-abi.h> |
25 | #include <asm/smp_plat.h> |
26 | |
27 | struct static_key paravirt_steal_enabled; |
28 | struct static_key paravirt_steal_rq_enabled; |
29 | |
30 | static u64 native_steal_clock(int cpu) |
31 | { |
32 | return 0; |
33 | } |
34 | |
35 | DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); |
36 | |
37 | struct pv_time_stolen_time_region { |
38 | struct pvclock_vcpu_stolen_time __rcu *kaddr; |
39 | }; |
40 | |
41 | static DEFINE_PER_CPU(struct pv_time_stolen_time_region, stolen_time_region); |
42 | |
43 | static bool steal_acc = true; |
44 | static int __init parse_no_stealacc(char *arg) |
45 | { |
46 | steal_acc = false; |
47 | return 0; |
48 | } |
49 | |
50 | early_param("no-steal-acc" , parse_no_stealacc); |
51 | |
52 | /* return stolen time in ns by asking the hypervisor */ |
53 | static u64 para_steal_clock(int cpu) |
54 | { |
55 | struct pvclock_vcpu_stolen_time *kaddr = NULL; |
56 | struct pv_time_stolen_time_region *reg; |
57 | u64 ret = 0; |
58 | |
59 | reg = per_cpu_ptr(&stolen_time_region, cpu); |
60 | |
61 | /* |
62 | * paravirt_steal_clock() may be called before the CPU |
63 | * online notification callback runs. Until the callback |
64 | * has run we just return zero. |
65 | */ |
66 | rcu_read_lock(); |
67 | kaddr = rcu_dereference(reg->kaddr); |
68 | if (!kaddr) { |
69 | rcu_read_unlock(); |
70 | return 0; |
71 | } |
72 | |
73 | ret = le64_to_cpu(READ_ONCE(kaddr->stolen_time)); |
74 | rcu_read_unlock(); |
75 | return ret; |
76 | } |
77 | |
78 | static int stolen_time_cpu_down_prepare(unsigned int cpu) |
79 | { |
80 | struct pvclock_vcpu_stolen_time *kaddr = NULL; |
81 | struct pv_time_stolen_time_region *reg; |
82 | |
83 | reg = this_cpu_ptr(&stolen_time_region); |
84 | if (!reg->kaddr) |
85 | return 0; |
86 | |
87 | kaddr = rcu_replace_pointer(reg->kaddr, NULL, true); |
88 | synchronize_rcu(); |
89 | memunmap(addr: kaddr); |
90 | |
91 | return 0; |
92 | } |
93 | |
94 | static int stolen_time_cpu_online(unsigned int cpu) |
95 | { |
96 | struct pvclock_vcpu_stolen_time *kaddr = NULL; |
97 | struct pv_time_stolen_time_region *reg; |
98 | struct arm_smccc_res res; |
99 | |
100 | reg = this_cpu_ptr(&stolen_time_region); |
101 | |
102 | arm_smccc_1_1_invoke(ARM_SMCCC_HV_PV_TIME_ST, &res); |
103 | |
104 | if (res.a0 == SMCCC_RET_NOT_SUPPORTED) |
105 | return -EINVAL; |
106 | |
107 | kaddr = memremap(res.a0, |
108 | sizeof(struct pvclock_vcpu_stolen_time), |
109 | MEMREMAP_WB); |
110 | |
111 | rcu_assign_pointer(reg->kaddr, kaddr); |
112 | |
113 | if (!reg->kaddr) { |
114 | pr_warn("Failed to map stolen time data structure\n" ); |
115 | return -ENOMEM; |
116 | } |
117 | |
118 | if (le32_to_cpu(kaddr->revision) != 0 || |
119 | le32_to_cpu(kaddr->attributes) != 0) { |
120 | pr_warn_once("Unexpected revision or attributes in stolen time data\n" ); |
121 | return -ENXIO; |
122 | } |
123 | |
124 | return 0; |
125 | } |
126 | |
127 | static int __init pv_time_init_stolen_time(void) |
128 | { |
129 | int ret; |
130 | |
131 | ret = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, |
132 | name: "hypervisor/arm/pvtime:online" , |
133 | startup: stolen_time_cpu_online, |
134 | teardown: stolen_time_cpu_down_prepare); |
135 | if (ret < 0) |
136 | return ret; |
137 | return 0; |
138 | } |
139 | |
140 | static bool __init has_pv_steal_clock(void) |
141 | { |
142 | struct arm_smccc_res res; |
143 | |
144 | arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
145 | ARM_SMCCC_HV_PV_TIME_FEATURES, &res); |
146 | |
147 | if (res.a0 != SMCCC_RET_SUCCESS) |
148 | return false; |
149 | |
150 | arm_smccc_1_1_invoke(ARM_SMCCC_HV_PV_TIME_FEATURES, |
151 | ARM_SMCCC_HV_PV_TIME_ST, &res); |
152 | |
153 | return (res.a0 == SMCCC_RET_SUCCESS); |
154 | } |
155 | |
156 | int __init pv_time_init(void) |
157 | { |
158 | int ret; |
159 | |
160 | if (!has_pv_steal_clock()) |
161 | return 0; |
162 | |
163 | ret = pv_time_init_stolen_time(); |
164 | if (ret) |
165 | return ret; |
166 | |
167 | static_call_update(pv_steal_clock, para_steal_clock); |
168 | |
169 | static_key_slow_inc(key: ¶virt_steal_enabled); |
170 | if (steal_acc) |
171 | static_key_slow_inc(key: ¶virt_steal_rq_enabled); |
172 | |
173 | pr_info("using stolen time PV\n" ); |
174 | |
175 | return 0; |
176 | } |
177 | |