1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright 2006 Andi Kleen, SUSE Labs. |
4 | * |
5 | * Fast user context implementation of clock_gettime, gettimeofday, and time. |
6 | * |
7 | * The code should have no internal unresolved relocations. |
8 | * Check with readelf after changing. |
9 | * Also alternative() doesn't work. |
10 | */ |
11 | /* |
12 | * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. |
13 | */ |
14 | |
15 | #include <linux/kernel.h> |
16 | #include <linux/time.h> |
17 | #include <linux/string.h> |
18 | #include <asm/io.h> |
19 | #include <asm/unistd.h> |
20 | #include <asm/timex.h> |
21 | #include <asm/clocksource.h> |
22 | #include <asm/vvar.h> |
23 | |
24 | #ifdef CONFIG_SPARC64 |
25 | #define SYSCALL_STRING \ |
26 | "ta 0x6d;" \ |
27 | "bcs,a 1f;" \ |
28 | " sub %%g0, %%o0, %%o0;" \ |
29 | "1:" |
30 | #else |
31 | #define SYSCALL_STRING \ |
32 | "ta 0x10;" \ |
33 | "bcs,a 1f;" \ |
34 | " sub %%g0, %%o0, %%o0;" \ |
35 | "1:" |
36 | #endif |
37 | |
38 | #define SYSCALL_CLOBBERS \ |
39 | "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ |
40 | "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ |
41 | "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ |
42 | "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ |
43 | "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ |
44 | "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ |
45 | "cc", "memory" |
46 | |
47 | /* |
48 | * Compute the vvar page's address in the process address space, and return it |
49 | * as a pointer to the vvar_data. |
50 | */ |
51 | notrace static __always_inline struct vvar_data *get_vvar_data(void) |
52 | { |
53 | unsigned long ret; |
54 | |
55 | /* |
56 | * vdso data page is the first vDSO page so grab the PC |
57 | * and move up a page to get to the data page. |
58 | */ |
59 | __asm__("rd %%pc, %0" : "=r" (ret)); |
60 | ret &= ~(8192 - 1); |
61 | ret -= 8192; |
62 | |
63 | return (struct vvar_data *) ret; |
64 | } |
65 | |
66 | notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts) |
67 | { |
68 | register long num __asm__("g1" ) = __NR_clock_gettime; |
69 | register long o0 __asm__("o0" ) = clock; |
70 | register long o1 __asm__("o1" ) = (long) ts; |
71 | |
72 | __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), |
73 | "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); |
74 | return o0; |
75 | } |
76 | |
77 | notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) |
78 | { |
79 | register long num __asm__("g1" ) = __NR_gettimeofday; |
80 | register long o0 __asm__("o0" ) = (long) tv; |
81 | register long o1 __asm__("o1" ) = (long) tz; |
82 | |
83 | __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), |
84 | "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); |
85 | return o0; |
86 | } |
87 | |
88 | #ifdef CONFIG_SPARC64 |
89 | notrace static __always_inline u64 vread_tick(void) |
90 | { |
91 | u64 ret; |
92 | |
93 | __asm__ __volatile__("rd %%tick, %0" : "=r" (ret)); |
94 | return ret; |
95 | } |
96 | |
97 | notrace static __always_inline u64 vread_tick_stick(void) |
98 | { |
99 | u64 ret; |
100 | |
101 | __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret)); |
102 | return ret; |
103 | } |
104 | #else |
105 | notrace static __always_inline u64 vread_tick(void) |
106 | { |
107 | register unsigned long long ret asm("o4" ); |
108 | |
109 | __asm__ __volatile__("rd %%tick, %L0\n\t" |
110 | "srlx %L0, 32, %H0" |
111 | : "=r" (ret)); |
112 | return ret; |
113 | } |
114 | |
115 | notrace static __always_inline u64 vread_tick_stick(void) |
116 | { |
117 | register unsigned long long ret asm("o4" ); |
118 | |
119 | __asm__ __volatile__("rd %%asr24, %L0\n\t" |
120 | "srlx %L0, 32, %H0" |
121 | : "=r" (ret)); |
122 | return ret; |
123 | } |
124 | #endif |
125 | |
126 | notrace static __always_inline u64 vgetsns(struct vvar_data *vvar) |
127 | { |
128 | u64 v; |
129 | u64 cycles; |
130 | |
131 | cycles = vread_tick(); |
132 | v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; |
133 | return v * vvar->clock.mult; |
134 | } |
135 | |
136 | notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar) |
137 | { |
138 | u64 v; |
139 | u64 cycles; |
140 | |
141 | cycles = vread_tick_stick(); |
142 | v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; |
143 | return v * vvar->clock.mult; |
144 | } |
145 | |
146 | notrace static __always_inline int do_realtime(struct vvar_data *vvar, |
147 | struct __kernel_old_timespec *ts) |
148 | { |
149 | unsigned long seq; |
150 | u64 ns; |
151 | |
152 | do { |
153 | seq = vvar_read_begin(vvar); |
154 | ts->tv_sec = vvar->wall_time_sec; |
155 | ns = vvar->wall_time_snsec; |
156 | ns += vgetsns(vvar); |
157 | ns >>= vvar->clock.shift; |
158 | } while (unlikely(vvar_read_retry(vvar, seq))); |
159 | |
160 | ts->tv_sec += __iter_div_u64_rem(dividend: ns, NSEC_PER_SEC, remainder: &ns); |
161 | ts->tv_nsec = ns; |
162 | |
163 | return 0; |
164 | } |
165 | |
166 | notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar, |
167 | struct __kernel_old_timespec *ts) |
168 | { |
169 | unsigned long seq; |
170 | u64 ns; |
171 | |
172 | do { |
173 | seq = vvar_read_begin(vvar); |
174 | ts->tv_sec = vvar->wall_time_sec; |
175 | ns = vvar->wall_time_snsec; |
176 | ns += vgetsns_stick(vvar); |
177 | ns >>= vvar->clock.shift; |
178 | } while (unlikely(vvar_read_retry(vvar, seq))); |
179 | |
180 | ts->tv_sec += __iter_div_u64_rem(dividend: ns, NSEC_PER_SEC, remainder: &ns); |
181 | ts->tv_nsec = ns; |
182 | |
183 | return 0; |
184 | } |
185 | |
186 | notrace static __always_inline int do_monotonic(struct vvar_data *vvar, |
187 | struct __kernel_old_timespec *ts) |
188 | { |
189 | unsigned long seq; |
190 | u64 ns; |
191 | |
192 | do { |
193 | seq = vvar_read_begin(vvar); |
194 | ts->tv_sec = vvar->monotonic_time_sec; |
195 | ns = vvar->monotonic_time_snsec; |
196 | ns += vgetsns(vvar); |
197 | ns >>= vvar->clock.shift; |
198 | } while (unlikely(vvar_read_retry(vvar, seq))); |
199 | |
200 | ts->tv_sec += __iter_div_u64_rem(dividend: ns, NSEC_PER_SEC, remainder: &ns); |
201 | ts->tv_nsec = ns; |
202 | |
203 | return 0; |
204 | } |
205 | |
206 | notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar, |
207 | struct __kernel_old_timespec *ts) |
208 | { |
209 | unsigned long seq; |
210 | u64 ns; |
211 | |
212 | do { |
213 | seq = vvar_read_begin(vvar); |
214 | ts->tv_sec = vvar->monotonic_time_sec; |
215 | ns = vvar->monotonic_time_snsec; |
216 | ns += vgetsns_stick(vvar); |
217 | ns >>= vvar->clock.shift; |
218 | } while (unlikely(vvar_read_retry(vvar, seq))); |
219 | |
220 | ts->tv_sec += __iter_div_u64_rem(dividend: ns, NSEC_PER_SEC, remainder: &ns); |
221 | ts->tv_nsec = ns; |
222 | |
223 | return 0; |
224 | } |
225 | |
226 | notrace static int do_realtime_coarse(struct vvar_data *vvar, |
227 | struct __kernel_old_timespec *ts) |
228 | { |
229 | unsigned long seq; |
230 | |
231 | do { |
232 | seq = vvar_read_begin(vvar); |
233 | ts->tv_sec = vvar->wall_time_coarse_sec; |
234 | ts->tv_nsec = vvar->wall_time_coarse_nsec; |
235 | } while (unlikely(vvar_read_retry(vvar, seq))); |
236 | return 0; |
237 | } |
238 | |
239 | notrace static int do_monotonic_coarse(struct vvar_data *vvar, |
240 | struct __kernel_old_timespec *ts) |
241 | { |
242 | unsigned long seq; |
243 | |
244 | do { |
245 | seq = vvar_read_begin(vvar); |
246 | ts->tv_sec = vvar->monotonic_time_coarse_sec; |
247 | ts->tv_nsec = vvar->monotonic_time_coarse_nsec; |
248 | } while (unlikely(vvar_read_retry(vvar, seq))); |
249 | |
250 | return 0; |
251 | } |
252 | |
253 | notrace int |
254 | __vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts) |
255 | { |
256 | struct vvar_data *vvd = get_vvar_data(); |
257 | |
258 | switch (clock) { |
259 | case CLOCK_REALTIME: |
260 | if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) |
261 | break; |
262 | return do_realtime(vvar: vvd, ts); |
263 | case CLOCK_MONOTONIC: |
264 | if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) |
265 | break; |
266 | return do_monotonic(vvar: vvd, ts); |
267 | case CLOCK_REALTIME_COARSE: |
268 | return do_realtime_coarse(vvar: vvd, ts); |
269 | case CLOCK_MONOTONIC_COARSE: |
270 | return do_monotonic_coarse(vvar: vvd, ts); |
271 | } |
272 | /* |
273 | * Unknown clock ID ? Fall back to the syscall. |
274 | */ |
275 | return vdso_fallback_gettime(clock, ts); |
276 | } |
277 | int |
278 | clock_gettime(clockid_t, struct __kernel_old_timespec *) |
279 | __attribute__((weak, alias("__vdso_clock_gettime" ))); |
280 | |
281 | notrace int |
282 | __vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts) |
283 | { |
284 | struct vvar_data *vvd = get_vvar_data(); |
285 | |
286 | switch (clock) { |
287 | case CLOCK_REALTIME: |
288 | if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) |
289 | break; |
290 | return do_realtime_stick(vvar: vvd, ts); |
291 | case CLOCK_MONOTONIC: |
292 | if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) |
293 | break; |
294 | return do_monotonic_stick(vvar: vvd, ts); |
295 | case CLOCK_REALTIME_COARSE: |
296 | return do_realtime_coarse(vvar: vvd, ts); |
297 | case CLOCK_MONOTONIC_COARSE: |
298 | return do_monotonic_coarse(vvar: vvd, ts); |
299 | } |
300 | /* |
301 | * Unknown clock ID ? Fall back to the syscall. |
302 | */ |
303 | return vdso_fallback_gettime(clock, ts); |
304 | } |
305 | |
306 | notrace int |
307 | __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) |
308 | { |
309 | struct vvar_data *vvd = get_vvar_data(); |
310 | |
311 | if (likely(vvd->vclock_mode != VCLOCK_NONE)) { |
312 | if (likely(tv != NULL)) { |
313 | union tstv_t { |
314 | struct __kernel_old_timespec ts; |
315 | struct __kernel_old_timeval tv; |
316 | } *tstv = (union tstv_t *) tv; |
317 | do_realtime(vvar: vvd, ts: &tstv->ts); |
318 | /* |
319 | * Assign before dividing to ensure that the division is |
320 | * done in the type of tv_usec, not tv_nsec. |
321 | * |
322 | * There cannot be > 1 billion usec in a second: |
323 | * do_realtime() has already distributed such overflow |
324 | * into tv_sec. So we can assign it to an int safely. |
325 | */ |
326 | tstv->tv.tv_usec = tstv->ts.tv_nsec; |
327 | tstv->tv.tv_usec /= 1000; |
328 | } |
329 | if (unlikely(tz != NULL)) { |
330 | /* Avoid memcpy. Some old compilers fail to inline it */ |
331 | tz->tz_minuteswest = vvd->tz_minuteswest; |
332 | tz->tz_dsttime = vvd->tz_dsttime; |
333 | } |
334 | return 0; |
335 | } |
336 | return vdso_fallback_gettimeofday(tv, tz); |
337 | } |
338 | int |
339 | gettimeofday(struct __kernel_old_timeval *, struct timezone *) |
340 | __attribute__((weak, alias("__vdso_gettimeofday" ))); |
341 | |
342 | notrace int |
343 | __vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz) |
344 | { |
345 | struct vvar_data *vvd = get_vvar_data(); |
346 | |
347 | if (likely(vvd->vclock_mode != VCLOCK_NONE)) { |
348 | if (likely(tv != NULL)) { |
349 | union tstv_t { |
350 | struct __kernel_old_timespec ts; |
351 | struct __kernel_old_timeval tv; |
352 | } *tstv = (union tstv_t *) tv; |
353 | do_realtime_stick(vvar: vvd, ts: &tstv->ts); |
354 | /* |
355 | * Assign before dividing to ensure that the division is |
356 | * done in the type of tv_usec, not tv_nsec. |
357 | * |
358 | * There cannot be > 1 billion usec in a second: |
359 | * do_realtime() has already distributed such overflow |
360 | * into tv_sec. So we can assign it to an int safely. |
361 | */ |
362 | tstv->tv.tv_usec = tstv->ts.tv_nsec; |
363 | tstv->tv.tv_usec /= 1000; |
364 | } |
365 | if (unlikely(tz != NULL)) { |
366 | /* Avoid memcpy. Some old compilers fail to inline it */ |
367 | tz->tz_minuteswest = vvd->tz_minuteswest; |
368 | tz->tz_dsttime = vvd->tz_dsttime; |
369 | } |
370 | return 0; |
371 | } |
372 | return vdso_fallback_gettimeofday(tv, tz); |
373 | } |
374 | |