1 | /* SPDX-License-Identifier: GPL-2.0 */ |
---|---|
2 | #ifndef __VDSO_HELPERS_H |
3 | #define __VDSO_HELPERS_H |
4 | |
5 | #ifndef __ASSEMBLY__ |
6 | |
7 | #include <vdso/datapage.h> |
8 | |
9 | static __always_inline u32 vdso_read_begin(const struct vdso_data *vd) |
10 | { |
11 | u32 seq; |
12 | |
13 | while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) |
14 | cpu_relax(); |
15 | |
16 | smp_rmb(); |
17 | return seq; |
18 | } |
19 | |
20 | static __always_inline u32 vdso_read_retry(const struct vdso_data *vd, |
21 | u32 start) |
22 | { |
23 | u32 seq; |
24 | |
25 | smp_rmb(); |
26 | seq = READ_ONCE(vd->seq); |
27 | return seq != start; |
28 | } |
29 | |
30 | static __always_inline void vdso_write_begin(struct vdso_data *vd) |
31 | { |
32 | /* |
33 | * WRITE_ONCE it is required otherwise the compiler can validly tear |
34 | * updates to vd[x].seq and it is possible that the value seen by the |
35 | * reader it is inconsistent. |
36 | */ |
37 | WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); |
38 | WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); |
39 | smp_wmb(); |
40 | } |
41 | |
42 | static __always_inline void vdso_write_end(struct vdso_data *vd) |
43 | { |
44 | smp_wmb(); |
45 | /* |
46 | * WRITE_ONCE it is required otherwise the compiler can validly tear |
47 | * updates to vd[x].seq and it is possible that the value seen by the |
48 | * reader it is inconsistent. |
49 | */ |
50 | WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); |
51 | WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); |
52 | } |
53 | |
54 | #endif /* !__ASSEMBLY__ */ |
55 | |
56 | #endif /* __VDSO_HELPERS_H */ |
57 |