1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | |
3 | /* |
4 | * This file provides wrappers with sanitizer instrumentation for non-atomic |
5 | * bit operations. |
6 | * |
7 | * To use this functionality, an arch's bitops.h file needs to define each of |
8 | * the below bit operations with an arch_ prefix (e.g. arch_set_bit(), |
9 | * arch___set_bit(), etc.). |
10 | */ |
11 | #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H |
12 | #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H |
13 | |
14 | #include <linux/instrumented.h> |
15 | |
16 | /** |
17 | * ___set_bit - Set a bit in memory |
18 | * @nr: the bit to set |
19 | * @addr: the address to start counting from |
20 | * |
21 | * Unlike set_bit(), this function is non-atomic. If it is called on the same |
22 | * region of memory concurrently, the effect may be that only one operation |
23 | * succeeds. |
24 | */ |
25 | static __always_inline void |
26 | ___set_bit(unsigned long nr, volatile unsigned long *addr) |
27 | { |
28 | instrument_write(v: addr + BIT_WORD(nr), size: sizeof(long)); |
29 | arch___set_bit(nr, addr); |
30 | } |
31 | |
32 | /** |
33 | * ___clear_bit - Clears a bit in memory |
34 | * @nr: the bit to clear |
35 | * @addr: the address to start counting from |
36 | * |
37 | * Unlike clear_bit(), this function is non-atomic. If it is called on the same |
38 | * region of memory concurrently, the effect may be that only one operation |
39 | * succeeds. |
40 | */ |
41 | static __always_inline void |
42 | ___clear_bit(unsigned long nr, volatile unsigned long *addr) |
43 | { |
44 | instrument_write(v: addr + BIT_WORD(nr), size: sizeof(long)); |
45 | arch___clear_bit(nr, addr); |
46 | } |
47 | |
48 | /** |
49 | * ___change_bit - Toggle a bit in memory |
50 | * @nr: the bit to change |
51 | * @addr: the address to start counting from |
52 | * |
53 | * Unlike change_bit(), this function is non-atomic. If it is called on the same |
54 | * region of memory concurrently, the effect may be that only one operation |
55 | * succeeds. |
56 | */ |
57 | static __always_inline void |
58 | ___change_bit(unsigned long nr, volatile unsigned long *addr) |
59 | { |
60 | instrument_write(v: addr + BIT_WORD(nr), size: sizeof(long)); |
61 | arch___change_bit(nr, addr); |
62 | } |
63 | |
64 | static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr) |
65 | { |
66 | if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) { |
67 | /* |
68 | * We treat non-atomic read-write bitops a little more special. |
69 | * Given the operations here only modify a single bit, assuming |
70 | * non-atomicity of the writer is sufficient may be reasonable |
71 | * for certain usage (and follows the permissible nature of the |
72 | * assume-plain-writes-atomic rule): |
73 | * 1. report read-modify-write races -> check read; |
74 | * 2. do not report races with marked readers, but do report |
75 | * races with unmarked readers -> check "atomic" write. |
76 | */ |
77 | kcsan_check_read(addr + BIT_WORD(nr), sizeof(long)); |
78 | /* |
79 | * Use generic write instrumentation, in case other sanitizers |
80 | * or tools are enabled alongside KCSAN. |
81 | */ |
82 | instrument_write(v: addr + BIT_WORD(nr), size: sizeof(long)); |
83 | } else { |
84 | instrument_read_write(v: addr + BIT_WORD(nr), size: sizeof(long)); |
85 | } |
86 | } |
87 | |
88 | /** |
89 | * ___test_and_set_bit - Set a bit and return its old value |
90 | * @nr: Bit to set |
91 | * @addr: Address to count from |
92 | * |
93 | * This operation is non-atomic. If two instances of this operation race, one |
94 | * can appear to succeed but actually fail. |
95 | */ |
96 | static __always_inline bool |
97 | ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) |
98 | { |
99 | __instrument_read_write_bitop(nr, addr); |
100 | return arch___test_and_set_bit(nr, addr); |
101 | } |
102 | |
103 | /** |
104 | * ___test_and_clear_bit - Clear a bit and return its old value |
105 | * @nr: Bit to clear |
106 | * @addr: Address to count from |
107 | * |
108 | * This operation is non-atomic. If two instances of this operation race, one |
109 | * can appear to succeed but actually fail. |
110 | */ |
111 | static __always_inline bool |
112 | ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) |
113 | { |
114 | __instrument_read_write_bitop(nr, addr); |
115 | return arch___test_and_clear_bit(nr, addr); |
116 | } |
117 | |
118 | /** |
119 | * ___test_and_change_bit - Change a bit and return its old value |
120 | * @nr: Bit to change |
121 | * @addr: Address to count from |
122 | * |
123 | * This operation is non-atomic. If two instances of this operation race, one |
124 | * can appear to succeed but actually fail. |
125 | */ |
126 | static __always_inline bool |
127 | ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) |
128 | { |
129 | __instrument_read_write_bitop(nr, addr); |
130 | return arch___test_and_change_bit(nr, addr); |
131 | } |
132 | |
133 | /** |
134 | * _test_bit - Determine whether a bit is set |
135 | * @nr: bit number to test |
136 | * @addr: Address to start counting from |
137 | */ |
138 | static __always_inline bool |
139 | _test_bit(unsigned long nr, const volatile unsigned long *addr) |
140 | { |
141 | instrument_atomic_read(v: addr + BIT_WORD(nr), size: sizeof(long)); |
142 | return arch_test_bit(nr, addr); |
143 | } |
144 | |
145 | /** |
146 | * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set |
147 | * @nr: bit number to test |
148 | * @addr: Address to start counting from |
149 | */ |
150 | static __always_inline bool |
151 | _test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) |
152 | { |
153 | instrument_atomic_read(v: addr + BIT_WORD(nr), size: sizeof(long)); |
154 | return arch_test_bit_acquire(nr, addr); |
155 | } |
156 | |
157 | #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ |
158 | |