1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_PERCPU_COUNTER_H |
3 | #define _LINUX_PERCPU_COUNTER_H |
4 | /* |
5 | * A simple "approximate counter" for use in ext2 and ext3 superblocks. |
6 | * |
7 | * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4. |
8 | */ |
9 | |
10 | #include <linux/spinlock.h> |
11 | #include <linux/smp.h> |
12 | #include <linux/list.h> |
13 | #include <linux/threads.h> |
14 | #include <linux/percpu.h> |
15 | #include <linux/types.h> |
16 | |
17 | /* percpu_counter batch for local add or sub */ |
18 | #define PERCPU_COUNTER_LOCAL_BATCH INT_MAX |
19 | |
20 | #ifdef CONFIG_SMP |
21 | |
22 | struct percpu_counter { |
23 | raw_spinlock_t lock; |
24 | s64 count; |
25 | #ifdef CONFIG_HOTPLUG_CPU |
26 | struct list_head list; /* All percpu_counters are on a list */ |
27 | #endif |
28 | s32 __percpu *counters; |
29 | }; |
30 | |
31 | extern int percpu_counter_batch; |
32 | |
33 | int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp, |
34 | struct lock_class_key *key); |
35 | |
36 | #define percpu_counter_init(fbc, value, gfp) \ |
37 | ({ \ |
38 | static struct lock_class_key __key; \ |
39 | \ |
40 | __percpu_counter_init(fbc, value, gfp, &__key); \ |
41 | }) |
42 | |
43 | void percpu_counter_destroy(struct percpu_counter *fbc); |
44 | void percpu_counter_set(struct percpu_counter *fbc, s64 amount); |
45 | void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, |
46 | s32 batch); |
47 | s64 __percpu_counter_sum(struct percpu_counter *fbc); |
48 | int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch); |
49 | void percpu_counter_sync(struct percpu_counter *fbc); |
50 | |
51 | static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs) |
52 | { |
53 | return __percpu_counter_compare(fbc, rhs, percpu_counter_batch); |
54 | } |
55 | |
56 | static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount) |
57 | { |
58 | percpu_counter_add_batch(fbc, amount, percpu_counter_batch); |
59 | } |
60 | |
61 | /* |
62 | * With percpu_counter_add_local() and percpu_counter_sub_local(), counts |
63 | * are accumulated in local per cpu counter and not in fbc->count until |
64 | * local count overflows PERCPU_COUNTER_LOCAL_BATCH. This makes counter |
65 | * write efficient. |
66 | * But percpu_counter_sum(), instead of percpu_counter_read(), needs to be |
67 | * used to add up the counts from each CPU to account for all the local |
68 | * counts. So percpu_counter_add_local() and percpu_counter_sub_local() |
69 | * should be used when a counter is updated frequently and read rarely. |
70 | */ |
71 | static inline void |
72 | percpu_counter_add_local(struct percpu_counter *fbc, s64 amount) |
73 | { |
74 | percpu_counter_add_batch(fbc, amount, PERCPU_COUNTER_LOCAL_BATCH); |
75 | } |
76 | |
77 | static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) |
78 | { |
79 | s64 ret = __percpu_counter_sum(fbc); |
80 | return ret < 0 ? 0 : ret; |
81 | } |
82 | |
83 | static inline s64 percpu_counter_sum(struct percpu_counter *fbc) |
84 | { |
85 | return __percpu_counter_sum(fbc); |
86 | } |
87 | |
88 | static inline s64 percpu_counter_read(struct percpu_counter *fbc) |
89 | { |
90 | return fbc->count; |
91 | } |
92 | |
93 | /* |
94 | * It is possible for the percpu_counter_read() to return a small negative |
95 | * number for some counter which should never be negative. |
96 | * |
97 | */ |
98 | static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) |
99 | { |
100 | /* Prevent reloads of fbc->count */ |
101 | s64 ret = READ_ONCE(fbc->count); |
102 | |
103 | if (ret >= 0) |
104 | return ret; |
105 | return 0; |
106 | } |
107 | |
108 | static inline bool percpu_counter_initialized(struct percpu_counter *fbc) |
109 | { |
110 | return (fbc->counters != NULL); |
111 | } |
112 | |
113 | #else /* !CONFIG_SMP */ |
114 | |
115 | struct percpu_counter { |
116 | s64 count; |
117 | }; |
118 | |
119 | static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount, |
120 | gfp_t gfp) |
121 | { |
122 | fbc->count = amount; |
123 | return 0; |
124 | } |
125 | |
126 | static inline void percpu_counter_destroy(struct percpu_counter *fbc) |
127 | { |
128 | } |
129 | |
130 | static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount) |
131 | { |
132 | fbc->count = amount; |
133 | } |
134 | |
135 | static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs) |
136 | { |
137 | if (fbc->count > rhs) |
138 | return 1; |
139 | else if (fbc->count < rhs) |
140 | return -1; |
141 | else |
142 | return 0; |
143 | } |
144 | |
145 | static inline int |
146 | __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch) |
147 | { |
148 | return percpu_counter_compare(fbc, rhs); |
149 | } |
150 | |
151 | static inline void |
152 | percpu_counter_add(struct percpu_counter *fbc, s64 amount) |
153 | { |
154 | unsigned long flags; |
155 | |
156 | local_irq_save(flags); |
157 | fbc->count += amount; |
158 | local_irq_restore(flags); |
159 | } |
160 | |
161 | /* non-SMP percpu_counter_add_local is the same with percpu_counter_add */ |
162 | static inline void |
163 | percpu_counter_add_local(struct percpu_counter *fbc, s64 amount) |
164 | { |
165 | percpu_counter_add(fbc, amount); |
166 | } |
167 | |
168 | static inline void |
169 | percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) |
170 | { |
171 | percpu_counter_add(fbc, amount); |
172 | } |
173 | |
174 | static inline s64 percpu_counter_read(struct percpu_counter *fbc) |
175 | { |
176 | return fbc->count; |
177 | } |
178 | |
179 | /* |
180 | * percpu_counter is intended to track positive numbers. In the UP case the |
181 | * number should never be negative. |
182 | */ |
183 | static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) |
184 | { |
185 | return fbc->count; |
186 | } |
187 | |
188 | static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) |
189 | { |
190 | return percpu_counter_read_positive(fbc); |
191 | } |
192 | |
193 | static inline s64 percpu_counter_sum(struct percpu_counter *fbc) |
194 | { |
195 | return percpu_counter_read(fbc); |
196 | } |
197 | |
198 | static inline bool percpu_counter_initialized(struct percpu_counter *fbc) |
199 | { |
200 | return true; |
201 | } |
202 | |
203 | static inline void percpu_counter_sync(struct percpu_counter *fbc) |
204 | { |
205 | } |
206 | #endif /* CONFIG_SMP */ |
207 | |
208 | static inline void percpu_counter_inc(struct percpu_counter *fbc) |
209 | { |
210 | percpu_counter_add(fbc, 1); |
211 | } |
212 | |
213 | static inline void percpu_counter_dec(struct percpu_counter *fbc) |
214 | { |
215 | percpu_counter_add(fbc, -1); |
216 | } |
217 | |
218 | static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount) |
219 | { |
220 | percpu_counter_add(fbc, -amount); |
221 | } |
222 | |
223 | static inline void |
224 | percpu_counter_sub_local(struct percpu_counter *fbc, s64 amount) |
225 | { |
226 | percpu_counter_add_local(fbc, -amount); |
227 | } |
228 | |
229 | #endif /* _LINUX_PERCPU_COUNTER_H */ |
230 | |