1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_MATH64_H |
3 | #define _LINUX_MATH64_H |
4 | |
5 | #include <linux/types.h> |
6 | #include <linux/math.h> |
7 | #include <vdso/math64.h> |
8 | #include <asm/div64.h> |
9 | |
10 | #if BITS_PER_LONG == 64 |
11 | |
12 | #define div64_long(x, y) div64_s64((x), (y)) |
13 | #define div64_ul(x, y) div64_u64((x), (y)) |
14 | |
15 | /** |
16 | * div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder |
17 | * @dividend: unsigned 64bit dividend |
18 | * @divisor: unsigned 32bit divisor |
19 | * @remainder: pointer to unsigned 32bit remainder |
20 | * |
21 | * Return: sets ``*remainder``, then returns dividend / divisor |
22 | * |
23 | * This is commonly provided by 32bit archs to provide an optimized 64bit |
24 | * divide. |
25 | */ |
26 | static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) |
27 | { |
28 | *remainder = dividend % divisor; |
29 | return dividend / divisor; |
30 | } |
31 | |
32 | /** |
33 | * div_s64_rem - signed 64bit divide with 32bit divisor with remainder |
34 | * @dividend: signed 64bit dividend |
35 | * @divisor: signed 32bit divisor |
36 | * @remainder: pointer to signed 32bit remainder |
37 | * |
38 | * Return: sets ``*remainder``, then returns dividend / divisor |
39 | */ |
40 | static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) |
41 | { |
42 | *remainder = dividend % divisor; |
43 | return dividend / divisor; |
44 | } |
45 | |
46 | /** |
47 | * div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder |
48 | * @dividend: unsigned 64bit dividend |
49 | * @divisor: unsigned 64bit divisor |
50 | * @remainder: pointer to unsigned 64bit remainder |
51 | * |
52 | * Return: sets ``*remainder``, then returns dividend / divisor |
53 | */ |
54 | static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder) |
55 | { |
56 | *remainder = dividend % divisor; |
57 | return dividend / divisor; |
58 | } |
59 | |
60 | /** |
61 | * div64_u64 - unsigned 64bit divide with 64bit divisor |
62 | * @dividend: unsigned 64bit dividend |
63 | * @divisor: unsigned 64bit divisor |
64 | * |
65 | * Return: dividend / divisor |
66 | */ |
67 | static inline u64 div64_u64(u64 dividend, u64 divisor) |
68 | { |
69 | return dividend / divisor; |
70 | } |
71 | |
72 | /** |
73 | * div64_s64 - signed 64bit divide with 64bit divisor |
74 | * @dividend: signed 64bit dividend |
75 | * @divisor: signed 64bit divisor |
76 | * |
77 | * Return: dividend / divisor |
78 | */ |
79 | static inline s64 div64_s64(s64 dividend, s64 divisor) |
80 | { |
81 | return dividend / divisor; |
82 | } |
83 | |
84 | #elif BITS_PER_LONG == 32 |
85 | |
86 | #define div64_long(x, y) div_s64((x), (y)) |
87 | #define div64_ul(x, y) div_u64((x), (y)) |
88 | |
89 | #ifndef div_u64_rem |
90 | static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) |
91 | { |
92 | *remainder = do_div(dividend, divisor); |
93 | return dividend; |
94 | } |
95 | #endif |
96 | |
97 | #ifndef div_s64_rem |
98 | extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder); |
99 | #endif |
100 | |
101 | #ifndef div64_u64_rem |
102 | extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder); |
103 | #endif |
104 | |
105 | #ifndef div64_u64 |
106 | extern u64 div64_u64(u64 dividend, u64 divisor); |
107 | #endif |
108 | |
109 | #ifndef div64_s64 |
110 | extern s64 div64_s64(s64 dividend, s64 divisor); |
111 | #endif |
112 | |
113 | #endif /* BITS_PER_LONG */ |
114 | |
115 | /** |
116 | * div_u64 - unsigned 64bit divide with 32bit divisor |
117 | * @dividend: unsigned 64bit dividend |
118 | * @divisor: unsigned 32bit divisor |
119 | * |
120 | * This is the most common 64bit divide and should be used if possible, |
121 | * as many 32bit archs can optimize this variant better than a full 64bit |
122 | * divide. |
123 | * |
124 | * Return: dividend / divisor |
125 | */ |
126 | #ifndef div_u64 |
127 | static inline u64 div_u64(u64 dividend, u32 divisor) |
128 | { |
129 | u32 remainder; |
130 | return div_u64_rem(dividend, divisor, remainder: &remainder); |
131 | } |
132 | #endif |
133 | |
134 | /** |
135 | * div_s64 - signed 64bit divide with 32bit divisor |
136 | * @dividend: signed 64bit dividend |
137 | * @divisor: signed 32bit divisor |
138 | * |
139 | * Return: dividend / divisor |
140 | */ |
141 | #ifndef div_s64 |
142 | static inline s64 div_s64(s64 dividend, s32 divisor) |
143 | { |
144 | s32 remainder; |
145 | return div_s64_rem(dividend, divisor, remainder: &remainder); |
146 | } |
147 | #endif |
148 | |
149 | u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder); |
150 | |
151 | #ifndef mul_u32_u32 |
152 | /* |
153 | * Many a GCC version messes this up and generates a 64x64 mult :-( |
154 | */ |
155 | static inline u64 mul_u32_u32(u32 a, u32 b) |
156 | { |
157 | return (u64)a * b; |
158 | } |
159 | #endif |
160 | |
161 | #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) |
162 | |
163 | #ifndef mul_u64_u32_shr |
164 | static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) |
165 | { |
166 | return (u64)(((unsigned __int128)a * mul) >> shift); |
167 | } |
168 | #endif /* mul_u64_u32_shr */ |
169 | |
170 | #ifndef mul_u64_u64_shr |
171 | static __always_inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift) |
172 | { |
173 | return (u64)(((unsigned __int128)a * mul) >> shift); |
174 | } |
175 | #endif /* mul_u64_u64_shr */ |
176 | |
177 | #else |
178 | |
179 | #ifndef mul_u64_u32_shr |
180 | static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) |
181 | { |
182 | u32 ah, al; |
183 | u64 ret; |
184 | |
185 | al = a; |
186 | ah = a >> 32; |
187 | |
188 | ret = mul_u32_u32(al, mul) >> shift; |
189 | if (ah) |
190 | ret += mul_u32_u32(ah, mul) << (32 - shift); |
191 | |
192 | return ret; |
193 | } |
194 | #endif /* mul_u64_u32_shr */ |
195 | |
196 | #ifndef mul_u64_u64_shr |
197 | static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift) |
198 | { |
199 | union { |
200 | u64 ll; |
201 | struct { |
202 | #ifdef __BIG_ENDIAN |
203 | u32 high, low; |
204 | #else |
205 | u32 low, high; |
206 | #endif |
207 | } l; |
208 | } rl, rm, rn, rh, a0, b0; |
209 | u64 c; |
210 | |
211 | a0.ll = a; |
212 | b0.ll = b; |
213 | |
214 | rl.ll = mul_u32_u32(a0.l.low, b0.l.low); |
215 | rm.ll = mul_u32_u32(a0.l.low, b0.l.high); |
216 | rn.ll = mul_u32_u32(a0.l.high, b0.l.low); |
217 | rh.ll = mul_u32_u32(a0.l.high, b0.l.high); |
218 | |
219 | /* |
220 | * Each of these lines computes a 64-bit intermediate result into "c", |
221 | * starting at bits 32-95. The low 32-bits go into the result of the |
222 | * multiplication, the high 32-bits are carried into the next step. |
223 | */ |
224 | rl.l.high = c = (u64)rl.l.high + rm.l.low + rn.l.low; |
225 | rh.l.low = c = (c >> 32) + rm.l.high + rn.l.high + rh.l.low; |
226 | rh.l.high = (c >> 32) + rh.l.high; |
227 | |
228 | /* |
229 | * The 128-bit result of the multiplication is in rl.ll and rh.ll, |
230 | * shift it right and throw away the high part of the result. |
231 | */ |
232 | if (shift == 0) |
233 | return rl.ll; |
234 | if (shift < 64) |
235 | return (rl.ll >> shift) | (rh.ll << (64 - shift)); |
236 | return rh.ll >> (shift & 63); |
237 | } |
238 | #endif /* mul_u64_u64_shr */ |
239 | |
240 | #endif |
241 | |
242 | #ifndef mul_s64_u64_shr |
243 | static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift) |
244 | { |
245 | u64 ret; |
246 | |
247 | /* |
248 | * Extract the sign before the multiplication and put it back |
249 | * afterwards if needed. |
250 | */ |
251 | ret = mul_u64_u64_shr(abs(a), mul: b, shift); |
252 | |
253 | if (a < 0) |
254 | ret = -((s64) ret); |
255 | |
256 | return ret; |
257 | } |
258 | #endif /* mul_s64_u64_shr */ |
259 | |
260 | #ifndef mul_u64_u32_div |
261 | static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor) |
262 | { |
263 | union { |
264 | u64 ll; |
265 | struct { |
266 | #ifdef __BIG_ENDIAN |
267 | u32 high, low; |
268 | #else |
269 | u32 low, high; |
270 | #endif |
271 | } l; |
272 | } u, rl, rh; |
273 | |
274 | u.ll = a; |
275 | rl.ll = mul_u32_u32(u.l.low, mul); |
276 | rh.ll = mul_u32_u32(u.l.high, mul) + rl.l.high; |
277 | |
278 | /* Bits 32-63 of the result will be in rh.l.low. */ |
279 | rl.l.high = do_div(rh.ll, divisor); |
280 | |
281 | /* Bits 0-31 of the result will be in rl.l.low. */ |
282 | do_div(rl.ll, divisor); |
283 | |
284 | rl.l.high = rh.l.low; |
285 | return rl.ll; |
286 | } |
287 | #endif /* mul_u64_u32_div */ |
288 | |
289 | u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div); |
290 | |
291 | /** |
292 | * DIV64_U64_ROUND_UP - unsigned 64bit divide with 64bit divisor rounded up |
293 | * @ll: unsigned 64bit dividend |
294 | * @d: unsigned 64bit divisor |
295 | * |
296 | * Divide unsigned 64bit dividend by unsigned 64bit divisor |
297 | * and round up. |
298 | * |
299 | * Return: dividend / divisor rounded up |
300 | */ |
301 | #define DIV64_U64_ROUND_UP(ll, d) \ |
302 | ({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); }) |
303 | |
304 | /** |
305 | * DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer |
306 | * @dividend: unsigned 64bit dividend |
307 | * @divisor: unsigned 64bit divisor |
308 | * |
309 | * Divide unsigned 64bit dividend by unsigned 64bit divisor |
310 | * and round to closest integer. |
311 | * |
312 | * Return: dividend / divisor rounded to nearest integer |
313 | */ |
314 | #define DIV64_U64_ROUND_CLOSEST(dividend, divisor) \ |
315 | ({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); }) |
316 | |
317 | /** |
318 | * DIV_U64_ROUND_CLOSEST - unsigned 64bit divide with 32bit divisor rounded to nearest integer |
319 | * @dividend: unsigned 64bit dividend |
320 | * @divisor: unsigned 32bit divisor |
321 | * |
322 | * Divide unsigned 64bit dividend by unsigned 32bit divisor |
323 | * and round to closest integer. |
324 | * |
325 | * Return: dividend / divisor rounded to nearest integer |
326 | */ |
327 | #define DIV_U64_ROUND_CLOSEST(dividend, divisor) \ |
328 | ({ u32 _tmp = (divisor); div_u64((u64)(dividend) + _tmp / 2, _tmp); }) |
329 | |
330 | /** |
331 | * DIV_S64_ROUND_CLOSEST - signed 64bit divide with 32bit divisor rounded to nearest integer |
332 | * @dividend: signed 64bit dividend |
333 | * @divisor: signed 32bit divisor |
334 | * |
335 | * Divide signed 64bit dividend by signed 32bit divisor |
336 | * and round to closest integer. |
337 | * |
338 | * Return: dividend / divisor rounded to nearest integer |
339 | */ |
340 | #define DIV_S64_ROUND_CLOSEST(dividend, divisor)( \ |
341 | { \ |
342 | s64 __x = (dividend); \ |
343 | s32 __d = (divisor); \ |
344 | ((__x > 0) == (__d > 0)) ? \ |
345 | div_s64((__x + (__d / 2)), __d) : \ |
346 | div_s64((__x - (__d / 2)), __d); \ |
347 | } \ |
348 | ) |
349 | #endif /* _LINUX_MATH64_H */ |
350 | |