1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __VDSO_MATH64_H
3#define __VDSO_MATH64_H
4
5static __always_inline u32
6__iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
7{
8 u32 ret = 0;
9
10 while (dividend >= divisor) {
11 /* The following asm() prevents the compiler from
12 optimising this loop into a modulo operation. */
13 asm("" : "+rm"(dividend));
14
15 dividend -= divisor;
16 ret++;
17 }
18
19 *remainder = dividend;
20
21 return ret;
22}
23
24#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
25
26#ifndef mul_u64_u32_add_u64_shr
27static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
28{
29 return (u64)((((unsigned __int128)a * mul) + b) >> shift);
30}
31#endif /* mul_u64_u32_add_u64_shr */
32
33#else
34
35#ifndef mul_u64_u32_add_u64_shr
36#ifndef mul_u32_u32
37static inline u64 mul_u32_u32(u32 a, u32 b)
38{
39 return (u64)a * b;
40}
41#define mul_u32_u32 mul_u32_u32
42#endif
43static __always_inline u64 mul_u64_u32_add_u64_shr(u64 a, u32 mul, u64 b, unsigned int shift)
44{
45 u32 ah = a >> 32, al = a;
46 bool ovf;
47 u64 ret;
48
49 ovf = __builtin_add_overflow(mul_u32_u32(al, mul), b, &ret);
50 ret >>= shift;
51 if (ovf && shift)
52 ret += 1ULL << (64 - shift);
53 if (ah)
54 ret += mul_u32_u32(ah, mul) << (32 - shift);
55
56 return ret;
57}
58#endif /* mul_u64_u32_add_u64_shr */
59
60#endif
61
62#endif /* __VDSO_MATH64_H */
63