1 | /* SPDX-License-Identifier: GPL-2.0 */ |
---|---|

2 | #ifndef _LINUX_U64_STATS_SYNC_H |

3 | #define _LINUX_U64_STATS_SYNC_H |

4 | |

5 | /* |

6 | * Protect against 64-bit values tearing on 32-bit architectures. This is |

7 | * typically used for statistics read/update in different subsystems. |

8 | * |

9 | * Key points : |

10 | * |

11 | * - Use a seqcount on 32-bit |

12 | * - The whole thing is a no-op on 64-bit architectures. |

13 | * |

14 | * Usage constraints: |

15 | * |

16 | * 1) Write side must ensure mutual exclusion, or one seqcount update could |

17 | * be lost, thus blocking readers forever. |

18 | * |

19 | * 2) Write side must disable preemption, or a seqcount reader can preempt the |

20 | * writer and also spin forever. |

21 | * |

22 | * 3) Write side must use the _irqsave() variant if other writers, or a reader, |

23 | * can be invoked from an IRQ context. On 64bit systems this variant does not |

24 | * disable interrupts. |

25 | * |

26 | * 4) If reader fetches several counters, there is no guarantee the whole values |

27 | * are consistent w.r.t. each other (remember point #2: seqcounts are not |

28 | * used for 64bit architectures). |

29 | * |

30 | * 5) Readers are allowed to sleep or be preempted/interrupted: they perform |

31 | * pure reads. |

32 | * |

33 | * Usage : |

34 | * |

35 | * Stats producer (writer) should use following template granted it already got |

36 | * an exclusive access to counters (a lock is already taken, or per cpu |

37 | * data is used [in a non preemptable context]) |

38 | * |

39 | * spin_lock_bh(...) or other synchronization to get exclusive access |

40 | * ... |

41 | * u64_stats_update_begin(&stats->syncp); |

42 | * u64_stats_add(&stats->bytes64, len); // non atomic operation |

43 | * u64_stats_inc(&stats->packets64); // non atomic operation |

44 | * u64_stats_update_end(&stats->syncp); |

45 | * |

46 | * While a consumer (reader) should use following template to get consistent |

47 | * snapshot for each variable (but no guarantee on several ones) |

48 | * |

49 | * u64 tbytes, tpackets; |

50 | * unsigned int start; |

51 | * |

52 | * do { |

53 | * start = u64_stats_fetch_begin(&stats->syncp); |

54 | * tbytes = u64_stats_read(&stats->bytes64); // non atomic operation |

55 | * tpackets = u64_stats_read(&stats->packets64); // non atomic operation |

56 | * } while (u64_stats_fetch_retry(&stats->syncp, start)); |

57 | * |

58 | * |

59 | * Example of use in drivers/net/loopback.c, using per_cpu containers, |

60 | * in BH disabled context. |

61 | */ |

62 | #include <linux/seqlock.h> |

63 | |

64 | struct u64_stats_sync { |

65 | #if BITS_PER_LONG == 32 |

66 | seqcount_t seq; |

67 | #endif |

68 | }; |

69 | |

70 | #if BITS_PER_LONG == 64 |

71 | #include <asm/local64.h> |

72 | |

73 | typedef struct { |

74 | local64_t v; |

75 | } u64_stats_t ; |

76 | |

77 | static inline u64 u64_stats_read(const u64_stats_t *p) |

78 | { |

79 | return local64_read(&p->v); |

80 | } |

81 | |

82 | static inline void u64_stats_set(u64_stats_t *p, u64 val) |

83 | { |

84 | local64_set(&p->v, val); |

85 | } |

86 | |

87 | static inline void u64_stats_add(u64_stats_t *p, unsigned long val) |

88 | { |

89 | local64_add(val, &p->v); |

90 | } |

91 | |

92 | static inline void u64_stats_inc(u64_stats_t *p) |

93 | { |

94 | local64_inc(&p->v); |

95 | } |

96 | |

97 | static inline void u64_stats_init(struct u64_stats_sync *syncp) { } |

98 | static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) { } |

99 | static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) { } |

100 | static inline unsigned long __u64_stats_irqsave(void) { return 0; } |

101 | static inline void __u64_stats_irqrestore(unsigned long flags) { } |

102 | static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp) |

103 | { |

104 | return 0; |

105 | } |

106 | static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, |

107 | unsigned int start) |

108 | { |

109 | return false; |

110 | } |

111 | |

112 | #else /* 64 bit */ |

113 | |

114 | typedef struct { |

115 | u64 v; |

116 | } u64_stats_t; |

117 | |

118 | static inline u64 u64_stats_read(const u64_stats_t *p) |

119 | { |

120 | return p->v; |

121 | } |

122 | |

123 | static inline void u64_stats_set(u64_stats_t *p, u64 val) |

124 | { |

125 | p->v = val; |

126 | } |

127 | |

128 | static inline void u64_stats_add(u64_stats_t *p, unsigned long val) |

129 | { |

130 | p->v += val; |

131 | } |

132 | |

133 | static inline void u64_stats_inc(u64_stats_t *p) |

134 | { |

135 | p->v++; |

136 | } |

137 | |

138 | #define u64_stats_init(syncp) \ |

139 | do { \ |

140 | struct u64_stats_sync *__s = (syncp); \ |

141 | seqcount_init(&__s->seq); \ |

142 | } while (0) |

143 | |

144 | static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) |

145 | { |

146 | preempt_disable_nested(); |

147 | write_seqcount_begin(&syncp->seq); |

148 | } |

149 | |

150 | static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) |

151 | { |

152 | write_seqcount_end(&syncp->seq); |

153 | preempt_enable_nested(); |

154 | } |

155 | |

156 | static inline unsigned long __u64_stats_irqsave(void) |

157 | { |

158 | unsigned long flags; |

159 | |

160 | local_irq_save(flags); |

161 | return flags; |

162 | } |

163 | |

164 | static inline void __u64_stats_irqrestore(unsigned long flags) |

165 | { |

166 | local_irq_restore(flags); |

167 | } |

168 | |

169 | static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp) |

170 | { |

171 | return read_seqcount_begin(&syncp->seq); |

172 | } |

173 | |

174 | static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, |

175 | unsigned int start) |

176 | { |

177 | return read_seqcount_retry(&syncp->seq, start); |

178 | } |

179 | #endif /* !64 bit */ |

180 | |

181 | static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) |

182 | { |

183 | __u64_stats_update_begin(syncp); |

184 | } |

185 | |

186 | static inline void u64_stats_update_end(struct u64_stats_sync *syncp) |

187 | { |

188 | __u64_stats_update_end(syncp); |

189 | } |

190 | |

191 | static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp) |

192 | { |

193 | unsigned long flags = __u64_stats_irqsave(); |

194 | |

195 | __u64_stats_update_begin(syncp); |

196 | return flags; |

197 | } |

198 | |

199 | static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp, |

200 | unsigned long flags) |

201 | { |

202 | __u64_stats_update_end(syncp); |

203 | __u64_stats_irqrestore(flags); |

204 | } |

205 | |

206 | static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp) |

207 | { |

208 | return __u64_stats_fetch_begin(syncp); |

209 | } |

210 | |

211 | static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, |

212 | unsigned int start) |

213 | { |

214 | return __u64_stats_fetch_retry(syncp, start); |

215 | } |

216 | |

217 | #endif /* _LINUX_U64_STATS_SYNC_H */ |

218 |