1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * net/sched/gen_estimator.c Simple rate estimator.
4 *
5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6 * Eric Dumazet <edumazet@google.com>
7 *
8 * Changes:
9 * Jamal Hadi Salim - moved it to net/core and reshulfed
10 * names to make it usable in general net subsystem.
11 */
12
13#include <linux/uaccess.h>
14#include <linux/bitops.h>
15#include <linux/module.h>
16#include <linux/types.h>
17#include <linux/kernel.h>
18#include <linux/jiffies.h>
19#include <linux/string.h>
20#include <linux/mm.h>
21#include <linux/socket.h>
22#include <linux/sockios.h>
23#include <linux/in.h>
24#include <linux/errno.h>
25#include <linux/interrupt.h>
26#include <linux/netdevice.h>
27#include <linux/skbuff.h>
28#include <linux/rtnetlink.h>
29#include <linux/init.h>
30#include <linux/slab.h>
31#include <linux/seqlock.h>
32#include <net/sock.h>
33#include <net/gen_stats.h>
34
35/* This code is NOT intended to be used for statistics collection,
36 * its purpose is to provide a base for statistical multiplexing
37 * for controlled load service.
38 * If you need only statistics, run a user level daemon which
39 * periodically reads byte counters.
40 */
41
42struct net_rate_estimator {
43 struct gnet_stats_basic_sync *bstats;
44 spinlock_t *stats_lock;
45 bool running;
46 struct gnet_stats_basic_sync __percpu *cpu_bstats;
47 u8 ewma_log;
48 u8 intvl_log; /* period : (250ms << intvl_log) */
49
50 seqcount_t seq;
51 u64 last_packets;
52 u64 last_bytes;
53
54 u64 avpps;
55 u64 avbps;
56
57 unsigned long next_jiffies;
58 struct timer_list timer;
59 struct rcu_head rcu;
60};
61
62static void est_fetch_counters(struct net_rate_estimator *e,
63 struct gnet_stats_basic_sync *b)
64{
65 gnet_stats_basic_sync_init(b);
66 if (e->stats_lock)
67 spin_lock(lock: e->stats_lock);
68
69 gnet_stats_add_basic(bstats: b, cpu: e->cpu_bstats, b: e->bstats, running: e->running);
70
71 if (e->stats_lock)
72 spin_unlock(lock: e->stats_lock);
73
74}
75
76static void est_timer(struct timer_list *t)
77{
78 struct net_rate_estimator *est = timer_container_of(est, t, timer);
79 struct gnet_stats_basic_sync b;
80 u64 b_bytes, b_packets;
81 u64 rate, brate;
82
83 est_fetch_counters(e: est, b: &b);
84 b_bytes = u64_stats_read(p: &b.bytes);
85 b_packets = u64_stats_read(p: &b.packets);
86
87 brate = (b_bytes - est->last_bytes) << (10 - est->intvl_log);
88 brate = (brate >> est->ewma_log) - (est->avbps >> est->ewma_log);
89
90 rate = (b_packets - est->last_packets) << (10 - est->intvl_log);
91 rate = (rate >> est->ewma_log) - (est->avpps >> est->ewma_log);
92
93 preempt_disable_nested();
94 write_seqcount_begin(&est->seq);
95 est->avbps += brate;
96 est->avpps += rate;
97 write_seqcount_end(&est->seq);
98 preempt_enable_nested();
99
100 est->last_bytes = b_bytes;
101 est->last_packets = b_packets;
102
103 est->next_jiffies += ((HZ/4) << est->intvl_log);
104
105 if (unlikely(time_after_eq(jiffies, est->next_jiffies))) {
106 /* Ouch... timer was delayed. */
107 est->next_jiffies = jiffies + 1;
108 }
109 mod_timer(timer: &est->timer, expires: est->next_jiffies);
110}
111
112/**
113 * gen_new_estimator - create a new rate estimator
114 * @bstats: basic statistics
115 * @cpu_bstats: bstats per cpu
116 * @rate_est: rate estimator statistics
117 * @lock: lock for statistics and control path
118 * @running: true if @bstats represents a running qdisc, thus @bstats'
119 * internal values might change during basic reads. Only used
120 * if @bstats_cpu is NULL
121 * @opt: rate estimator configuration TLV
122 *
123 * Creates a new rate estimator with &bstats as source and &rate_est
124 * as destination. A new timer with the interval specified in the
125 * configuration TLV is created. Upon each interval, the latest statistics
126 * will be read from &bstats and the estimated rate will be stored in
127 * &rate_est with the statistics lock grabbed during this period.
128 *
129 * Returns 0 on success or a negative error code.
130 *
131 */
132int gen_new_estimator(struct gnet_stats_basic_sync *bstats,
133 struct gnet_stats_basic_sync __percpu *cpu_bstats,
134 struct net_rate_estimator __rcu **rate_est,
135 spinlock_t *lock,
136 bool running,
137 struct nlattr *opt)
138{
139 struct gnet_estimator *parm = nla_data(nla: opt);
140 struct net_rate_estimator *old, *est;
141 struct gnet_stats_basic_sync b;
142 int intvl_log;
143
144 if (nla_len(nla: opt) < sizeof(*parm))
145 return -EINVAL;
146
147 /* allowed timer periods are :
148 * -2 : 250ms, -1 : 500ms, 0 : 1 sec
149 * 1 : 2 sec, 2 : 4 sec, 3 : 8 sec
150 */
151 if (parm->interval < -2 || parm->interval > 3)
152 return -EINVAL;
153
154 if (parm->ewma_log == 0 || parm->ewma_log >= 31)
155 return -EINVAL;
156
157 est = kzalloc(sizeof(*est), GFP_KERNEL);
158 if (!est)
159 return -ENOBUFS;
160
161 seqcount_init(&est->seq);
162 intvl_log = parm->interval + 2;
163 est->bstats = bstats;
164 est->stats_lock = lock;
165 est->running = running;
166 est->ewma_log = parm->ewma_log;
167 est->intvl_log = intvl_log;
168 est->cpu_bstats = cpu_bstats;
169
170 if (lock)
171 local_bh_disable();
172 est_fetch_counters(e: est, b: &b);
173 if (lock)
174 local_bh_enable();
175 est->last_bytes = u64_stats_read(p: &b.bytes);
176 est->last_packets = u64_stats_read(p: &b.packets);
177
178 if (lock)
179 spin_lock_bh(lock);
180 old = rcu_dereference_protected(*rate_est, 1);
181 if (old) {
182 timer_delete_sync(timer: &old->timer);
183 est->avbps = old->avbps;
184 est->avpps = old->avpps;
185 }
186
187 est->next_jiffies = jiffies + ((HZ/4) << intvl_log);
188 timer_setup(&est->timer, est_timer, 0);
189 mod_timer(timer: &est->timer, expires: est->next_jiffies);
190
191 rcu_assign_pointer(*rate_est, est);
192 if (lock)
193 spin_unlock_bh(lock);
194 if (old)
195 kfree_rcu(old, rcu);
196 return 0;
197}
198EXPORT_SYMBOL(gen_new_estimator);
199
200/**
201 * gen_kill_estimator - remove a rate estimator
202 * @rate_est: rate estimator
203 *
204 * Removes the rate estimator.
205 *
206 */
207void gen_kill_estimator(struct net_rate_estimator __rcu **rate_est)
208{
209 struct net_rate_estimator *est;
210
211 est = unrcu_pointer(xchg(rate_est, NULL));
212 if (est) {
213 timer_shutdown_sync(timer: &est->timer);
214 kfree_rcu(est, rcu);
215 }
216}
217EXPORT_SYMBOL(gen_kill_estimator);
218
219/**
220 * gen_replace_estimator - replace rate estimator configuration
221 * @bstats: basic statistics
222 * @cpu_bstats: bstats per cpu
223 * @rate_est: rate estimator statistics
224 * @lock: lock for statistics and control path
225 * @running: true if @bstats represents a running qdisc, thus @bstats'
226 * internal values might change during basic reads. Only used
227 * if @cpu_bstats is NULL
228 * @opt: rate estimator configuration TLV
229 *
230 * Replaces the configuration of a rate estimator by calling
231 * gen_kill_estimator() and gen_new_estimator().
232 *
233 * Returns 0 on success or a negative error code.
234 */
235int gen_replace_estimator(struct gnet_stats_basic_sync *bstats,
236 struct gnet_stats_basic_sync __percpu *cpu_bstats,
237 struct net_rate_estimator __rcu **rate_est,
238 spinlock_t *lock,
239 bool running, struct nlattr *opt)
240{
241 return gen_new_estimator(bstats, cpu_bstats, rate_est,
242 lock, running, opt);
243}
244EXPORT_SYMBOL(gen_replace_estimator);
245
246/**
247 * gen_estimator_active - test if estimator is currently in use
248 * @rate_est: rate estimator
249 *
250 * Returns true if estimator is active, and false if not.
251 */
252bool gen_estimator_active(struct net_rate_estimator __rcu **rate_est)
253{
254 return !!rcu_access_pointer(*rate_est);
255}
256EXPORT_SYMBOL(gen_estimator_active);
257
258bool gen_estimator_read(struct net_rate_estimator __rcu **rate_est,
259 struct gnet_stats_rate_est64 *sample)
260{
261 struct net_rate_estimator *est;
262 unsigned seq;
263
264 rcu_read_lock();
265 est = rcu_dereference(*rate_est);
266 if (!est) {
267 rcu_read_unlock();
268 return false;
269 }
270
271 do {
272 seq = read_seqcount_begin(&est->seq);
273 sample->bps = est->avbps >> 8;
274 sample->pps = est->avpps >> 8;
275 } while (read_seqcount_retry(&est->seq, seq));
276
277 rcu_read_unlock();
278 return true;
279}
280EXPORT_SYMBOL(gen_estimator_read);
281

source code of linux/net/core/gen_estimator.c