1 | /* SPDX-License-Identifier: GPL-2.0 |
2 | * |
3 | * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT. |
4 | * Do not use in new code. |
5 | */ |
6 | #ifndef _BLK_CGROUP_RWSTAT_H |
7 | #define _BLK_CGROUP_RWSTAT_H |
8 | |
9 | #include "blk-cgroup.h" |
10 | |
11 | enum blkg_rwstat_type { |
12 | BLKG_RWSTAT_READ, |
13 | BLKG_RWSTAT_WRITE, |
14 | BLKG_RWSTAT_SYNC, |
15 | BLKG_RWSTAT_ASYNC, |
16 | BLKG_RWSTAT_DISCARD, |
17 | |
18 | BLKG_RWSTAT_NR, |
19 | BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR, |
20 | }; |
21 | |
22 | /* |
23 | * blkg_[rw]stat->aux_cnt is excluded for local stats but included for |
24 | * recursive. Used to carry stats of dead children. |
25 | */ |
26 | struct blkg_rwstat { |
27 | struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR]; |
28 | atomic64_t aux_cnt[BLKG_RWSTAT_NR]; |
29 | }; |
30 | |
31 | struct blkg_rwstat_sample { |
32 | u64 cnt[BLKG_RWSTAT_NR]; |
33 | }; |
34 | |
35 | static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat, |
36 | unsigned int idx) |
37 | { |
38 | return atomic64_read(v: &rwstat->aux_cnt[idx]) + |
39 | percpu_counter_sum_positive(fbc: &rwstat->cpu_cnt[idx]); |
40 | } |
41 | |
42 | int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp); |
43 | void blkg_rwstat_exit(struct blkg_rwstat *rwstat); |
44 | u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, |
45 | const struct blkg_rwstat_sample *rwstat); |
46 | u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, |
47 | int off); |
48 | void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, |
49 | int off, struct blkg_rwstat_sample *sum); |
50 | |
51 | |
52 | /** |
53 | * blkg_rwstat_add - add a value to a blkg_rwstat |
54 | * @rwstat: target blkg_rwstat |
55 | * @op: REQ_OP and flags |
56 | * @val: value to add |
57 | * |
58 | * Add @val to @rwstat. The counters are chosen according to @rw. The |
59 | * caller is responsible for synchronizing calls to this function. |
60 | */ |
61 | static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat, |
62 | blk_opf_t opf, uint64_t val) |
63 | { |
64 | struct percpu_counter *cnt; |
65 | |
66 | if (op_is_discard(op: opf)) |
67 | cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD]; |
68 | else if (op_is_write(op: opf)) |
69 | cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE]; |
70 | else |
71 | cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ]; |
72 | |
73 | percpu_counter_add_batch(fbc: cnt, amount: val, BLKG_STAT_CPU_BATCH); |
74 | |
75 | if (op_is_sync(op: opf)) |
76 | cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC]; |
77 | else |
78 | cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC]; |
79 | |
80 | percpu_counter_add_batch(fbc: cnt, amount: val, BLKG_STAT_CPU_BATCH); |
81 | } |
82 | |
83 | /** |
84 | * blkg_rwstat_read - read the current values of a blkg_rwstat |
85 | * @rwstat: blkg_rwstat to read |
86 | * |
87 | * Read the current snapshot of @rwstat and return it in the aux counts. |
88 | */ |
89 | static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat, |
90 | struct blkg_rwstat_sample *result) |
91 | { |
92 | int i; |
93 | |
94 | for (i = 0; i < BLKG_RWSTAT_NR; i++) |
95 | result->cnt[i] = |
96 | percpu_counter_sum_positive(fbc: &rwstat->cpu_cnt[i]); |
97 | } |
98 | |
99 | /** |
100 | * blkg_rwstat_total - read the total count of a blkg_rwstat |
101 | * @rwstat: blkg_rwstat to read |
102 | * |
103 | * Return the total count of @rwstat regardless of the IO direction. This |
104 | * function can be called without synchronization and takes care of u64 |
105 | * atomicity. |
106 | */ |
107 | static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat) |
108 | { |
109 | struct blkg_rwstat_sample tmp = { }; |
110 | |
111 | blkg_rwstat_read(rwstat, result: &tmp); |
112 | return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]; |
113 | } |
114 | |
115 | /** |
116 | * blkg_rwstat_reset - reset a blkg_rwstat |
117 | * @rwstat: blkg_rwstat to reset |
118 | */ |
119 | static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat) |
120 | { |
121 | int i; |
122 | |
123 | for (i = 0; i < BLKG_RWSTAT_NR; i++) { |
124 | percpu_counter_set(fbc: &rwstat->cpu_cnt[i], amount: 0); |
125 | atomic64_set(v: &rwstat->aux_cnt[i], i: 0); |
126 | } |
127 | } |
128 | |
129 | /** |
130 | * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count |
131 | * @to: the destination blkg_rwstat |
132 | * @from: the source |
133 | * |
134 | * Add @from's count including the aux one to @to's aux count. |
135 | */ |
136 | static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to, |
137 | struct blkg_rwstat *from) |
138 | { |
139 | u64 sum[BLKG_RWSTAT_NR]; |
140 | int i; |
141 | |
142 | for (i = 0; i < BLKG_RWSTAT_NR; i++) |
143 | sum[i] = percpu_counter_sum_positive(fbc: &from->cpu_cnt[i]); |
144 | |
145 | for (i = 0; i < BLKG_RWSTAT_NR; i++) |
146 | atomic64_add(i: sum[i] + atomic64_read(v: &from->aux_cnt[i]), |
147 | v: &to->aux_cnt[i]); |
148 | } |
149 | #endif /* _BLK_CGROUP_RWSTAT_H */ |
150 | |