1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <linux/dim.h>
7
8/*
9 * Net DIM profiles:
10 * There are different set of profiles for each CQ period mode.
11 * There are different set of profiles for RX/TX CQs.
12 * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES
13 */
14#define NET_DIM_PARAMS_NUM_PROFILES 5
15#define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256
16#define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128
17#define NET_DIM_DEF_PROFILE_CQE 1
18#define NET_DIM_DEF_PROFILE_EQE 1
19
20#define NET_DIM_RX_EQE_PROFILES { \
21 {.usec = 1, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
22 {.usec = 8, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
23 {.usec = 64, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
24 {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
25 {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,} \
26}
27
28#define NET_DIM_RX_CQE_PROFILES { \
29 {.usec = 2, .pkts = 256,}, \
30 {.usec = 8, .pkts = 128,}, \
31 {.usec = 16, .pkts = 64,}, \
32 {.usec = 32, .pkts = 64,}, \
33 {.usec = 64, .pkts = 64,} \
34}
35
36#define NET_DIM_TX_EQE_PROFILES { \
37 {.usec = 1, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
38 {.usec = 8, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
39 {.usec = 32, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
40 {.usec = 64, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \
41 {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,} \
42}
43
44#define NET_DIM_TX_CQE_PROFILES { \
45 {.usec = 5, .pkts = 128,}, \
46 {.usec = 8, .pkts = 64,}, \
47 {.usec = 16, .pkts = 32,}, \
48 {.usec = 32, .pkts = 32,}, \
49 {.usec = 64, .pkts = 32,} \
50}
51
52static const struct dim_cq_moder
53rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
54 NET_DIM_RX_EQE_PROFILES,
55 NET_DIM_RX_CQE_PROFILES,
56};
57
58static const struct dim_cq_moder
59tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
60 NET_DIM_TX_EQE_PROFILES,
61 NET_DIM_TX_CQE_PROFILES,
62};
63
64struct dim_cq_moder
65net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
66{
67 struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
68
69 cq_moder.cq_period_mode = cq_period_mode;
70 return cq_moder;
71}
72EXPORT_SYMBOL(net_dim_get_rx_moderation);
73
74struct dim_cq_moder
75net_dim_get_def_rx_moderation(u8 cq_period_mode)
76{
77 u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
78 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
79
80 return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
81}
82EXPORT_SYMBOL(net_dim_get_def_rx_moderation);
83
84struct dim_cq_moder
85net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
86{
87 struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
88
89 cq_moder.cq_period_mode = cq_period_mode;
90 return cq_moder;
91}
92EXPORT_SYMBOL(net_dim_get_tx_moderation);
93
94struct dim_cq_moder
95net_dim_get_def_tx_moderation(u8 cq_period_mode)
96{
97 u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
98 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
99
100 return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
101}
102EXPORT_SYMBOL(net_dim_get_def_tx_moderation);
103
104static int net_dim_step(struct dim *dim)
105{
106 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
107 return DIM_TOO_TIRED;
108
109 switch (dim->tune_state) {
110 case DIM_PARKING_ON_TOP:
111 case DIM_PARKING_TIRED:
112 break;
113 case DIM_GOING_RIGHT:
114 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
115 return DIM_ON_EDGE;
116 dim->profile_ix++;
117 dim->steps_right++;
118 break;
119 case DIM_GOING_LEFT:
120 if (dim->profile_ix == 0)
121 return DIM_ON_EDGE;
122 dim->profile_ix--;
123 dim->steps_left++;
124 break;
125 }
126
127 dim->tired++;
128 return DIM_STEPPED;
129}
130
131static void net_dim_exit_parking(struct dim *dim)
132{
133 dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;
134 net_dim_step(dim);
135}
136
137static int net_dim_stats_compare(struct dim_stats *curr,
138 struct dim_stats *prev)
139{
140 if (!prev->bpms)
141 return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME;
142
143 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
144 return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER :
145 DIM_STATS_WORSE;
146
147 if (!prev->ppms)
148 return curr->ppms ? DIM_STATS_BETTER :
149 DIM_STATS_SAME;
150
151 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
152 return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER :
153 DIM_STATS_WORSE;
154
155 if (!prev->epms)
156 return DIM_STATS_SAME;
157
158 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
159 return (curr->epms < prev->epms) ? DIM_STATS_BETTER :
160 DIM_STATS_WORSE;
161
162 return DIM_STATS_SAME;
163}
164
165static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim)
166{
167 int prev_state = dim->tune_state;
168 int prev_ix = dim->profile_ix;
169 int stats_res;
170 int step_res;
171
172 switch (dim->tune_state) {
173 case DIM_PARKING_ON_TOP:
174 stats_res = net_dim_stats_compare(curr: curr_stats,
175 prev: &dim->prev_stats);
176 if (stats_res != DIM_STATS_SAME)
177 net_dim_exit_parking(dim);
178 break;
179
180 case DIM_PARKING_TIRED:
181 dim->tired--;
182 if (!dim->tired)
183 net_dim_exit_parking(dim);
184 break;
185
186 case DIM_GOING_RIGHT:
187 case DIM_GOING_LEFT:
188 stats_res = net_dim_stats_compare(curr: curr_stats,
189 prev: &dim->prev_stats);
190 if (stats_res != DIM_STATS_BETTER)
191 dim_turn(dim);
192
193 if (dim_on_top(dim)) {
194 dim_park_on_top(dim);
195 break;
196 }
197
198 step_res = net_dim_step(dim);
199 switch (step_res) {
200 case DIM_ON_EDGE:
201 dim_park_on_top(dim);
202 break;
203 case DIM_TOO_TIRED:
204 dim_park_tired(dim);
205 break;
206 }
207
208 break;
209 }
210
211 if (prev_state != DIM_PARKING_ON_TOP ||
212 dim->tune_state != DIM_PARKING_ON_TOP)
213 dim->prev_stats = *curr_stats;
214
215 return dim->profile_ix != prev_ix;
216}
217
218void net_dim(struct dim *dim, struct dim_sample end_sample)
219{
220 struct dim_stats curr_stats;
221 u16 nevents;
222
223 switch (dim->state) {
224 case DIM_MEASURE_IN_PROGRESS:
225 nevents = BIT_GAP(BITS_PER_TYPE(u16),
226 end_sample.event_ctr,
227 dim->start_sample.event_ctr);
228 if (nevents < DIM_NEVENTS)
229 break;
230 if (!dim_calc_stats(start: &dim->start_sample, end: &end_sample, curr_stats: &curr_stats))
231 break;
232 if (net_dim_decision(curr_stats: &curr_stats, dim)) {
233 dim->state = DIM_APPLY_NEW_PROFILE;
234 schedule_work(work: &dim->work);
235 break;
236 }
237 fallthrough;
238 case DIM_START_MEASURE:
239 dim_update_sample(event_ctr: end_sample.event_ctr, packets: end_sample.pkt_ctr,
240 bytes: end_sample.byte_ctr, s: &dim->start_sample);
241 dim->state = DIM_MEASURE_IN_PROGRESS;
242 break;
243 case DIM_APPLY_NEW_PROFILE:
244 break;
245 }
246}
247EXPORT_SYMBOL(net_dim);
248

source code of linux/lib/dim/net_dim.c