1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3  *
4  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  *
35  * $FreeBSD$
36  */
37 
38 /* This file implements Dynamic Interrupt Moderation, DIM */
39 
40 #ifndef _LINUXKPI_LINUX_NET_DIM_H
41 #define	_LINUXKPI_LINUX_NET_DIM_H
42 
43 #include <asm/types.h>
44 
45 #include <linux/workqueue.h>
46 #include <linux/ktime.h>
47 
48 struct net_dim_cq_moder {
49 	u16	usec;
50 	u16	pkts;
51 	u8	cq_period_mode;
52 };
53 
54 struct net_dim_sample {
55 	ktime_t	time;
56 	u32	pkt_ctr;
57 	u32	byte_ctr;
58 	u16	event_ctr;
59 };
60 
61 struct net_dim_stats {
62 	int	ppms;			/* packets per msec */
63 	int	bpms;			/* bytes per msec */
64 	int	epms;			/* events per msec */
65 };
66 
67 struct net_dim {			/* Adaptive Moderation */
68 	u8	state;
69 	struct net_dim_stats prev_stats;
70 	struct net_dim_sample start_sample;
71 	struct work_struct work;
72 	u16	event_ctr;
73 	u8	profile_ix;
74 	u8	mode;
75 	u8	tune_state;
76 	u8	steps_right;
77 	u8	steps_left;
78 	u8	tired;
79 };
80 
81 enum {
82 	NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
83 	NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
84 	NET_DIM_CQ_PERIOD_NUM_MODES = 0x2,
85 	NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF,
86 };
87 
88 /* Adaptive moderation logic */
89 enum {
90 	NET_DIM_START_MEASURE,
91 	NET_DIM_MEASURE_IN_PROGRESS,
92 	NET_DIM_APPLY_NEW_PROFILE,
93 };
94 
95 enum {
96 	NET_DIM_PARKING_ON_TOP,
97 	NET_DIM_PARKING_TIRED,
98 	NET_DIM_GOING_RIGHT,
99 	NET_DIM_GOING_LEFT,
100 };
101 
102 enum {
103 	NET_DIM_STATS_WORSE,
104 	NET_DIM_STATS_SAME,
105 	NET_DIM_STATS_BETTER,
106 };
107 
108 enum {
109 	NET_DIM_STEPPED,
110 	NET_DIM_TOO_TIRED,
111 	NET_DIM_ON_EDGE,
112 };
113 
114 #define	NET_DIM_PARAMS_NUM_PROFILES 5
115 /* Adaptive moderation profiles */
116 #define	NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
117 #define	NET_DIM_DEF_PROFILE_CQE 1
118 #define	NET_DIM_DEF_PROFILE_EQE 1
119 
120 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
121 #define	NET_DIM_EQE_PROFILES { \
122 	{1,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
123 	{8,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
124 	{64,  NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
125 	{128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
126 	{256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
127 }
128 
129 #define	NET_DIM_CQE_PROFILES { \
130 	{2,  256},             \
131 	{8,  128},             \
132 	{16, 64},              \
133 	{32, 64},              \
134 	{64, 64}               \
135 }
136 
137 static const struct net_dim_cq_moder
138 	net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
139 	NET_DIM_EQE_PROFILES,
140 	NET_DIM_CQE_PROFILES,
141 };
142 
143 static inline struct net_dim_cq_moder
144 net_dim_get_profile(u8 cq_period_mode,
145     int ix)
146 {
147 	struct net_dim_cq_moder cq_moder;
148 
149 	cq_moder = net_dim_profile[cq_period_mode][ix];
150 	cq_moder.cq_period_mode = cq_period_mode;
151 	return cq_moder;
152 }
153 
154 static inline struct net_dim_cq_moder
155 net_dim_get_def_profile(u8 rx_cq_period_mode)
156 {
157 	int default_profile_ix;
158 
159 	if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE)
160 		default_profile_ix = NET_DIM_DEF_PROFILE_CQE;
161 	else	/* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */
162 		default_profile_ix = NET_DIM_DEF_PROFILE_EQE;
163 
164 	return net_dim_get_profile(rx_cq_period_mode, default_profile_ix);
165 }
166 
167 static inline bool
168 net_dim_on_top(struct net_dim *dim)
169 {
170 	switch (dim->tune_state) {
171 	case NET_DIM_PARKING_ON_TOP:
172 	case NET_DIM_PARKING_TIRED:
173 		return true;
174 	case NET_DIM_GOING_RIGHT:
175 		return (dim->steps_left > 1) && (dim->steps_right == 1);
176 	default:	/* NET_DIM_GOING_LEFT */
177 		return (dim->steps_right > 1) && (dim->steps_left == 1);
178 	}
179 }
180 
181 static inline void
182 net_dim_turn(struct net_dim *dim)
183 {
184 	switch (dim->tune_state) {
185 	case NET_DIM_PARKING_ON_TOP:
186 	case NET_DIM_PARKING_TIRED:
187 		break;
188 	case NET_DIM_GOING_RIGHT:
189 		dim->tune_state = NET_DIM_GOING_LEFT;
190 		dim->steps_left = 0;
191 		break;
192 	case NET_DIM_GOING_LEFT:
193 		dim->tune_state = NET_DIM_GOING_RIGHT;
194 		dim->steps_right = 0;
195 		break;
196 	}
197 }
198 
199 static inline int
200 net_dim_step(struct net_dim *dim)
201 {
202 	if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
203 		return NET_DIM_TOO_TIRED;
204 
205 	switch (dim->tune_state) {
206 	case NET_DIM_PARKING_ON_TOP:
207 	case NET_DIM_PARKING_TIRED:
208 		break;
209 	case NET_DIM_GOING_RIGHT:
210 		if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
211 			return NET_DIM_ON_EDGE;
212 		dim->profile_ix++;
213 		dim->steps_right++;
214 		break;
215 	case NET_DIM_GOING_LEFT:
216 		if (dim->profile_ix == 0)
217 			return NET_DIM_ON_EDGE;
218 		dim->profile_ix--;
219 		dim->steps_left++;
220 		break;
221 	}
222 
223 	dim->tired++;
224 	return NET_DIM_STEPPED;
225 }
226 
227 static inline void
228 net_dim_park_on_top(struct net_dim *dim)
229 {
230 	dim->steps_right = 0;
231 	dim->steps_left = 0;
232 	dim->tired = 0;
233 	dim->tune_state = NET_DIM_PARKING_ON_TOP;
234 }
235 
236 static inline void
237 net_dim_park_tired(struct net_dim *dim)
238 {
239 	dim->steps_right = 0;
240 	dim->steps_left = 0;
241 	dim->tune_state = NET_DIM_PARKING_TIRED;
242 }
243 
244 static inline void
245 net_dim_exit_parking(struct net_dim *dim)
246 {
247 	dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
248 	NET_DIM_GOING_RIGHT;
249 	net_dim_step(dim);
250 }
251 
252 #define	IS_SIGNIFICANT_DIFF(val, ref) \
253 	(((100UL * abs((val) - (ref))) / (ref)) > 10)	/* more than 10%
254 							 * difference */
255 
256 static inline int
257 net_dim_stats_compare(struct net_dim_stats *curr,
258     struct net_dim_stats *prev)
259 {
260 	if (!prev->bpms)
261 		return curr->bpms ? NET_DIM_STATS_BETTER :
262 		NET_DIM_STATS_SAME;
263 
264 	if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
265 		return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
266 		    NET_DIM_STATS_WORSE;
267 
268 	if (!prev->ppms)
269 		return curr->ppms ? NET_DIM_STATS_BETTER :
270 		    NET_DIM_STATS_SAME;
271 
272 	if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
273 		return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
274 		    NET_DIM_STATS_WORSE;
275 
276 	if (!prev->epms)
277 		return NET_DIM_STATS_SAME;
278 
279 	if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
280 		return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
281 		    NET_DIM_STATS_WORSE;
282 
283 	return NET_DIM_STATS_SAME;
284 }
285 
286 static inline bool
287 net_dim_decision(struct net_dim_stats *curr_stats,
288     struct net_dim *dim)
289 {
290 	int prev_state = dim->tune_state;
291 	int prev_ix = dim->profile_ix;
292 	int stats_res;
293 	int step_res;
294 
295 	switch (dim->tune_state) {
296 	case NET_DIM_PARKING_ON_TOP:
297 		stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
298 		if (stats_res != NET_DIM_STATS_SAME)
299 			net_dim_exit_parking(dim);
300 		break;
301 
302 	case NET_DIM_PARKING_TIRED:
303 		dim->tired--;
304 		if (!dim->tired)
305 			net_dim_exit_parking(dim);
306 		break;
307 
308 	case NET_DIM_GOING_RIGHT:
309 	case NET_DIM_GOING_LEFT:
310 		stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
311 		if (stats_res != NET_DIM_STATS_BETTER)
312 			net_dim_turn(dim);
313 
314 		if (net_dim_on_top(dim)) {
315 			net_dim_park_on_top(dim);
316 			break;
317 		}
318 		step_res = net_dim_step(dim);
319 		switch (step_res) {
320 		case NET_DIM_ON_EDGE:
321 			net_dim_park_on_top(dim);
322 			break;
323 		case NET_DIM_TOO_TIRED:
324 			net_dim_park_tired(dim);
325 			break;
326 		}
327 
328 		break;
329 	}
330 
331 	if ((prev_state != NET_DIM_PARKING_ON_TOP) ||
332 	    (dim->tune_state != NET_DIM_PARKING_ON_TOP))
333 		dim->prev_stats = *curr_stats;
334 
335 	return dim->profile_ix != prev_ix;
336 }
337 
338 static inline void
339 net_dim_sample(u16 event_ctr,
340     u64 packets,
341     u64 bytes,
342     struct net_dim_sample *s)
343 {
344 	s->time = ktime_get();
345 	s->pkt_ctr = packets;
346 	s->byte_ctr = bytes;
347 	s->event_ctr = event_ctr;
348 }
349 
350 #define	NET_DIM_NEVENTS 64
351 #define	BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
352 
353 static inline void
354 net_dim_calc_stats(struct net_dim_sample *start,
355     struct net_dim_sample *end,
356     struct net_dim_stats *curr_stats)
357 {
358 	/* u32 holds up to 71 minutes, should be enough */
359 	u32 delta_us = ktime_us_delta(end->time, start->time);
360 	u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
361 	u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
362 	    start->byte_ctr);
363 
364 	if (!delta_us)
365 		return;
366 
367 	curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
368 	curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
369 	curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
370 	    delta_us);
371 }
372 
373 static inline void
374 net_dim(struct net_dim *dim,
375     u64 packets, u64 bytes)
376 {
377 	struct net_dim_stats curr_stats;
378 	struct net_dim_sample end_sample;
379 	u16 nevents;
380 
381 	dim->event_ctr++;
382 
383 	switch (dim->state) {
384 	case NET_DIM_MEASURE_IN_PROGRESS:
385 		nevents = BIT_GAP(BITS_PER_TYPE(u16),
386 		    dim->event_ctr,
387 		    dim->start_sample.event_ctr);
388 		if (nevents < NET_DIM_NEVENTS)
389 			break;
390 		net_dim_sample(dim->event_ctr, packets, bytes, &end_sample);
391 		net_dim_calc_stats(&dim->start_sample, &end_sample,
392 		    &curr_stats);
393 		if (net_dim_decision(&curr_stats, dim)) {
394 			dim->state = NET_DIM_APPLY_NEW_PROFILE;
395 			schedule_work(&dim->work);
396 			break;
397 		}
398 		/* FALLTHROUGH */
399 	case NET_DIM_START_MEASURE:
400 		net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample);
401 		dim->state = NET_DIM_MEASURE_IN_PROGRESS;
402 		break;
403 	case NET_DIM_APPLY_NEW_PROFILE:
404 		break;
405 	default:
406 		break;
407 	}
408 }
409 
410 #endif					/* _LINUXKPI_LINUX_NET_DIM_H */
411