1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3  *
4  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 
36 /* This file implements Dynamic Interrupt Moderation, DIM */
37 
38 #ifndef _LINUXKPI_LINUX_NET_DIM_H
39 #define	_LINUXKPI_LINUX_NET_DIM_H
40 
41 #include <asm/types.h>
42 
43 #include <linux/workqueue.h>
44 #include <linux/ktime.h>
45 
46 struct net_dim_cq_moder {
47 	u16	usec;
48 	u16	pkts;
49 	u8	cq_period_mode;
50 };
51 
52 struct net_dim_sample {
53 	ktime_t	time;
54 	u32	pkt_ctr;
55 	u32	byte_ctr;
56 	u16	event_ctr;
57 };
58 
59 struct net_dim_stats {
60 	int	ppms;			/* packets per msec */
61 	int	bpms;			/* bytes per msec */
62 	int	epms;			/* events per msec */
63 };
64 
65 struct net_dim {			/* Adaptive Moderation */
66 	u8	state;
67 	struct net_dim_stats prev_stats;
68 	struct net_dim_sample start_sample;
69 	struct work_struct work;
70 	u16	event_ctr;
71 	u8	profile_ix;
72 	u8	mode;
73 	u8	tune_state;
74 	u8	steps_right;
75 	u8	steps_left;
76 	u8	tired;
77 };
78 
79 enum {
80 	NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
81 	NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
82 	NET_DIM_CQ_PERIOD_NUM_MODES = 0x2,
83 	NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF,
84 };
85 
86 /* Adaptive moderation logic */
87 enum {
88 	NET_DIM_START_MEASURE,
89 	NET_DIM_MEASURE_IN_PROGRESS,
90 	NET_DIM_APPLY_NEW_PROFILE,
91 };
92 
93 enum {
94 	NET_DIM_PARKING_ON_TOP,
95 	NET_DIM_PARKING_TIRED,
96 	NET_DIM_GOING_RIGHT,
97 	NET_DIM_GOING_LEFT,
98 };
99 
100 enum {
101 	NET_DIM_STATS_WORSE,
102 	NET_DIM_STATS_SAME,
103 	NET_DIM_STATS_BETTER,
104 };
105 
106 enum {
107 	NET_DIM_STEPPED,
108 	NET_DIM_TOO_TIRED,
109 	NET_DIM_ON_EDGE,
110 };
111 
112 #define	NET_DIM_PARAMS_NUM_PROFILES 5
113 /* Adaptive moderation profiles */
114 #define	NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
115 #define	NET_DIM_DEF_PROFILE_CQE 1
116 #define	NET_DIM_DEF_PROFILE_EQE 1
117 
118 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
119 #define	NET_DIM_EQE_PROFILES { \
120 	{1,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
121 	{8,   NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
122 	{64,  NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
123 	{128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
124 	{256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
125 }
126 
127 #define	NET_DIM_CQE_PROFILES { \
128 	{2,  256},             \
129 	{8,  128},             \
130 	{16, 64},              \
131 	{32, 64},              \
132 	{64, 64}               \
133 }
134 
135 static const struct net_dim_cq_moder
136 	net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
137 	NET_DIM_EQE_PROFILES,
138 	NET_DIM_CQE_PROFILES,
139 };
140 
141 static inline struct net_dim_cq_moder
142 net_dim_get_profile(u8 cq_period_mode,
143     int ix)
144 {
145 	struct net_dim_cq_moder cq_moder;
146 
147 	cq_moder = net_dim_profile[cq_period_mode][ix];
148 	cq_moder.cq_period_mode = cq_period_mode;
149 	return cq_moder;
150 }
151 
152 static inline struct net_dim_cq_moder
153 net_dim_get_def_profile(u8 rx_cq_period_mode)
154 {
155 	int default_profile_ix;
156 
157 	if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE)
158 		default_profile_ix = NET_DIM_DEF_PROFILE_CQE;
159 	else	/* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */
160 		default_profile_ix = NET_DIM_DEF_PROFILE_EQE;
161 
162 	return net_dim_get_profile(rx_cq_period_mode, default_profile_ix);
163 }
164 
165 static inline bool
166 net_dim_on_top(struct net_dim *dim)
167 {
168 	switch (dim->tune_state) {
169 	case NET_DIM_PARKING_ON_TOP:
170 	case NET_DIM_PARKING_TIRED:
171 		return true;
172 	case NET_DIM_GOING_RIGHT:
173 		return (dim->steps_left > 1) && (dim->steps_right == 1);
174 	default:	/* NET_DIM_GOING_LEFT */
175 		return (dim->steps_right > 1) && (dim->steps_left == 1);
176 	}
177 }
178 
179 static inline void
180 net_dim_turn(struct net_dim *dim)
181 {
182 	switch (dim->tune_state) {
183 	case NET_DIM_PARKING_ON_TOP:
184 	case NET_DIM_PARKING_TIRED:
185 		break;
186 	case NET_DIM_GOING_RIGHT:
187 		dim->tune_state = NET_DIM_GOING_LEFT;
188 		dim->steps_left = 0;
189 		break;
190 	case NET_DIM_GOING_LEFT:
191 		dim->tune_state = NET_DIM_GOING_RIGHT;
192 		dim->steps_right = 0;
193 		break;
194 	}
195 }
196 
197 static inline int
198 net_dim_step(struct net_dim *dim)
199 {
200 	if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
201 		return NET_DIM_TOO_TIRED;
202 
203 	switch (dim->tune_state) {
204 	case NET_DIM_PARKING_ON_TOP:
205 	case NET_DIM_PARKING_TIRED:
206 		break;
207 	case NET_DIM_GOING_RIGHT:
208 		if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
209 			return NET_DIM_ON_EDGE;
210 		dim->profile_ix++;
211 		dim->steps_right++;
212 		break;
213 	case NET_DIM_GOING_LEFT:
214 		if (dim->profile_ix == 0)
215 			return NET_DIM_ON_EDGE;
216 		dim->profile_ix--;
217 		dim->steps_left++;
218 		break;
219 	}
220 
221 	dim->tired++;
222 	return NET_DIM_STEPPED;
223 }
224 
225 static inline void
226 net_dim_park_on_top(struct net_dim *dim)
227 {
228 	dim->steps_right = 0;
229 	dim->steps_left = 0;
230 	dim->tired = 0;
231 	dim->tune_state = NET_DIM_PARKING_ON_TOP;
232 }
233 
234 static inline void
235 net_dim_park_tired(struct net_dim *dim)
236 {
237 	dim->steps_right = 0;
238 	dim->steps_left = 0;
239 	dim->tune_state = NET_DIM_PARKING_TIRED;
240 }
241 
242 static inline void
243 net_dim_exit_parking(struct net_dim *dim)
244 {
245 	dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
246 	NET_DIM_GOING_RIGHT;
247 	net_dim_step(dim);
248 }
249 
250 #define	IS_SIGNIFICANT_DIFF(val, ref) \
251 	(((100UL * abs((val) - (ref))) / (ref)) > 10)	/* more than 10%
252 							 * difference */
253 
254 static inline int
255 net_dim_stats_compare(struct net_dim_stats *curr,
256     struct net_dim_stats *prev)
257 {
258 	if (!prev->bpms)
259 		return curr->bpms ? NET_DIM_STATS_BETTER :
260 		NET_DIM_STATS_SAME;
261 
262 	if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
263 		return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
264 		    NET_DIM_STATS_WORSE;
265 
266 	if (!prev->ppms)
267 		return curr->ppms ? NET_DIM_STATS_BETTER :
268 		    NET_DIM_STATS_SAME;
269 
270 	if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
271 		return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
272 		    NET_DIM_STATS_WORSE;
273 
274 	if (!prev->epms)
275 		return NET_DIM_STATS_SAME;
276 
277 	if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
278 		return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
279 		    NET_DIM_STATS_WORSE;
280 
281 	return NET_DIM_STATS_SAME;
282 }
283 
284 static inline bool
285 net_dim_decision(struct net_dim_stats *curr_stats,
286     struct net_dim *dim)
287 {
288 	int prev_state = dim->tune_state;
289 	int prev_ix = dim->profile_ix;
290 	int stats_res;
291 	int step_res;
292 
293 	switch (dim->tune_state) {
294 	case NET_DIM_PARKING_ON_TOP:
295 		stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
296 		if (stats_res != NET_DIM_STATS_SAME)
297 			net_dim_exit_parking(dim);
298 		break;
299 
300 	case NET_DIM_PARKING_TIRED:
301 		dim->tired--;
302 		if (!dim->tired)
303 			net_dim_exit_parking(dim);
304 		break;
305 
306 	case NET_DIM_GOING_RIGHT:
307 	case NET_DIM_GOING_LEFT:
308 		stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
309 		if (stats_res != NET_DIM_STATS_BETTER)
310 			net_dim_turn(dim);
311 
312 		if (net_dim_on_top(dim)) {
313 			net_dim_park_on_top(dim);
314 			break;
315 		}
316 		step_res = net_dim_step(dim);
317 		switch (step_res) {
318 		case NET_DIM_ON_EDGE:
319 			net_dim_park_on_top(dim);
320 			break;
321 		case NET_DIM_TOO_TIRED:
322 			net_dim_park_tired(dim);
323 			break;
324 		}
325 
326 		break;
327 	}
328 
329 	if ((prev_state != NET_DIM_PARKING_ON_TOP) ||
330 	    (dim->tune_state != NET_DIM_PARKING_ON_TOP))
331 		dim->prev_stats = *curr_stats;
332 
333 	return dim->profile_ix != prev_ix;
334 }
335 
336 static inline void
337 net_dim_sample(u16 event_ctr,
338     u64 packets,
339     u64 bytes,
340     struct net_dim_sample *s)
341 {
342 	s->time = ktime_get();
343 	s->pkt_ctr = packets;
344 	s->byte_ctr = bytes;
345 	s->event_ctr = event_ctr;
346 }
347 
348 #define	NET_DIM_NEVENTS 64
349 #define	BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
350 
351 static inline void
352 net_dim_calc_stats(struct net_dim_sample *start,
353     struct net_dim_sample *end,
354     struct net_dim_stats *curr_stats)
355 {
356 	/* u32 holds up to 71 minutes, should be enough */
357 	u32 delta_us = ktime_us_delta(end->time, start->time);
358 	u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
359 	u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
360 	    start->byte_ctr);
361 
362 	if (!delta_us)
363 		return;
364 
365 	curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
366 	curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
367 	curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
368 	    delta_us);
369 }
370 
371 static inline void
372 net_dim(struct net_dim *dim,
373     u64 packets, u64 bytes)
374 {
375 	struct net_dim_stats curr_stats;
376 	struct net_dim_sample end_sample;
377 	u16 nevents;
378 
379 	dim->event_ctr++;
380 
381 	switch (dim->state) {
382 	case NET_DIM_MEASURE_IN_PROGRESS:
383 		nevents = BIT_GAP(BITS_PER_TYPE(u16),
384 		    dim->event_ctr,
385 		    dim->start_sample.event_ctr);
386 		if (nevents < NET_DIM_NEVENTS)
387 			break;
388 		net_dim_sample(dim->event_ctr, packets, bytes, &end_sample);
389 		net_dim_calc_stats(&dim->start_sample, &end_sample,
390 		    &curr_stats);
391 		if (net_dim_decision(&curr_stats, dim)) {
392 			dim->state = NET_DIM_APPLY_NEW_PROFILE;
393 			schedule_work(&dim->work);
394 			break;
395 		}
396 		/* FALLTHROUGH */
397 	case NET_DIM_START_MEASURE:
398 		net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample);
399 		dim->state = NET_DIM_MEASURE_IN_PROGRESS;
400 		break;
401 	case NET_DIM_APPLY_NEW_PROFILE:
402 		break;
403 	default:
404 		break;
405 	}
406 }
407 
408 #endif					/* _LINUXKPI_LINUX_NET_DIM_H */
409