xref: /linux/include/net/gro.h (revision c6fbb759) 
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 
3 #ifndef _NET_IPV6_GRO_H
4 #define _NET_IPV6_GRO_H
5 
6 #include <linux/indirect_call_wrapper.h>
7 #include <linux/ip.h>
8 #include <linux/ipv6.h>
9 #include <net/ip6_checksum.h>
10 #include <linux/skbuff.h>
11 #include <net/udp.h>
12 
13 struct napi_gro_cb {
14 	/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
15 	void	*frag0;
16 
17 	/* Length of frag0. */
18 	unsigned int frag0_len;
19 
20 	/* This indicates where we are processing relative to skb->data. */
21 	int	data_offset;
22 
23 	/* This is non-zero if the packet cannot be merged with the new skb. */
24 	u16	flush;
25 
26 	/* Save the IP ID here and check when we get to the transport layer */
27 	u16	flush_id;
28 
29 	/* Number of segments aggregated. */
30 	u16	count;
31 
32 	/* Used in ipv6_gro_receive() and foo-over-udp */
33 	u16	proto;
34 
35 	/* jiffies when first packet was created/queued */
36 	unsigned long age;
37 
38 /* Used in napi_gro_cb::free */
39 #define NAPI_GRO_FREE             1
40 #define NAPI_GRO_FREE_STOLEN_HEAD 2
41 	/* portion of the cb set to zero at every gro iteration */
42 	struct_group(zeroed,
43 
44 		/* Start offset for remote checksum offload */
45 		u16	gro_remcsum_start;
46 
47 		/* This is non-zero if the packet may be of the same flow. */
48 		u8	same_flow:1;
49 
50 		/* Used in tunnel GRO receive */
51 		u8	encap_mark:1;
52 
53 		/* GRO checksum is valid */
54 		u8	csum_valid:1;
55 
56 		/* Number of checksums via CHECKSUM_UNNECESSARY */
57 		u8	csum_cnt:3;
58 
59 		/* Free the skb? */
60 		u8	free:2;
61 
62 		/* Used in foo-over-udp, set in udp[46]_gro_receive */
63 		u8	is_ipv6:1;
64 
65 		/* Used in GRE, set in fou/gue_gro_receive */
66 		u8	is_fou:1;
67 
68 		/* Used to determine if flush_id can be ignored */
69 		u8	is_atomic:1;
70 
71 		/* Number of gro_receive callbacks this packet already went through */
72 		u8 recursion_counter:4;
73 
74 		/* GRO is done by frag_list pointer chaining. */
75 		u8	is_flist:1;
76 	);
77 
78 	/* used to support CHECKSUM_COMPLETE for tunneling protocols */
79 	__wsum	csum;
80 
81 	/* used in skb_gro_receive() slow path */
82 	struct sk_buff *last;
83 };
84 
85 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
86 
87 #define GRO_RECURSION_LIMIT 15
88 static inline int gro_recursion_inc_test(struct sk_buff *skb)
89 {
90 	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
91 }
92 
93 typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
94 static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
95 					       struct list_head *head,
96 					       struct sk_buff *skb)
97 {
98 	if (unlikely(gro_recursion_inc_test(skb))) {
99 		NAPI_GRO_CB(skb)->flush |= 1;
100 		return NULL;
101 	}
102 
103 	return cb(head, skb);
104 }
105 
106 typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
107 					    struct sk_buff *);
108 static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
109 						  struct sock *sk,
110 						  struct list_head *head,
111 						  struct sk_buff *skb)
112 {
113 	if (unlikely(gro_recursion_inc_test(skb))) {
114 		NAPI_GRO_CB(skb)->flush |= 1;
115 		return NULL;
116 	}
117 
118 	return cb(sk, head, skb);
119 }
120 
121 static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
122 {
123 	return NAPI_GRO_CB(skb)->data_offset;
124 }
125 
126 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
127 {
128 	return skb->len - NAPI_GRO_CB(skb)->data_offset;
129 }
130 
131 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
132 {
133 	NAPI_GRO_CB(skb)->data_offset += len;
134 }
135 
136 static inline void *skb_gro_header_fast(struct sk_buff *skb,
137 					unsigned int offset)
138 {
139 	return NAPI_GRO_CB(skb)->frag0 + offset;
140 }
141 
142 static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
143 {
144 	return NAPI_GRO_CB(skb)->frag0_len < hlen;
145 }
146 
147 static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
148 {
149 	NAPI_GRO_CB(skb)->frag0 = NULL;
150 	NAPI_GRO_CB(skb)->frag0_len = 0;
151 }
152 
153 static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
154 					unsigned int offset)
155 {
156 	if (!pskb_may_pull(skb, hlen))
157 		return NULL;
158 
159 	skb_gro_frag0_invalidate(skb);
160 	return skb->data + offset;
161 }
162 
163 static inline void *skb_gro_header(struct sk_buff *skb,
164 					unsigned int hlen, unsigned int offset)
165 {
166 	void *ptr;
167 
168 	ptr = skb_gro_header_fast(skb, offset);
169 	if (skb_gro_header_hard(skb, hlen))
170 		ptr = skb_gro_header_slow(skb, hlen, offset);
171 	return ptr;
172 }
173 
174 static inline void *skb_gro_network_header(struct sk_buff *skb)
175 {
176 	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
177 	       skb_network_offset(skb);
178 }
179 
180 static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto)
181 {
182 	const struct iphdr *iph = skb_gro_network_header(skb);
183 
184 	return csum_tcpudp_nofold(iph->saddr, iph->daddr,
185 				  skb_gro_len(skb), proto, 0);
186 }
187 
188 static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
189 					const void *start, unsigned int len)
190 {
191 	if (NAPI_GRO_CB(skb)->csum_valid)
192 		NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,
193 						wsum_negate(NAPI_GRO_CB(skb)->csum)));
194 }
195 
196 /* GRO checksum functions. These are logical equivalents of the normal
197  * checksum functions (in skbuff.h) except that they operate on the GRO
198  * offsets and fields in sk_buff.
199  */
200 
201 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
202 
203 static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
204 {
205 	return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
206 }
207 
208 static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
209 						      bool zero_okay,
210 						      __sum16 check)
211 {
212 	return ((skb->ip_summed != CHECKSUM_PARTIAL ||
213 		skb_checksum_start_offset(skb) <
214 		 skb_gro_offset(skb)) &&
215 		!skb_at_gro_remcsum_start(skb) &&
216 		NAPI_GRO_CB(skb)->csum_cnt == 0 &&
217 		(!zero_okay || check));
218 }
219 
220 static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
221 							   __wsum psum)
222 {
223 	if (NAPI_GRO_CB(skb)->csum_valid &&
224 	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))
225 		return 0;
226 
227 	NAPI_GRO_CB(skb)->csum = psum;
228 
229 	return __skb_gro_checksum_complete(skb);
230 }
231 
232 static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
233 {
234 	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
235 		/* Consume a checksum from CHECKSUM_UNNECESSARY */
236 		NAPI_GRO_CB(skb)->csum_cnt--;
237 	} else {
238 		/* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
239 		 * verified a new top level checksum or an encapsulated one
240 		 * during GRO. This saves work if we fallback to normal path.
241 		 */
242 		__skb_incr_checksum_unnecessary(skb);
243 	}
244 }
245 
246 #define __skb_gro_checksum_validate(skb, proto, zero_okay, check,	\
247 				    compute_pseudo)			\
248 ({									\
249 	__sum16 __ret = 0;						\
250 	if (__skb_gro_checksum_validate_needed(skb, zero_okay, check))	\
251 		__ret = __skb_gro_checksum_validate_complete(skb,	\
252 				compute_pseudo(skb, proto));		\
253 	if (!__ret)							\
254 		skb_gro_incr_csum_unnecessary(skb);			\
255 	__ret;								\
256 })
257 
258 #define skb_gro_checksum_validate(skb, proto, compute_pseudo)		\
259 	__skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
260 
261 #define skb_gro_checksum_validate_zero_check(skb, proto, check,		\
262 					     compute_pseudo)		\
263 	__skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
264 
265 #define skb_gro_checksum_simple_validate(skb)				\
266 	__skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
267 
268 static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
269 {
270 	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
271 		!NAPI_GRO_CB(skb)->csum_valid);
272 }
273 
274 static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
275 					      __wsum pseudo)
276 {
277 	NAPI_GRO_CB(skb)->csum = ~pseudo;
278 	NAPI_GRO_CB(skb)->csum_valid = 1;
279 }
280 
281 #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo)	\
282 do {									\
283 	if (__skb_gro_checksum_convert_check(skb))			\
284 		__skb_gro_checksum_convert(skb, 			\
285 					   compute_pseudo(skb, proto));	\
286 } while (0)
287 
288 struct gro_remcsum {
289 	int offset;
290 	__wsum delta;
291 };
292 
293 static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
294 {
295 	grc->offset = 0;
296 	grc->delta = 0;
297 }
298 
299 static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
300 					    unsigned int off, size_t hdrlen,
301 					    int start, int offset,
302 					    struct gro_remcsum *grc,
303 					    bool nopartial)
304 {
305 	__wsum delta;
306 	size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
307 
308 	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
309 
310 	if (!nopartial) {
311 		NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
312 		return ptr;
313 	}
314 
315 	ptr = skb_gro_header(skb, off + plen, off);
316 	if (!ptr)
317 		return NULL;
318 
319 	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
320 			       start, offset);
321 
322 	/* Adjust skb->csum since we changed the packet */
323 	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
324 
325 	grc->offset = off + hdrlen + offset;
326 	grc->delta = delta;
327 
328 	return ptr;
329 }
330 
331 static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
332 					   struct gro_remcsum *grc)
333 {
334 	void *ptr;
335 	size_t plen = grc->offset + sizeof(u16);
336 
337 	if (!grc->delta)
338 		return;
339 
340 	ptr = skb_gro_header(skb, plen, grc->offset);
341 	if (!ptr)
342 		return;
343 
344 	remcsum_unadjust((__sum16 *)ptr, grc->delta);
345 }
346 
347 #ifdef CONFIG_XFRM_OFFLOAD
348 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
349 {
350 	if (PTR_ERR(pp) != -EINPROGRESS)
351 		NAPI_GRO_CB(skb)->flush |= flush;
352 }
353 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
354 					       struct sk_buff *pp,
355 					       int flush,
356 					       struct gro_remcsum *grc)
357 {
358 	if (PTR_ERR(pp) != -EINPROGRESS) {
359 		NAPI_GRO_CB(skb)->flush |= flush;
360 		skb_gro_remcsum_cleanup(skb, grc);
361 		skb->remcsum_offload = 0;
362 	}
363 }
364 #else
365 static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
366 {
367 	NAPI_GRO_CB(skb)->flush |= flush;
368 }
369 static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
370 					       struct sk_buff *pp,
371 					       int flush,
372 					       struct gro_remcsum *grc)
373 {
374 	NAPI_GRO_CB(skb)->flush |= flush;
375 	skb_gro_remcsum_cleanup(skb, grc);
376 	skb->remcsum_offload = 0;
377 }
378 #endif
379 
380 INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
381 							   struct sk_buff *));
382 INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
383 INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
384 							   struct sk_buff *));
385 INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
386 
387 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *,
388 							   struct sk_buff *));
389 INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
390 
391 INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *,
392 							   struct sk_buff *));
393 INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int));
394 
395 #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb)	\
396 ({								\
397 	unlikely(gro_recursion_inc_test(skb)) ?			\
398 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
399 		INDIRECT_CALL_INET(cb, f2, f1, head, skb);	\
400 })
401 
402 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
403 				struct udphdr *uh, struct sock *sk);
404 int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
405 
406 static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
407 {
408 	struct udphdr *uh;
409 	unsigned int hlen, off;
410 
411 	off  = skb_gro_offset(skb);
412 	hlen = off + sizeof(*uh);
413 	uh   = skb_gro_header(skb, hlen, off);
414 
415 	return uh;
416 }
417 
418 static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto)
419 {
420 	const struct ipv6hdr *iph = skb_gro_network_header(skb);
421 
422 	return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr,
423 					    skb_gro_len(skb), proto, 0));
424 }
425 
426 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
427 
428 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
429 static inline void gro_normal_list(struct napi_struct *napi)
430 {
431 	if (!napi->rx_count)
432 		return;
433 	netif_receive_skb_list_internal(&napi->rx_list);
434 	INIT_LIST_HEAD(&napi->rx_list);
435 	napi->rx_count = 0;
436 }
437 
438 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
439  * pass the whole batch up to the stack.
440  */
441 static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
442 {
443 	list_add_tail(&skb->list, &napi->rx_list);
444 	napi->rx_count += segs;
445 	if (napi->rx_count >= READ_ONCE(gro_normal_batch))
446 		gro_normal_list(napi);
447 }
448 
449 
450 #endif /* _NET_IPV6_GRO_H */
451