1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Routines having to do with the 'struct sk_buff' memory handlers.
4 *
5 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
6 * Florian La Roche <rzsfl@rz.uni-sb.de>
7 *
8 * Fixes:
9 * Alan Cox : Fixed the worst of the load
10 * balancer bugs.
11 * Dave Platt : Interrupt stacking fix.
12 * Richard Kooijman : Timestamp fixes.
13 * Alan Cox : Changed buffer format.
14 * Alan Cox : destructor hook for AF_UNIX etc.
15 * Linus Torvalds : Better skb_clone.
16 * Alan Cox : Added skb_copy.
17 * Alan Cox : Added all the changed routines Linus
18 * only put in the headers
19 * Ray VanTassle : Fixed --skb->lock in free
20 * Alan Cox : skb_copy copy arp field
21 * Andi Kleen : slabified it.
22 * Robert Olsson : Removed skb_head_pool
23 *
24 * NOTE:
25 * The __skb_ routines should be called with interrupts
26 * disabled, or you better be *real* sure that the operation is atomic
27 * with respect to whatever list is being frobbed (e.g. via lock_sock()
28 * or via disabling bottom half handlers, etc).
29 */
30
31 /*
32 * The functions in this file will not compile correctly with gcc 2.4.x
33 */
34
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37 #include <linux/module.h>
38 #include <linux/types.h>
39 #include <linux/kernel.h>
40 #include <linux/mm.h>
41 #include <linux/interrupt.h>
42 #include <linux/in.h>
43 #include <linux/inet.h>
44 #include <linux/slab.h>
45 #include <linux/tcp.h>
46 #include <linux/udp.h>
47 #include <linux/sctp.h>
48 #include <linux/netdevice.h>
49 #ifdef CONFIG_NET_CLS_ACT
50 #include <net/pkt_sched.h>
51 #endif
52 #include <linux/string.h>
53 #include <linux/skbuff.h>
54 #include <linux/skbuff_ref.h>
55 #include <linux/splice.h>
56 #include <linux/cache.h>
57 #include <linux/rtnetlink.h>
58 #include <linux/init.h>
59 #include <linux/scatterlist.h>
60 #include <linux/errqueue.h>
61 #include <linux/prefetch.h>
62 #include <linux/bitfield.h>
63 #include <linux/if_vlan.h>
64 #include <linux/mpls.h>
65 #include <linux/kcov.h>
66 #include <linux/iov_iter.h>
67
68 #include <net/protocol.h>
69 #include <net/dst.h>
70 #include <net/sock.h>
71 #include <net/checksum.h>
72 #include <net/gso.h>
73 #include <net/hotdata.h>
74 #include <net/ip6_checksum.h>
75 #include <net/xfrm.h>
76 #include <net/mpls.h>
77 #include <net/mptcp.h>
78 #include <net/mctp.h>
79 #include <net/page_pool/helpers.h>
80 #include <net/dropreason.h>
81
82 #include <linux/uaccess.h>
83 #include <trace/events/skb.h>
84 #include <linux/highmem.h>
85 #include <linux/capability.h>
86 #include <linux/user_namespace.h>
87 #include <linux/indirect_call_wrapper.h>
88 #include <linux/textsearch.h>
89
90 #include "dev.h"
91 #include "sock_destructor.h"
92
93 #ifdef CONFIG_SKB_EXTENSIONS
94 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
95 #endif
96
97 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)
98
99 /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
100 * This should ensure that SKB_SMALL_HEAD_HEADROOM is a unique
101 * size, and we can differentiate heads from skb_small_head_cache
102 * vs system slabs by looking at their size (skb_end_offset()).
103 */
104 #define SKB_SMALL_HEAD_CACHE_SIZE \
105 (is_power_of_2(SKB_SMALL_HEAD_SIZE) ? \
106 (SKB_SMALL_HEAD_SIZE + L1_CACHE_BYTES) : \
107 SKB_SMALL_HEAD_SIZE)
108
109 #define SKB_SMALL_HEAD_HEADROOM \
110 SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
111
112 /* kcm_write_msgs() relies on casting paged frags to bio_vec to use
113 * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the
114 * netmem is a page.
115 */
116 static_assert(offsetof(struct bio_vec, bv_page) ==
117 offsetof(skb_frag_t, netmem));
118 static_assert(sizeof_field(struct bio_vec, bv_page) ==
119 sizeof_field(skb_frag_t, netmem));
120
121 static_assert(offsetof(struct bio_vec, bv_len) == offsetof(skb_frag_t, len));
122 static_assert(sizeof_field(struct bio_vec, bv_len) ==
123 sizeof_field(skb_frag_t, len));
124
125 static_assert(offsetof(struct bio_vec, bv_offset) ==
126 offsetof(skb_frag_t, offset));
127 static_assert(sizeof_field(struct bio_vec, bv_offset) ==
128 sizeof_field(skb_frag_t, offset));
129
130 #undef FN
131 #define FN(reason) [SKB_DROP_REASON_##reason] = #reason,
132 static const char * const drop_reasons[] = {
133 [SKB_CONSUMED] = "CONSUMED",
134 DEFINE_DROP_REASON(FN, FN)
135 };
136
137 static const struct drop_reason_list drop_reasons_core = {
138 .reasons = drop_reasons,
139 .n_reasons = ARRAY_SIZE(drop_reasons),
140 };
141
142 const struct drop_reason_list __rcu *
143 drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_NUM] = {
144 [SKB_DROP_REASON_SUBSYS_CORE] = RCU_INITIALIZER(&drop_reasons_core),
145 };
146 EXPORT_SYMBOL(drop_reasons_by_subsys);
147
148 /**
149 * drop_reasons_register_subsys - register another drop reason subsystem
150 * @subsys: the subsystem to register, must not be the core
151 * @list: the list of drop reasons within the subsystem, must point to
152 * a statically initialized list
153 */
drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,const struct drop_reason_list * list)154 void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,
155 const struct drop_reason_list *list)
156 {
157 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
158 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
159 "invalid subsystem %d\n", subsys))
160 return;
161
162 /* must point to statically allocated memory, so INIT is OK */
163 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], list);
164 }
165 EXPORT_SYMBOL_GPL(drop_reasons_register_subsys);
166
167 /**
168 * drop_reasons_unregister_subsys - unregister a drop reason subsystem
169 * @subsys: the subsystem to remove, must not be the core
170 *
171 * Note: This will synchronize_rcu() to ensure no users when it returns.
172 */
drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys)173 void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys)
174 {
175 if (WARN(subsys <= SKB_DROP_REASON_SUBSYS_CORE ||
176 subsys >= ARRAY_SIZE(drop_reasons_by_subsys),
177 "invalid subsystem %d\n", subsys))
178 return;
179
180 RCU_INIT_POINTER(drop_reasons_by_subsys[subsys], NULL);
181
182 synchronize_rcu();
183 }
184 EXPORT_SYMBOL_GPL(drop_reasons_unregister_subsys);
185
186 /**
187 * skb_panic - private function for out-of-line support
188 * @skb: buffer
189 * @sz: size
190 * @addr: address
191 * @msg: skb_over_panic or skb_under_panic
192 *
193 * Out-of-line support for skb_put() and skb_push().
194 * Called via the wrapper skb_over_panic() or skb_under_panic().
195 * Keep out of line to prevent kernel bloat.
196 * __builtin_return_address is not used because it is not always reliable.
197 */
skb_panic(struct sk_buff * skb,unsigned int sz,void * addr,const char msg[])198 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
199 const char msg[])
200 {
201 pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
202 msg, addr, skb->len, sz, skb->head, skb->data,
203 (unsigned long)skb->tail, (unsigned long)skb->end,
204 skb->dev ? skb->dev->name : "<NULL>");
205 BUG();
206 }
207
skb_over_panic(struct sk_buff * skb,unsigned int sz,void * addr)208 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
209 {
210 skb_panic(skb, sz, addr, __func__);
211 }
212
skb_under_panic(struct sk_buff * skb,unsigned int sz,void * addr)213 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
214 {
215 skb_panic(skb, sz, addr, __func__);
216 }
217
218 #define NAPI_SKB_CACHE_SIZE 64
219 #define NAPI_SKB_CACHE_BULK 16
220 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2)
221
222 #if PAGE_SIZE == SZ_4K
223
224 #define NAPI_HAS_SMALL_PAGE_FRAG 1
225 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc)
226
227 /* specialized page frag allocator using a single order 0 page
228 * and slicing it into 1K sized fragment. Constrained to systems
229 * with a very limited amount of 1K fragments fitting a single
230 * page - to avoid excessive truesize underestimation
231 */
232
233 struct page_frag_1k {
234 void *va;
235 u16 offset;
236 bool pfmemalloc;
237 };
238
page_frag_alloc_1k(struct page_frag_1k * nc,gfp_t gfp)239 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp)
240 {
241 struct page *page;
242 int offset;
243
244 offset = nc->offset - SZ_1K;
245 if (likely(offset >= 0))
246 goto use_frag;
247
248 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
249 if (!page)
250 return NULL;
251
252 nc->va = page_address(page);
253 nc->pfmemalloc = page_is_pfmemalloc(page);
254 offset = PAGE_SIZE - SZ_1K;
255 page_ref_add(page, offset / SZ_1K);
256
257 use_frag:
258 nc->offset = offset;
259 return nc->va + offset;
260 }
261 #else
262
263 /* the small page is actually unused in this build; add dummy helpers
264 * to please the compiler and avoid later preprocessor's conditionals
265 */
266 #define NAPI_HAS_SMALL_PAGE_FRAG 0
267 #define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false
268
269 struct page_frag_1k {
270 };
271
page_frag_alloc_1k(struct page_frag_1k * nc,gfp_t gfp_mask)272 static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask)
273 {
274 return NULL;
275 }
276
277 #endif
278
279 struct napi_alloc_cache {
280 struct page_frag_cache page;
281 struct page_frag_1k page_small;
282 unsigned int skb_count;
283 void *skb_cache[NAPI_SKB_CACHE_SIZE];
284 };
285
286 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
287 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
288
289 /* Double check that napi_get_frags() allocates skbs with
290 * skb->head being backed by slab, not a page fragment.
291 * This is to make sure bug fixed in 3226b158e67c
292 * ("net: avoid 32 x truesize under-estimation for tiny skbs")
293 * does not accidentally come back.
294 */
napi_get_frags_check(struct napi_struct * napi)295 void napi_get_frags_check(struct napi_struct *napi)
296 {
297 struct sk_buff *skb;
298
299 local_bh_disable();
300 skb = napi_get_frags(napi);
301 WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag);
302 napi_free_frags(napi);
303 local_bh_enable();
304 }
305
__napi_alloc_frag_align(unsigned int fragsz,unsigned int align_mask)306 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
307 {
308 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
309
310 fragsz = SKB_DATA_ALIGN(fragsz);
311
312 return __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC,
313 align_mask);
314 }
315 EXPORT_SYMBOL(__napi_alloc_frag_align);
316
__netdev_alloc_frag_align(unsigned int fragsz,unsigned int align_mask)317 void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
318 {
319 void *data;
320
321 fragsz = SKB_DATA_ALIGN(fragsz);
322 if (in_hardirq() || irqs_disabled()) {
323 struct page_frag_cache *nc = this_cpu_ptr(&netdev_alloc_cache);
324
325 data = __page_frag_alloc_align(nc, fragsz, GFP_ATOMIC,
326 align_mask);
327 } else {
328 struct napi_alloc_cache *nc;
329
330 local_bh_disable();
331 nc = this_cpu_ptr(&napi_alloc_cache);
332 data = __page_frag_alloc_align(&nc->page, fragsz, GFP_ATOMIC,
333 align_mask);
334 local_bh_enable();
335 }
336 return data;
337 }
338 EXPORT_SYMBOL(__netdev_alloc_frag_align);
339
napi_skb_cache_get(void)340 static struct sk_buff *napi_skb_cache_get(void)
341 {
342 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
343 struct sk_buff *skb;
344
345 if (unlikely(!nc->skb_count)) {
346 nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
347 GFP_ATOMIC,
348 NAPI_SKB_CACHE_BULK,
349 nc->skb_cache);
350 if (unlikely(!nc->skb_count))
351 return NULL;
352 }
353
354 skb = nc->skb_cache[--nc->skb_count];
355 kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache));
356
357 return skb;
358 }
359
__finalize_skb_around(struct sk_buff * skb,void * data,unsigned int size)360 static inline void __finalize_skb_around(struct sk_buff *skb, void *data,
361 unsigned int size)
362 {
363 struct skb_shared_info *shinfo;
364
365 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
366
367 /* Assumes caller memset cleared SKB */
368 skb->truesize = SKB_TRUESIZE(size);
369 refcount_set(&skb->users, 1);
370 skb->head = data;
371 skb->data = data;
372 skb_reset_tail_pointer(skb);
373 skb_set_end_offset(skb, size);
374 skb->mac_header = (typeof(skb->mac_header))~0U;
375 skb->transport_header = (typeof(skb->transport_header))~0U;
376 skb->alloc_cpu = raw_smp_processor_id();
377 /* make sure we initialize shinfo sequentially */
378 shinfo = skb_shinfo(skb);
379 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
380 atomic_set(&shinfo->dataref, 1);
381
382 skb_set_kcov_handle(skb, kcov_common_handle());
383 }
384
__slab_build_skb(struct sk_buff * skb,void * data,unsigned int * size)385 static inline void *__slab_build_skb(struct sk_buff *skb, void *data,
386 unsigned int *size)
387 {
388 void *resized;
389
390 /* Must find the allocation size (and grow it to match). */
391 *size = ksize(data);
392 /* krealloc() will immediately return "data" when
393 * "ksize(data)" is requested: it is the existing upper
394 * bounds. As a result, GFP_ATOMIC will be ignored. Note
395 * that this "new" pointer needs to be passed back to the
396 * caller for use so the __alloc_size hinting will be
397 * tracked correctly.
398 */
399 resized = krealloc(data, *size, GFP_ATOMIC);
400 WARN_ON_ONCE(resized != data);
401 return resized;
402 }
403
404 /* build_skb() variant which can operate on slab buffers.
405 * Note that this should be used sparingly as slab buffers
406 * cannot be combined efficiently by GRO!
407 */
slab_build_skb(void * data)408 struct sk_buff *slab_build_skb(void *data)
409 {
410 struct sk_buff *skb;
411 unsigned int size;
412
413 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
414 if (unlikely(!skb))
415 return NULL;
416
417 memset(skb, 0, offsetof(struct sk_buff, tail));
418 data = __slab_build_skb(skb, data, &size);
419 __finalize_skb_around(skb, data, size);
420
421 return skb;
422 }
423 EXPORT_SYMBOL(slab_build_skb);
424
425 /* Caller must provide SKB that is memset cleared */
__build_skb_around(struct sk_buff * skb,void * data,unsigned int frag_size)426 static void __build_skb_around(struct sk_buff *skb, void *data,
427 unsigned int frag_size)
428 {
429 unsigned int size = frag_size;
430
431 /* frag_size == 0 is considered deprecated now. Callers
432 * using slab buffer should use slab_build_skb() instead.
433 */
434 if (WARN_ONCE(size == 0, "Use slab_build_skb() instead"))
435 data = __slab_build_skb(skb, data, &size);
436
437 __finalize_skb_around(skb, data, size);
438 }
439
440 /**
441 * __build_skb - build a network buffer
442 * @data: data buffer provided by caller
443 * @frag_size: size of data (must not be 0)
444 *
445 * Allocate a new &sk_buff. Caller provides space holding head and
446 * skb_shared_info. @data must have been allocated from the page
447 * allocator or vmalloc(). (A @frag_size of 0 to indicate a kmalloc()
448 * allocation is deprecated, and callers should use slab_build_skb()
449 * instead.)
450 * The return is the new skb buffer.
451 * On a failure the return is %NULL, and @data is not freed.
452 * Notes :
453 * Before IO, driver allocates only data buffer where NIC put incoming frame
454 * Driver should add room at head (NET_SKB_PAD) and
455 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
456 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
457 * before giving packet to stack.
458 * RX rings only contains data buffers, not full skbs.
459 */
__build_skb(void * data,unsigned int frag_size)460 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
461 {
462 struct sk_buff *skb;
463
464 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
465 if (unlikely(!skb))
466 return NULL;
467
468 memset(skb, 0, offsetof(struct sk_buff, tail));
469 __build_skb_around(skb, data, frag_size);
470
471 return skb;
472 }
473
474 /* build_skb() is wrapper over __build_skb(), that specifically
475 * takes care of skb->head and skb->pfmemalloc
476 */
build_skb(void * data,unsigned int frag_size)477 struct sk_buff *build_skb(void *data, unsigned int frag_size)
478 {
479 struct sk_buff *skb = __build_skb(data, frag_size);
480
481 if (likely(skb && frag_size)) {
482 skb->head_frag = 1;
483 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
484 }
485 return skb;
486 }
487 EXPORT_SYMBOL(build_skb);
488
489 /**
490 * build_skb_around - build a network buffer around provided skb
491 * @skb: sk_buff provide by caller, must be memset cleared
492 * @data: data buffer provided by caller
493 * @frag_size: size of data
494 */
build_skb_around(struct sk_buff * skb,void * data,unsigned int frag_size)495 struct sk_buff *build_skb_around(struct sk_buff *skb,
496 void *data, unsigned int frag_size)
497 {
498 if (unlikely(!skb))
499 return NULL;
500
501 __build_skb_around(skb, data, frag_size);
502
503 if (frag_size) {
504 skb->head_frag = 1;
505 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
506 }
507 return skb;
508 }
509 EXPORT_SYMBOL(build_skb_around);
510
511 /**
512 * __napi_build_skb - build a network buffer
513 * @data: data buffer provided by caller
514 * @frag_size: size of data
515 *
516 * Version of __build_skb() that uses NAPI percpu caches to obtain
517 * skbuff_head instead of inplace allocation.
518 *
519 * Returns a new &sk_buff on success, %NULL on allocation failure.
520 */
__napi_build_skb(void * data,unsigned int frag_size)521 static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size)
522 {
523 struct sk_buff *skb;
524
525 skb = napi_skb_cache_get();
526 if (unlikely(!skb))
527 return NULL;
528
529 memset(skb, 0, offsetof(struct sk_buff, tail));
530 __build_skb_around(skb, data, frag_size);
531
532 return skb;
533 }
534
535 /**
536 * napi_build_skb - build a network buffer
537 * @data: data buffer provided by caller
538 * @frag_size: size of data
539 *
540 * Version of __napi_build_skb() that takes care of skb->head_frag
541 * and skb->pfmemalloc when the data is a page or page fragment.
542 *
543 * Returns a new &sk_buff on success, %NULL on allocation failure.
544 */
napi_build_skb(void * data,unsigned int frag_size)545 struct sk_buff *napi_build_skb(void *data, unsigned int frag_size)
546 {
547 struct sk_buff *skb = __napi_build_skb(data, frag_size);
548
549 if (likely(skb) && frag_size) {
550 skb->head_frag = 1;
551 skb_propagate_pfmemalloc(virt_to_head_page(data), skb);
552 }
553
554 return skb;
555 }
556 EXPORT_SYMBOL(napi_build_skb);
557
558 /*
559 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
560 * the caller if emergency pfmemalloc reserves are being used. If it is and
561 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
562 * may be used. Otherwise, the packet data may be discarded until enough
563 * memory is free
564 */
kmalloc_reserve(unsigned int * size,gfp_t flags,int node,bool * pfmemalloc)565 static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
566 bool *pfmemalloc)
567 {
568 bool ret_pfmemalloc = false;
569 size_t obj_size;
570 void *obj;
571
572 obj_size = SKB_HEAD_ALIGN(*size);
573 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
574 !(flags & KMALLOC_NOT_NORMAL_BITS)) {
575 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
576 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
577 node);
578 *size = SKB_SMALL_HEAD_CACHE_SIZE;
579 if (obj || !(gfp_pfmemalloc_allowed(flags)))
580 goto out;
581 /* Try again but now we are using pfmemalloc reserves */
582 ret_pfmemalloc = true;
583 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node);
584 goto out;
585 }
586
587 obj_size = kmalloc_size_roundup(obj_size);
588 /* The following cast might truncate high-order bits of obj_size, this
589 * is harmless because kmalloc(obj_size >= 2^32) will fail anyway.
590 */
591 *size = (unsigned int)obj_size;
592
593 /*
594 * Try a regular allocation, when that fails and we're not entitled
595 * to the reserves, fail.
596 */
597 obj = kmalloc_node_track_caller(obj_size,
598 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
599 node);
600 if (obj || !(gfp_pfmemalloc_allowed(flags)))
601 goto out;
602
603 /* Try again but now we are using pfmemalloc reserves */
604 ret_pfmemalloc = true;
605 obj = kmalloc_node_track_caller(obj_size, flags, node);
606
607 out:
608 if (pfmemalloc)
609 *pfmemalloc = ret_pfmemalloc;
610
611 return obj;
612 }
613
614 /* Allocate a new skbuff. We do this ourselves so we can fill in a few
615 * 'private' fields and also do memory statistics to find all the
616 * [BEEP] leaks.
617 *
618 */
619
620 /**
621 * __alloc_skb - allocate a network buffer
622 * @size: size to allocate
623 * @gfp_mask: allocation mask
624 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
625 * instead of head cache and allocate a cloned (child) skb.
626 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
627 * allocations in case the data is required for writeback
628 * @node: numa node to allocate memory on
629 *
630 * Allocate a new &sk_buff. The returned buffer has no headroom and a
631 * tail room of at least size bytes. The object has a reference count
632 * of one. The return is the buffer. On a failure the return is %NULL.
633 *
634 * Buffers may only be allocated from interrupts using a @gfp_mask of
635 * %GFP_ATOMIC.
636 */
__alloc_skb(unsigned int size,gfp_t gfp_mask,int flags,int node)637 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
638 int flags, int node)
639 {
640 struct kmem_cache *cache;
641 struct sk_buff *skb;
642 bool pfmemalloc;
643 u8 *data;
644
645 cache = (flags & SKB_ALLOC_FCLONE)
646 ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;
647
648 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
649 gfp_mask |= __GFP_MEMALLOC;
650
651 /* Get the HEAD */
652 if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI &&
653 likely(node == NUMA_NO_NODE || node == numa_mem_id()))
654 skb = napi_skb_cache_get();
655 else
656 skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node);
657 if (unlikely(!skb))
658 return NULL;
659 prefetchw(skb);
660
661 /* We do our best to align skb_shared_info on a separate cache
662 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
663 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
664 * Both skb->head and skb_shared_info are cache line aligned.
665 */
666 data = kmalloc_reserve(&size, gfp_mask, node, &pfmemalloc);
667 if (unlikely(!data))
668 goto nodata;
669 /* kmalloc_size_roundup() might give us more room than requested.
670 * Put skb_shared_info exactly at the end of allocated zone,
671 * to allow max possible filling before reallocation.
672 */
673 prefetchw(data + SKB_WITH_OVERHEAD(size));
674
675 /*
676 * Only clear those fields we need to clear, not those that we will
677 * actually initialise below. Hence, don't put any more fields after
678 * the tail pointer in struct sk_buff!
679 */
680 memset(skb, 0, offsetof(struct sk_buff, tail));
681 __build_skb_around(skb, data, size);
682 skb->pfmemalloc = pfmemalloc;
683
684 if (flags & SKB_ALLOC_FCLONE) {
685 struct sk_buff_fclones *fclones;
686
687 fclones = container_of(skb, struct sk_buff_fclones, skb1);
688
689 skb->fclone = SKB_FCLONE_ORIG;
690 refcount_set(&fclones->fclone_ref, 1);
691 }
692
693 return skb;
694
695 nodata:
696 kmem_cache_free(cache, skb);
697 return NULL;
698 }
699 EXPORT_SYMBOL(__alloc_skb);
700
701 /**
702 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
703 * @dev: network device to receive on
704 * @len: length to allocate
705 * @gfp_mask: get_free_pages mask, passed to alloc_skb
706 *
707 * Allocate a new &sk_buff and assign it a usage count of one. The
708 * buffer has NET_SKB_PAD headroom built in. Users should allocate
709 * the headroom they think they need without accounting for the
710 * built in space. The built in space is used for optimisations.
711 *
712 * %NULL is returned if there is no free memory.
713 */
__netdev_alloc_skb(struct net_device * dev,unsigned int len,gfp_t gfp_mask)714 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
715 gfp_t gfp_mask)
716 {
717 struct page_frag_cache *nc;
718 struct sk_buff *skb;
719 bool pfmemalloc;
720 void *data;
721
722 len += NET_SKB_PAD;
723
724 /* If requested length is either too small or too big,
725 * we use kmalloc() for skb->head allocation.
726 */
727 if (len <= SKB_WITH_OVERHEAD(1024) ||
728 len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
729 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
730 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
731 if (!skb)
732 goto skb_fail;
733 goto skb_success;
734 }
735
736 len = SKB_HEAD_ALIGN(len);
737
738 if (sk_memalloc_socks())
739 gfp_mask |= __GFP_MEMALLOC;
740
741 if (in_hardirq() || irqs_disabled()) {
742 nc = this_cpu_ptr(&netdev_alloc_cache);
743 data = page_frag_alloc(nc, len, gfp_mask);
744 pfmemalloc = nc->pfmemalloc;
745 } else {
746 local_bh_disable();
747 nc = this_cpu_ptr(&napi_alloc_cache.page);
748 data = page_frag_alloc(nc, len, gfp_mask);
749 pfmemalloc = nc->pfmemalloc;
750 local_bh_enable();
751 }
752
753 if (unlikely(!data))
754 return NULL;
755
756 skb = __build_skb(data, len);
757 if (unlikely(!skb)) {
758 skb_free_frag(data);
759 return NULL;
760 }
761
762 if (pfmemalloc)
763 skb->pfmemalloc = 1;
764 skb->head_frag = 1;
765
766 skb_success:
767 skb_reserve(skb, NET_SKB_PAD);
768 skb->dev = dev;
769
770 skb_fail:
771 return skb;
772 }
773 EXPORT_SYMBOL(__netdev_alloc_skb);
774
775 /**
776 * napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
777 * @napi: napi instance this buffer was allocated for
778 * @len: length to allocate
779 *
780 * Allocate a new sk_buff for use in NAPI receive. This buffer will
781 * attempt to allocate the head from a special reserved region used
782 * only for NAPI Rx allocation. By doing this we can save several
783 * CPU cycles by avoiding having to disable and re-enable IRQs.
784 *
785 * %NULL is returned if there is no free memory.
786 */
napi_alloc_skb(struct napi_struct * napi,unsigned int len)787 struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int len)
788 {
789 gfp_t gfp_mask = GFP_ATOMIC | __GFP_NOWARN;
790 struct napi_alloc_cache *nc;
791 struct sk_buff *skb;
792 bool pfmemalloc;
793 void *data;
794
795 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
796 len += NET_SKB_PAD + NET_IP_ALIGN;
797
798 /* If requested length is either too small or too big,
799 * we use kmalloc() for skb->head allocation.
800 * When the small frag allocator is available, prefer it over kmalloc
801 * for small fragments
802 */
803 if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) ||
804 len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
805 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
806 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
807 NUMA_NO_NODE);
808 if (!skb)
809 goto skb_fail;
810 goto skb_success;
811 }
812
813 nc = this_cpu_ptr(&napi_alloc_cache);
814
815 if (sk_memalloc_socks())
816 gfp_mask |= __GFP_MEMALLOC;
817
818 if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) {
819 /* we are artificially inflating the allocation size, but
820 * that is not as bad as it may look like, as:
821 * - 'len' less than GRO_MAX_HEAD makes little sense
822 * - On most systems, larger 'len' values lead to fragment
823 * size above 512 bytes
824 * - kmalloc would use the kmalloc-1k slab for such values
825 * - Builds with smaller GRO_MAX_HEAD will very likely do
826 * little networking, as that implies no WiFi and no
827 * tunnels support, and 32 bits arches.
828 */
829 len = SZ_1K;
830
831 data = page_frag_alloc_1k(&nc->page_small, gfp_mask);
832 pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small);
833 } else {
834 len = SKB_HEAD_ALIGN(len);
835
836 data = page_frag_alloc(&nc->page, len, gfp_mask);
837 pfmemalloc = nc->page.pfmemalloc;
838 }
839
840 if (unlikely(!data))
841 return NULL;
842
843 skb = __napi_build_skb(data, len);
844 if (unlikely(!skb)) {
845 skb_free_frag(data);
846 return NULL;
847 }
848
849 if (pfmemalloc)
850 skb->pfmemalloc = 1;
851 skb->head_frag = 1;
852
853 skb_success:
854 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
855 skb->dev = napi->dev;
856
857 skb_fail:
858 return skb;
859 }
860 EXPORT_SYMBOL(napi_alloc_skb);
861
skb_add_rx_frag_netmem(struct sk_buff * skb,int i,netmem_ref netmem,int off,int size,unsigned int truesize)862 void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
863 int off, int size, unsigned int truesize)
864 {
865 DEBUG_NET_WARN_ON_ONCE(size > truesize);
866
867 skb_fill_netmem_desc(skb, i, netmem, off, size);
868 skb->len += size;
869 skb->data_len += size;
870 skb->truesize += truesize;
871 }
872 EXPORT_SYMBOL(skb_add_rx_frag_netmem);
873
skb_coalesce_rx_frag(struct sk_buff * skb,int i,int size,unsigned int truesize)874 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
875 unsigned int truesize)
876 {
877 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
878
879 DEBUG_NET_WARN_ON_ONCE(size > truesize);
880
881 skb_frag_size_add(frag, size);
882 skb->len += size;
883 skb->data_len += size;
884 skb->truesize += truesize;
885 }
886 EXPORT_SYMBOL(skb_coalesce_rx_frag);
887
skb_drop_list(struct sk_buff ** listp)888 static void skb_drop_list(struct sk_buff **listp)
889 {
890 kfree_skb_list(*listp);
891 *listp = NULL;
892 }
893
skb_drop_fraglist(struct sk_buff * skb)894 static inline void skb_drop_fraglist(struct sk_buff *skb)
895 {
896 skb_drop_list(&skb_shinfo(skb)->frag_list);
897 }
898
skb_clone_fraglist(struct sk_buff * skb)899 static void skb_clone_fraglist(struct sk_buff *skb)
900 {
901 struct sk_buff *list;
902
903 skb_walk_frags(skb, list)
904 skb_get(list);
905 }
906
is_pp_page(struct page * page)907 static bool is_pp_page(struct page *page)
908 {
909 return (page->pp_magic & ~0x3UL) == PP_SIGNATURE;
910 }
911
skb_pp_cow_data(struct page_pool * pool,struct sk_buff ** pskb,unsigned int headroom)912 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
913 unsigned int headroom)
914 {
915 #if IS_ENABLED(CONFIG_PAGE_POOL)
916 u32 size, truesize, len, max_head_size, off;
917 struct sk_buff *skb = *pskb, *nskb;
918 int err, i, head_off;
919 void *data;
920
921 /* XDP does not support fraglist so we need to linearize
922 * the skb.
923 */
924 if (skb_has_frag_list(skb))
925 return -EOPNOTSUPP;
926
927 max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom);
928 if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
929 return -ENOMEM;
930
931 size = min_t(u32, skb->len, max_head_size);
932 truesize = SKB_HEAD_ALIGN(size) + headroom;
933 data = page_pool_dev_alloc_va(pool, &truesize);
934 if (!data)
935 return -ENOMEM;
936
937 nskb = napi_build_skb(data, truesize);
938 if (!nskb) {
939 page_pool_free_va(pool, data, true);
940 return -ENOMEM;
941 }
942
943 skb_reserve(nskb, headroom);
944 skb_copy_header(nskb, skb);
945 skb_mark_for_recycle(nskb);
946
947 err = skb_copy_bits(skb, 0, nskb->data, size);
948 if (err) {
949 consume_skb(nskb);
950 return err;
951 }
952 skb_put(nskb, size);
953
954 head_off = skb_headroom(nskb) - skb_headroom(skb);
955 skb_headers_offset_update(nskb, head_off);
956
957 off = size;
958 len = skb->len - off;
959 for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
960 struct page *page;
961 u32 page_off;
962
963 size = min_t(u32, len, PAGE_SIZE);
964 truesize = size;
965
966 page = page_pool_dev_alloc(pool, &page_off, &truesize);
967 if (!page) {
968 consume_skb(nskb);
969 return -ENOMEM;
970 }
971
972 skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
973 err = skb_copy_bits(skb, off, page_address(page) + page_off,
974 size);
975 if (err) {
976 consume_skb(nskb);
977 return err;
978 }
979
980 len -= size;
981 off += size;
982 }
983
984 consume_skb(skb);
985 *pskb = nskb;
986
987 return 0;
988 #else
989 return -EOPNOTSUPP;
990 #endif
991 }
992 EXPORT_SYMBOL(skb_pp_cow_data);
993
skb_cow_data_for_xdp(struct page_pool * pool,struct sk_buff ** pskb,struct bpf_prog * prog)994 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
995 struct bpf_prog *prog)
996 {
997 if (!prog->aux->xdp_has_frags)
998 return -EINVAL;
999
1000 return skb_pp_cow_data(pool, pskb, XDP_PACKET_HEADROOM);
1001 }
1002 EXPORT_SYMBOL(skb_cow_data_for_xdp);
1003
1004 #if IS_ENABLED(CONFIG_PAGE_POOL)
napi_pp_put_page(struct page * page)1005 bool napi_pp_put_page(struct page *page)
1006 {
1007 page = compound_head(page);
1008
1009 /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation
1010 * in order to preserve any existing bits, such as bit 0 for the
1011 * head page of compound page and bit 1 for pfmemalloc page, so
1012 * mask those bits for freeing side when doing below checking,
1013 * and page_is_pfmemalloc() is checked in __page_pool_put_page()
1014 * to avoid recycling the pfmemalloc page.
1015 */
1016 if (unlikely(!is_pp_page(page)))
1017 return false;
1018
1019 page_pool_put_full_page(page->pp, page, false);
1020
1021 return true;
1022 }
1023 EXPORT_SYMBOL(napi_pp_put_page);
1024 #endif
1025
skb_pp_recycle(struct sk_buff * skb,void * data)1026 static bool skb_pp_recycle(struct sk_buff *skb, void *data)
1027 {
1028 if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
1029 return false;
1030 return napi_pp_put_page(virt_to_page(data));
1031 }
1032
1033 /**
1034 * skb_pp_frag_ref() - Increase fragment references of a page pool aware skb
1035 * @skb: page pool aware skb
1036 *
1037 * Increase the fragment reference count (pp_ref_count) of a skb. This is
1038 * intended to gain fragment references only for page pool aware skbs,
1039 * i.e. when skb->pp_recycle is true, and not for fragments in a
1040 * non-pp-recycling skb. It has a fallback to increase references on normal
1041 * pages, as page pool aware skbs may also have normal page fragments.
1042 */
skb_pp_frag_ref(struct sk_buff * skb)1043 static int skb_pp_frag_ref(struct sk_buff *skb)
1044 {
1045 struct skb_shared_info *shinfo;
1046 struct page *head_page;
1047 int i;
1048
1049 if (!skb->pp_recycle)
1050 return -EINVAL;
1051
1052 shinfo = skb_shinfo(skb);
1053
1054 for (i = 0; i < shinfo->nr_frags; i++) {
1055 head_page = compound_head(skb_frag_page(&shinfo->frags[i]));
1056 if (likely(is_pp_page(head_page)))
1057 page_pool_ref_page(head_page);
1058 else
1059 page_ref_inc(head_page);
1060 }
1061 return 0;
1062 }
1063
skb_kfree_head(void * head,unsigned int end_offset)1064 static void skb_kfree_head(void *head, unsigned int end_offset)
1065 {
1066 if (end_offset == SKB_SMALL_HEAD_HEADROOM)
1067 kmem_cache_free(net_hotdata.skb_small_head_cache, head);
1068 else
1069 kfree(head);
1070 }
1071
skb_free_head(struct sk_buff * skb)1072 static void skb_free_head(struct sk_buff *skb)
1073 {
1074 unsigned char *head = skb->head;
1075
1076 if (skb->head_frag) {
1077 if (skb_pp_recycle(skb, head))
1078 return;
1079 skb_free_frag(head);
1080 } else {
1081 skb_kfree_head(head, skb_end_offset(skb));
1082 }
1083 }
1084
skb_release_data(struct sk_buff * skb,enum skb_drop_reason reason)1085 static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason)
1086 {
1087 struct skb_shared_info *shinfo = skb_shinfo(skb);
1088 int i;
1089
1090 if (!skb_data_unref(skb, shinfo))
1091 goto exit;
1092
1093 if (skb_zcopy(skb)) {
1094 bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS;
1095
1096 skb_zcopy_clear(skb, true);
1097 if (skip_unref)
1098 goto free_head;
1099 }
1100
1101 for (i = 0; i < shinfo->nr_frags; i++)
1102 __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle);
1103
1104 free_head:
1105 if (shinfo->frag_list)
1106 kfree_skb_list_reason(shinfo->frag_list, reason);
1107
1108 skb_free_head(skb);
1109 exit:
1110 /* When we clone an SKB we copy the reycling bit. The pp_recycle
1111 * bit is only set on the head though, so in order to avoid races
1112 * while trying to recycle fragments on __skb_frag_unref() we need
1113 * to make one SKB responsible for triggering the recycle path.
1114 * So disable the recycling bit if an SKB is cloned and we have
1115 * additional references to the fragmented part of the SKB.
1116 * Eventually the last SKB will have the recycling bit set and it's
1117 * dataref set to 0, which will trigger the recycling
1118 */
1119 skb->pp_recycle = 0;
1120 }
1121
1122 /*
1123 * Free an skbuff by memory without cleaning the state.
1124 */
kfree_skbmem(struct sk_buff * skb)1125 static void kfree_skbmem(struct sk_buff *skb)
1126 {
1127 struct sk_buff_fclones *fclones;
1128
1129 switch (skb->fclone) {
1130 case SKB_FCLONE_UNAVAILABLE:
1131 kmem_cache_free(net_hotdata.skbuff_cache, skb);
1132 return;
1133
1134 case SKB_FCLONE_ORIG:
1135 fclones = container_of(skb, struct sk_buff_fclones, skb1);
1136
1137 /* We usually free the clone (TX completion) before original skb
1138 * This test would have no chance to be true for the clone,
1139 * while here, branch prediction will be good.
1140 */
1141 if (refcount_read(&fclones->fclone_ref) == 1)
1142 goto fastpath;
1143 break;
1144
1145 default: /* SKB_FCLONE_CLONE */
1146 fclones = container_of(skb, struct sk_buff_fclones, skb2);
1147 break;
1148 }
1149 if (!refcount_dec_and_test(&fclones->fclone_ref))
1150 return;
1151 fastpath:
1152 kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
1153 }
1154
skb_release_head_state(struct sk_buff * skb)1155 void skb_release_head_state(struct sk_buff *skb)
1156 {
1157 skb_dst_drop(skb);
1158 if (skb->destructor) {
1159 DEBUG_NET_WARN_ON_ONCE(in_hardirq());
1160 skb->destructor(skb);
1161 }
1162 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1163 nf_conntrack_put(skb_nfct(skb));
1164 #endif
1165 skb_ext_put(skb);
1166 }
1167
1168 /* Free everything but the sk_buff shell. */
skb_release_all(struct sk_buff * skb,enum skb_drop_reason reason)1169 static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason)
1170 {
1171 skb_release_head_state(skb);
1172 if (likely(skb->head))
1173 skb_release_data(skb, reason);
1174 }
1175
1176 /**
1177 * __kfree_skb - private function
1178 * @skb: buffer
1179 *
1180 * Free an sk_buff. Release anything attached to the buffer.
1181 * Clean the state. This is an internal helper function. Users should
1182 * always call kfree_skb
1183 */
1184
__kfree_skb(struct sk_buff * skb)1185 void __kfree_skb(struct sk_buff *skb)
1186 {
1187 skb_release_all(skb, SKB_DROP_REASON_NOT_SPECIFIED);
1188 kfree_skbmem(skb);
1189 }
1190 EXPORT_SYMBOL(__kfree_skb);
1191
1192 static __always_inline
__kfree_skb_reason(struct sk_buff * skb,enum skb_drop_reason reason)1193 bool __kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
1194 {
1195 if (unlikely(!skb_unref(skb)))
1196 return false;
1197
1198 DEBUG_NET_WARN_ON_ONCE(reason == SKB_NOT_DROPPED_YET ||
1199 u32_get_bits(reason,
1200 SKB_DROP_REASON_SUBSYS_MASK) >=
1201 SKB_DROP_REASON_SUBSYS_NUM);
1202
1203 if (reason == SKB_CONSUMED)
1204 trace_consume_skb(skb, __builtin_return_address(0));
1205 else
1206 trace_kfree_skb(skb, __builtin_return_address(0), reason);
1207 return true;
1208 }
1209
1210 /**
1211 * kfree_skb_reason - free an sk_buff with special reason
1212 * @skb: buffer to free
1213 * @reason: reason why this skb is dropped
1214 *
1215 * Drop a reference to the buffer and free it if the usage count has
1216 * hit zero. Meanwhile, pass the drop reason to 'kfree_skb'
1217 * tracepoint.
1218 */
1219 void __fix_address
kfree_skb_reason(struct sk_buff * skb,enum skb_drop_reason reason)1220 kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
1221 {
1222 if (__kfree_skb_reason(skb, reason))
1223 __kfree_skb(skb);
1224 }
1225 EXPORT_SYMBOL(kfree_skb_reason);
1226
1227 #define KFREE_SKB_BULK_SIZE 16
1228
1229 struct skb_free_array {
1230 unsigned int skb_count;
1231 void *skb_array[KFREE_SKB_BULK_SIZE];
1232 };
1233
kfree_skb_add_bulk(struct sk_buff * skb,struct skb_free_array * sa,enum skb_drop_reason reason)1234 static void kfree_skb_add_bulk(struct sk_buff *skb,
1235 struct skb_free_array *sa,
1236 enum skb_drop_reason reason)
1237 {
1238 /* if SKB is a clone, don't handle this case */
1239 if (unlikely(skb->fclone != SKB_FCLONE_UNAVAILABLE)) {
1240 __kfree_skb(skb);
1241 return;
1242 }
1243
1244 skb_release_all(skb, reason);
1245 sa->skb_array[sa->skb_count++] = skb;
1246
1247 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
1248 kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
1249 sa->skb_array);
1250 sa->skb_count = 0;
1251 }
1252 }
1253
1254 void __fix_address
kfree_skb_list_reason(struct sk_buff * segs,enum skb_drop_reason reason)1255 kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
1256 {
1257 struct skb_free_array sa;
1258
1259 sa.skb_count = 0;
1260
1261 while (segs) {
1262 struct sk_buff *next = segs->next;
1263
1264 if (__kfree_skb_reason(segs, reason)) {
1265 skb_poison_list(segs);
1266 kfree_skb_add_bulk(segs, &sa, reason);
1267 }
1268
1269 segs = next;
1270 }
1271
1272 if (sa.skb_count)
1273 kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
1274 }
1275 EXPORT_SYMBOL(kfree_skb_list_reason);
1276
1277 /* Dump skb information and contents.
1278 *
1279 * Must only be called from net_ratelimit()-ed paths.
1280 *
1281 * Dumps whole packets if full_pkt, only headers otherwise.
1282 */
skb_dump(const char * level,const struct sk_buff * skb,bool full_pkt)1283 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
1284 {
1285 struct skb_shared_info *sh = skb_shinfo(skb);
1286 struct net_device *dev = skb->dev;
1287 struct sock *sk = skb->sk;
1288 struct sk_buff *list_skb;
1289 bool has_mac, has_trans;
1290 int headroom, tailroom;
1291 int i, len, seg_len;
1292
1293 if (full_pkt)
1294 len = skb->len;
1295 else
1296 len = min_t(int, skb->len, MAX_HEADER + 128);
1297
1298 headroom = skb_headroom(skb);
1299 tailroom = skb_tailroom(skb);
1300
1301 has_mac = skb_mac_header_was_set(skb);
1302 has_trans = skb_transport_header_was_set(skb);
1303
1304 printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
1305 "mac=(%d,%d) mac_len=%u net=(%d,%d) trans=%d\n"
1306 "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
1307 "csum(0x%x start=%u offset=%u ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
1308 "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n"
1309 "priority=0x%x mark=0x%x alloc_cpu=%u vlan_all=0x%x\n"
1310 "encapsulation=%d inner(proto=0x%04x, mac=%u, net=%u, trans=%u)\n",
1311 level, skb->len, headroom, skb_headlen(skb), tailroom,
1312 has_mac ? skb->mac_header : -1,
1313 has_mac ? skb_mac_header_len(skb) : -1,
1314 skb->mac_len,
1315 skb->network_header,
1316 has_trans ? skb_network_header_len(skb) : -1,
1317 has_trans ? skb->transport_header : -1,
1318 sh->tx_flags, sh->nr_frags,
1319 sh->gso_size, sh->gso_type, sh->gso_segs,
1320 skb->csum, skb->csum_start, skb->csum_offset, skb->ip_summed,
1321 skb->csum_complete_sw, skb->csum_valid, skb->csum_level,
1322 skb->hash, skb->sw_hash, skb->l4_hash,
1323 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif,
1324 skb->priority, skb->mark, skb->alloc_cpu, skb->vlan_all,
1325 skb->encapsulation, skb->inner_protocol, skb->inner_mac_header,
1326 skb->inner_network_header, skb->inner_transport_header);
1327
1328 if (dev)
1329 printk("%sdev name=%s feat=%pNF\n",
1330 level, dev->name, &dev->features);
1331 if (sk)
1332 printk("%ssk family=%hu type=%u proto=%u\n",
1333 level, sk->sk_family, sk->sk_type, sk->sk_protocol);
1334
1335 if (full_pkt && headroom)
1336 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
1337 16, 1, skb->head, headroom, false);
1338
1339 seg_len = min_t(int, skb_headlen(skb), len);
1340 if (seg_len)
1341 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
1342 16, 1, skb->data, seg_len, false);
1343 len -= seg_len;
1344
1345 if (full_pkt && tailroom)
1346 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
1347 16, 1, skb_tail_pointer(skb), tailroom, false);
1348
1349 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
1350 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1351 u32 p_off, p_len, copied;
1352 struct page *p;
1353 u8 *vaddr;
1354
1355 skb_frag_foreach_page(frag, skb_frag_off(frag),
1356 skb_frag_size(frag), p, p_off, p_len,
1357 copied) {
1358 seg_len = min_t(int, p_len, len);
1359 vaddr = kmap_atomic(p);
1360 print_hex_dump(level, "skb frag: ",
1361 DUMP_PREFIX_OFFSET,
1362 16, 1, vaddr + p_off, seg_len, false);
1363 kunmap_atomic(vaddr);
1364 len -= seg_len;
1365 if (!len)
1366 break;
1367 }
1368 }
1369
1370 if (full_pkt && skb_has_frag_list(skb)) {
1371 printk("skb fraglist:\n");
1372 skb_walk_frags(skb, list_skb)
1373 skb_dump(level, list_skb, true);
1374 }
1375 }
1376 EXPORT_SYMBOL(skb_dump);
1377
1378 /**
1379 * skb_tx_error - report an sk_buff xmit error
1380 * @skb: buffer that triggered an error
1381 *
1382 * Report xmit error if a device callback is tracking this skb.
1383 * skb must be freed afterwards.
1384 */
skb_tx_error(struct sk_buff * skb)1385 void skb_tx_error(struct sk_buff *skb)
1386 {
1387 if (skb) {
1388 skb_zcopy_downgrade_managed(skb);
1389 skb_zcopy_clear(skb, true);
1390 }
1391 }
1392 EXPORT_SYMBOL(skb_tx_error);
1393
1394 #ifdef CONFIG_TRACEPOINTS
1395 /**
1396 * consume_skb - free an skbuff
1397 * @skb: buffer to free
1398 *
1399 * Drop a ref to the buffer and free it if the usage count has hit zero
1400 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
1401 * is being dropped after a failure and notes that
1402 */
consume_skb(struct sk_buff * skb)1403 void consume_skb(struct sk_buff *skb)
1404 {
1405 if (!skb_unref(skb))
1406 return;
1407
1408 trace_consume_skb(skb, __builtin_return_address(0));
1409 __kfree_skb(skb);
1410 }
1411 EXPORT_SYMBOL(consume_skb);
1412 #endif
1413
1414 /**
1415 * __consume_stateless_skb - free an skbuff, assuming it is stateless
1416 * @skb: buffer to free
1417 *
1418 * Alike consume_skb(), but this variant assumes that this is the last
1419 * skb reference and all the head states have been already dropped
1420 */
__consume_stateless_skb(struct sk_buff * skb)1421 void __consume_stateless_skb(struct sk_buff *skb)
1422 {
1423 trace_consume_skb(skb, __builtin_return_address(0));
1424 skb_release_data(skb, SKB_CONSUMED);
1425 kfree_skbmem(skb);
1426 }
1427
napi_skb_cache_put(struct sk_buff * skb)1428 static void napi_skb_cache_put(struct sk_buff *skb)
1429 {
1430 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
1431 u32 i;
1432
1433 if (!kasan_mempool_poison_object(skb))
1434 return;
1435
1436 nc->skb_cache[nc->skb_count++] = skb;
1437
1438 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
1439 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
1440 kasan_mempool_unpoison_object(nc->skb_cache[i],
1441 kmem_cache_size(net_hotdata.skbuff_cache));
1442
1443 kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF,
1444 nc->skb_cache + NAPI_SKB_CACHE_HALF);
1445 nc->skb_count = NAPI_SKB_CACHE_HALF;
1446 }
1447 }
1448
__napi_kfree_skb(struct sk_buff * skb,enum skb_drop_reason reason)1449 void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason)
1450 {
1451 skb_release_all(skb, reason);
1452 napi_skb_cache_put(skb);
1453 }
1454
napi_skb_free_stolen_head(struct sk_buff * skb)1455 void napi_skb_free_stolen_head(struct sk_buff *skb)
1456 {
1457 if (unlikely(skb->slow_gro)) {
1458 nf_reset_ct(skb);
1459 skb_dst_drop(skb);
1460 skb_ext_put(skb);
1461 skb_orphan(skb);
1462 skb->slow_gro = 0;
1463 }
1464 napi_skb_cache_put(skb);
1465 }
1466
napi_consume_skb(struct sk_buff * skb,int budget)1467 void napi_consume_skb(struct sk_buff *skb, int budget)
1468 {
1469 /* Zero budget indicate non-NAPI context called us, like netpoll */
1470 if (unlikely(!budget)) {
1471 dev_consume_skb_any(skb);
1472 return;
1473 }
1474
1475 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
1476
1477 if (!skb_unref(skb))
1478 return;
1479
1480 /* if reaching here SKB is ready to free */
1481 trace_consume_skb(skb, __builtin_return_address(0));
1482
1483 /* if SKB is a clone, don't handle this case */
1484 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
1485 __kfree_skb(skb);
1486 return;
1487 }
1488
1489 skb_release_all(skb, SKB_CONSUMED);
1490 napi_skb_cache_put(skb);
1491 }
1492 EXPORT_SYMBOL(napi_consume_skb);
1493
1494 /* Make sure a field is contained by headers group */
1495 #define CHECK_SKB_FIELD(field) \
1496 BUILD_BUG_ON(offsetof(struct sk_buff, field) != \
1497 offsetof(struct sk_buff, headers.field)); \
1498
__copy_skb_header(struct sk_buff * new,const struct sk_buff * old)1499 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1500 {
1501 new->tstamp = old->tstamp;
1502 /* We do not copy old->sk */
1503 new->dev = old->dev;
1504 memcpy(new->cb, old->cb, sizeof(old->cb));
1505 skb_dst_copy(new, old);
1506 __skb_ext_copy(new, old);
1507 __nf_copy(new, old, false);
1508
1509 /* Note : this field could be in the headers group.
1510 * It is not yet because we do not want to have a 16 bit hole
1511 */
1512 new->queue_mapping = old->queue_mapping;
1513
1514 memcpy(&new->headers, &old->headers, sizeof(new->headers));
1515 CHECK_SKB_FIELD(protocol);
1516 CHECK_SKB_FIELD(csum);
1517 CHECK_SKB_FIELD(hash);
1518 CHECK_SKB_FIELD(priority);
1519 CHECK_SKB_FIELD(skb_iif);
1520 CHECK_SKB_FIELD(vlan_proto);
1521 CHECK_SKB_FIELD(vlan_tci);
1522 CHECK_SKB_FIELD(transport_header);
1523 CHECK_SKB_FIELD(network_header);
1524 CHECK_SKB_FIELD(mac_header);
1525 CHECK_SKB_FIELD(inner_protocol);
1526 CHECK_SKB_FIELD(inner_transport_header);
1527 CHECK_SKB_FIELD(inner_network_header);
1528 CHECK_SKB_FIELD(inner_mac_header);
1529 CHECK_SKB_FIELD(mark);
1530 #ifdef CONFIG_NETWORK_SECMARK
1531 CHECK_SKB_FIELD(secmark);
1532 #endif
1533 #ifdef CONFIG_NET_RX_BUSY_POLL
1534 CHECK_SKB_FIELD(napi_id);
1535 #endif
1536 CHECK_SKB_FIELD(alloc_cpu);
1537 #ifdef CONFIG_XPS
1538 CHECK_SKB_FIELD(sender_cpu);
1539 #endif
1540 #ifdef CONFIG_NET_SCHED
1541 CHECK_SKB_FIELD(tc_index);
1542 #endif
1543
1544 }
1545
1546 /*
1547 * You should not add any new code to this function. Add it to
1548 * __copy_skb_header above instead.
1549 */
__skb_clone(struct sk_buff * n,struct sk_buff * skb)1550 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
1551 {
1552 #define C(x) n->x = skb->x
1553
1554 n->next = n->prev = NULL;
1555 n->sk = NULL;
1556 __copy_skb_header(n, skb);
1557
1558 C(len);
1559 C(data_len);
1560 C(mac_len);
1561 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
1562 n->cloned = 1;
1563 n->nohdr = 0;
1564 n->peeked = 0;
1565 C(pfmemalloc);
1566 C(pp_recycle);
1567 n->destructor = NULL;
1568 C(tail);
1569 C(end);
1570 C(head);
1571 C(head_frag);
1572 C(data);
1573 C(truesize);
1574 refcount_set(&n->users, 1);
1575
1576 atomic_inc(&(skb_shinfo(skb)->dataref));
1577 skb->cloned = 1;
1578
1579 return n;
1580 #undef C
1581 }
1582
1583 /**
1584 * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
1585 * @first: first sk_buff of the msg
1586 */
alloc_skb_for_msg(struct sk_buff * first)1587 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first)
1588 {
1589 struct sk_buff *n;
1590
1591 n = alloc_skb(0, GFP_ATOMIC);
1592 if (!n)
1593 return NULL;
1594
1595 n->len = first->len;
1596 n->data_len = first->len;
1597 n->truesize = first->truesize;
1598
1599 skb_shinfo(n)->frag_list = first;
1600
1601 __copy_skb_header(n, first);
1602 n->destructor = NULL;
1603
1604 return n;
1605 }
1606 EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
1607
1608 /**
1609 * skb_morph - morph one skb into another
1610 * @dst: the skb to receive the contents
1611 * @src: the skb to supply the contents
1612 *
1613 * This is identical to skb_clone except that the target skb is
1614 * supplied by the user.
1615 *
1616 * The target skb is returned upon exit.
1617 */
skb_morph(struct sk_buff * dst,struct sk_buff * src)1618 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
1619 {
1620 skb_release_all(dst, SKB_CONSUMED);
1621 return __skb_clone(dst, src);
1622 }
1623 EXPORT_SYMBOL_GPL(skb_morph);
1624
mm_account_pinned_pages(struct mmpin * mmp,size_t size)1625 int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
1626 {
1627 unsigned long max_pg, num_pg, new_pg, old_pg, rlim;
1628 struct user_struct *user;
1629
1630 if (capable(CAP_IPC_LOCK) || !size)
1631 return 0;
1632
1633 rlim = rlimit(RLIMIT_MEMLOCK);
1634 if (rlim == RLIM_INFINITY)
1635 return 0;
1636
1637 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
1638 max_pg = rlim >> PAGE_SHIFT;
1639 user = mmp->user ? : current_user();
1640
1641 old_pg = atomic_long_read(&user->locked_vm);
1642 do {
1643 new_pg = old_pg + num_pg;
1644 if (new_pg > max_pg)
1645 return -ENOBUFS;
1646 } while (!atomic_long_try_cmpxchg(&user->locked_vm, &old_pg, new_pg));
1647
1648 if (!mmp->user) {
1649 mmp->user = get_uid(user);
1650 mmp->num_pg = num_pg;
1651 } else {
1652 mmp->num_pg += num_pg;
1653 }
1654
1655 return 0;
1656 }
1657 EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
1658
mm_unaccount_pinned_pages(struct mmpin * mmp)1659 void mm_unaccount_pinned_pages(struct mmpin *mmp)
1660 {
1661 if (mmp->user) {
1662 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
1663 free_uid(mmp->user);
1664 }
1665 }
1666 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
1667
msg_zerocopy_alloc(struct sock * sk,size_t size)1668 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
1669 {
1670 struct ubuf_info_msgzc *uarg;
1671 struct sk_buff *skb;
1672
1673 WARN_ON_ONCE(!in_task());
1674
1675 skb = sock_omalloc(sk, 0, GFP_KERNEL);
1676 if (!skb)
1677 return NULL;
1678
1679 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1680 uarg = (void *)skb->cb;
1681 uarg->mmp.user = NULL;
1682
1683 if (mm_account_pinned_pages(&uarg->mmp, size)) {
1684 kfree_skb(skb);
1685 return NULL;
1686 }
1687
1688 uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
1689 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1690 uarg->len = 1;
1691 uarg->bytelen = size;
1692 uarg->zerocopy = 1;
1693 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
1694 refcount_set(&uarg->ubuf.refcnt, 1);
1695 sock_hold(sk);
1696
1697 return &uarg->ubuf;
1698 }
1699
skb_from_uarg(struct ubuf_info_msgzc * uarg)1700 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
1701 {
1702 return container_of((void *)uarg, struct sk_buff, cb);
1703 }
1704
msg_zerocopy_realloc(struct sock * sk,size_t size,struct ubuf_info * uarg)1705 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
1706 struct ubuf_info *uarg)
1707 {
1708 if (uarg) {
1709 struct ubuf_info_msgzc *uarg_zc;
1710 const u32 byte_limit = 1 << 19; /* limit to a few TSO */
1711 u32 bytelen, next;
1712
1713 /* there might be non MSG_ZEROCOPY users */
1714 if (uarg->ops != &msg_zerocopy_ubuf_ops)
1715 return NULL;
1716
1717 /* realloc only when socket is locked (TCP, UDP cork),
1718 * so uarg->len and sk_zckey access is serialized
1719 */
1720 if (!sock_owned_by_user(sk)) {
1721 WARN_ON_ONCE(1);
1722 return NULL;
1723 }
1724
1725 uarg_zc = uarg_to_msgzc(uarg);
1726 bytelen = uarg_zc->bytelen + size;
1727 if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1728 /* TCP can create new skb to attach new uarg */
1729 if (sk->sk_type == SOCK_STREAM)
1730 goto new_alloc;
1731 return NULL;
1732 }
1733
1734 next = (u32)atomic_read(&sk->sk_zckey);
1735 if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
1736 if (mm_account_pinned_pages(&uarg_zc->mmp, size))
1737 return NULL;
1738 uarg_zc->len++;
1739 uarg_zc->bytelen = bytelen;
1740 atomic_set(&sk->sk_zckey, ++next);
1741
1742 /* no extra ref when appending to datagram (MSG_MORE) */
1743 if (sk->sk_type == SOCK_STREAM)
1744 net_zcopy_get(uarg);
1745
1746 return uarg;
1747 }
1748 }
1749
1750 new_alloc:
1751 return msg_zerocopy_alloc(sk, size);
1752 }
1753 EXPORT_SYMBOL_GPL(msg_zerocopy_realloc);
1754
skb_zerocopy_notify_extend(struct sk_buff * skb,u32 lo,u16 len)1755 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1756 {
1757 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1758 u32 old_lo, old_hi;
1759 u64 sum_len;
1760
1761 old_lo = serr->ee.ee_info;
1762 old_hi = serr->ee.ee_data;
1763 sum_len = old_hi - old_lo + 1ULL + len;
1764
1765 if (sum_len >= (1ULL << 32))
1766 return false;
1767
1768 if (lo != old_hi + 1)
1769 return false;
1770
1771 serr->ee.ee_data += len;
1772 return true;
1773 }
1774
__msg_zerocopy_callback(struct ubuf_info_msgzc * uarg)1775 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
1776 {
1777 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1778 struct sock_exterr_skb *serr;
1779 struct sock *sk = skb->sk;
1780 struct sk_buff_head *q;
1781 unsigned long flags;
1782 bool is_zerocopy;
1783 u32 lo, hi;
1784 u16 len;
1785
1786 mm_unaccount_pinned_pages(&uarg->mmp);
1787
1788 /* if !len, there was only 1 call, and it was aborted
1789 * so do not queue a completion notification
1790 */
1791 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1792 goto release;
1793
1794 len = uarg->len;
1795 lo = uarg->id;
1796 hi = uarg->id + len - 1;
1797 is_zerocopy = uarg->zerocopy;
1798
1799 serr = SKB_EXT_ERR(skb);
1800 memset(serr, 0, sizeof(*serr));
1801 serr->ee.ee_errno = 0;
1802 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1803 serr->ee.ee_data = hi;
1804 serr->ee.ee_info = lo;
1805 if (!is_zerocopy)
1806 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1807
1808 q = &sk->sk_error_queue;
1809 spin_lock_irqsave(&q->lock, flags);
1810 tail = skb_peek_tail(q);
1811 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1812 !skb_zerocopy_notify_extend(tail, lo, len)) {
1813 __skb_queue_tail(q, skb);
1814 skb = NULL;
1815 }
1816 spin_unlock_irqrestore(&q->lock, flags);
1817
1818 sk_error_report(sk);
1819
1820 release:
1821 consume_skb(skb);
1822 sock_put(sk);
1823 }
1824
msg_zerocopy_complete(struct sk_buff * skb,struct ubuf_info * uarg,bool success)1825 static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
1826 bool success)
1827 {
1828 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
1829
1830 uarg_zc->zerocopy = uarg_zc->zerocopy & success;
1831
1832 if (refcount_dec_and_test(&uarg->refcnt))
1833 __msg_zerocopy_callback(uarg_zc);
1834 }
1835
msg_zerocopy_put_abort(struct ubuf_info * uarg,bool have_uref)1836 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1837 {
1838 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
1839
1840 atomic_dec(&sk->sk_zckey);
1841 uarg_to_msgzc(uarg)->len--;
1842
1843 if (have_uref)
1844 msg_zerocopy_complete(NULL, uarg, true);
1845 }
1846 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
1847
1848 const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
1849 .complete = msg_zerocopy_complete,
1850 };
1851 EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
1852
skb_zerocopy_iter_stream(struct sock * sk,struct sk_buff * skb,struct msghdr * msg,int len,struct ubuf_info * uarg)1853 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1854 struct msghdr *msg, int len,
1855 struct ubuf_info *uarg)
1856 {
1857 struct ubuf_info *orig_uarg = skb_zcopy(skb);
1858 int err, orig_len = skb->len;
1859
1860 if (uarg->ops->link_skb) {
1861 err = uarg->ops->link_skb(skb, uarg);
1862 if (err)
1863 return err;
1864 } else {
1865 /* An skb can only point to one uarg. This edge case happens
1866 * when TCP appends to an skb, but zerocopy_realloc triggered
1867 * a new alloc.
1868 */
1869 if (orig_uarg && uarg != orig_uarg)
1870 return -EEXIST;
1871 }
1872
1873 err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
1874 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1875 struct sock *save_sk = skb->sk;
1876
1877 /* Streams do not free skb on error. Reset to prev state. */
1878 iov_iter_revert(&msg->msg_iter, skb->len - orig_len);
1879 skb->sk = sk;
1880 ___pskb_trim(skb, orig_len);
1881 skb->sk = save_sk;
1882 return err;
1883 }
1884
1885 if (!uarg->ops->link_skb)
1886 skb_zcopy_set(skb, uarg, NULL);
1887 return skb->len - orig_len;
1888 }
1889 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1890
__skb_zcopy_downgrade_managed(struct sk_buff * skb)1891 void __skb_zcopy_downgrade_managed(struct sk_buff *skb)
1892 {
1893 int i;
1894
1895 skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS;
1896 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1897 skb_frag_ref(skb, i);
1898 }
1899 EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed);
1900
skb_zerocopy_clone(struct sk_buff * nskb,struct sk_buff * orig,gfp_t gfp_mask)1901 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1902 gfp_t gfp_mask)
1903 {
1904 if (skb_zcopy(orig)) {
1905 if (skb_zcopy(nskb)) {
1906 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1907 if (!gfp_mask) {
1908 WARN_ON_ONCE(1);
1909 return -ENOMEM;
1910 }
1911 if (skb_uarg(nskb) == skb_uarg(orig))
1912 return 0;
1913 if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1914 return -EIO;
1915 }
1916 skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1917 }
1918 return 0;
1919 }
1920
1921 /**
1922 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
1923 * @skb: the skb to modify
1924 * @gfp_mask: allocation priority
1925 *
1926 * This must be called on skb with SKBFL_ZEROCOPY_ENABLE.
1927 * It will copy all frags into kernel and drop the reference
1928 * to userspace pages.
1929 *
1930 * If this function is called from an interrupt gfp_mask() must be
1931 * %GFP_ATOMIC.
1932 *
1933 * Returns 0 on success or a negative error code on failure
1934 * to allocate kernel memory to copy to.
1935 */
skb_copy_ubufs(struct sk_buff * skb,gfp_t gfp_mask)1936 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1937 {
1938 int num_frags = skb_shinfo(skb)->nr_frags;
1939 struct page *page, *head = NULL;
1940 int i, order, psize, new_frags;
1941 u32 d_off;
1942
1943 if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1944 return -EINVAL;
1945
1946 if (!num_frags)
1947 goto release;
1948
1949 /* We might have to allocate high order pages, so compute what minimum
1950 * page order is needed.
1951 */
1952 order = 0;
1953 while ((PAGE_SIZE << order) * MAX_SKB_FRAGS < __skb_pagelen(skb))
1954 order++;
1955 psize = (PAGE_SIZE << order);
1956
1957 new_frags = (__skb_pagelen(skb) + psize - 1) >> (PAGE_SHIFT + order);
1958 for (i = 0; i < new_frags; i++) {
1959 page = alloc_pages(gfp_mask | __GFP_COMP, order);
1960 if (!page) {
1961 while (head) {
1962 struct page *next = (struct page *)page_private(head);
1963 put_page(head);
1964 head = next;
1965 }
1966 return -ENOMEM;
1967 }
1968 set_page_private(page, (unsigned long)head);
1969 head = page;
1970 }
1971
1972 page = head;
1973 d_off = 0;
1974 for (i = 0; i < num_frags; i++) {
1975 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1976 u32 p_off, p_len, copied;
1977 struct page *p;
1978 u8 *vaddr;
1979
1980 skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
1981 p, p_off, p_len, copied) {
1982 u32 copy, done = 0;
1983 vaddr = kmap_atomic(p);
1984
1985 while (done < p_len) {
1986 if (d_off == psize) {
1987 d_off = 0;
1988 page = (struct page *)page_private(page);
1989 }
1990 copy = min_t(u32, psize - d_off, p_len - done);
1991 memcpy(page_address(page) + d_off,
1992 vaddr + p_off + done, copy);
1993 done += copy;
1994 d_off += copy;
1995 }
1996 kunmap_atomic(vaddr);
1997 }
1998 }
1999
2000 /* skb frags release userspace buffers */
2001 for (i = 0; i < num_frags; i++)
2002 skb_frag_unref(skb, i);
2003
2004 /* skb frags point to kernel buffers */
2005 for (i = 0; i < new_frags - 1; i++) {
2006 __skb_fill_netmem_desc(skb, i, page_to_netmem(head), 0, psize);
2007 head = (struct page *)page_private(head);
2008 }
2009 __skb_fill_netmem_desc(skb, new_frags - 1, page_to_netmem(head), 0,
2010 d_off);
2011 skb_shinfo(skb)->nr_frags = new_frags;
2012
2013 release:
2014 skb_zcopy_clear(skb, false);
2015 return 0;
2016 }
2017 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
2018
2019 /**
2020 * skb_clone - duplicate an sk_buff
2021 * @skb: buffer to clone
2022 * @gfp_mask: allocation priority
2023 *
2024 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
2025 * copies share the same packet data but not structure. The new
2026 * buffer has a reference count of 1. If the allocation fails the
2027 * function returns %NULL otherwise the new buffer is returned.
2028 *
2029 * If this function is called from an interrupt gfp_mask() must be
2030 * %GFP_ATOMIC.
2031 */
2032
skb_clone(struct sk_buff * skb,gfp_t gfp_mask)2033 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
2034 {
2035 struct sk_buff_fclones *fclones = container_of(skb,
2036 struct sk_buff_fclones,
2037 skb1);
2038 struct sk_buff *n;
2039
2040 if (skb_orphan_frags(skb, gfp_mask))
2041 return NULL;
2042
2043 if (skb->fclone == SKB_FCLONE_ORIG &&
2044 refcount_read(&fclones->fclone_ref) == 1) {
2045 n = &fclones->skb2;
2046 refcount_set(&fclones->fclone_ref, 2);
2047 n->fclone = SKB_FCLONE_CLONE;
2048 } else {
2049 if (skb_pfmemalloc(skb))
2050 gfp_mask |= __GFP_MEMALLOC;
2051
2052 n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
2053 if (!n)
2054 return NULL;
2055
2056 n->fclone = SKB_FCLONE_UNAVAILABLE;
2057 }
2058
2059 return __skb_clone(n, skb);
2060 }
2061 EXPORT_SYMBOL(skb_clone);
2062
skb_headers_offset_update(struct sk_buff * skb,int off)2063 void skb_headers_offset_update(struct sk_buff *skb, int off)
2064 {
2065 /* Only adjust this if it actually is csum_start rather than csum */
2066 if (skb->ip_summed == CHECKSUM_PARTIAL)
2067 skb->csum_start += off;
2068 /* {transport,network,mac}_header and tail are relative to skb->head */
2069 skb->transport_header += off;
2070 skb->network_header += off;
2071 if (skb_mac_header_was_set(skb))
2072 skb->mac_header += off;
2073 skb->inner_transport_header += off;
2074 skb->inner_network_header += off;
2075 skb->inner_mac_header += off;
2076 }
2077 EXPORT_SYMBOL(skb_headers_offset_update);
2078
skb_copy_header(struct sk_buff * new,const struct sk_buff * old)2079 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
2080 {
2081 __copy_skb_header(new, old);
2082
2083 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
2084 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
2085 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
2086 }
2087 EXPORT_SYMBOL(skb_copy_header);
2088
skb_alloc_rx_flag(const struct sk_buff * skb)2089 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
2090 {
2091 if (skb_pfmemalloc(skb))
2092 return SKB_ALLOC_RX;
2093 return 0;
2094 }
2095
2096 /**
2097 * skb_copy - create private copy of an sk_buff
2098 * @skb: buffer to copy
2099 * @gfp_mask: allocation priority
2100 *
2101 * Make a copy of both an &sk_buff and its data. This is used when the
2102 * caller wishes to modify the data and needs a private copy of the
2103 * data to alter. Returns %NULL on failure or the pointer to the buffer
2104 * on success. The returned buffer has a reference count of 1.
2105 *
2106 * As by-product this function converts non-linear &sk_buff to linear
2107 * one, so that &sk_buff becomes completely private and caller is allowed
2108 * to modify all the data of returned buffer. This means that this
2109 * function is not recommended for use in circumstances when only
2110 * header is going to be modified. Use pskb_copy() instead.
2111 */
2112
skb_copy(const struct sk_buff * skb,gfp_t gfp_mask)2113 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
2114 {
2115 struct sk_buff *n;
2116 unsigned int size;
2117 int headerlen;
2118
2119 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
2120 return NULL;
2121
2122 headerlen = skb_headroom(skb);
2123 size = skb_end_offset(skb) + skb->data_len;
2124 n = __alloc_skb(size, gfp_mask,
2125 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
2126 if (!n)
2127 return NULL;
2128
2129 /* Set the data pointer */
2130 skb_reserve(n, headerlen);
2131 /* Set the tail pointer and length */
2132 skb_put(n, skb->len);
2133
2134 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
2135
2136 skb_copy_header(n, skb);
2137 return n;
2138 }
2139 EXPORT_SYMBOL(skb_copy);
2140
2141 /**
2142 * __pskb_copy_fclone - create copy of an sk_buff with private head.
2143 * @skb: buffer to copy
2144 * @headroom: headroom of new skb
2145 * @gfp_mask: allocation priority
2146 * @fclone: if true allocate the copy of the skb from the fclone
2147 * cache instead of the head cache; it is recommended to set this
2148 * to true for the cases where the copy will likely be cloned
2149 *
2150 * Make a copy of both an &sk_buff and part of its data, located
2151 * in header. Fragmented data remain shared. This is used when
2152 * the caller wishes to modify only header of &sk_buff and needs
2153 * private copy of the header to alter. Returns %NULL on failure
2154 * or the pointer to the buffer on success.
2155 * The returned buffer has a reference count of 1.
2156 */
2157
__pskb_copy_fclone(struct sk_buff * skb,int headroom,gfp_t gfp_mask,bool fclone)2158 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
2159 gfp_t gfp_mask, bool fclone)
2160 {
2161 unsigned int size = skb_headlen(skb) + headroom;
2162 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
2163 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
2164
2165 if (!n)
2166 goto out;
2167
2168 /* Set the data pointer */
2169 skb_reserve(n, headroom);
2170 /* Set the tail pointer and length */
2171 skb_put(n, skb_headlen(skb));
2172 /* Copy the bytes */
2173 skb_copy_from_linear_data(skb, n->data, n->len);
2174
2175 n->truesize += skb->data_len;
2176 n->data_len = skb->data_len;
2177 n->len = skb->len;
2178
2179 if (skb_shinfo(skb)->nr_frags) {
2180 int i;
2181
2182 if (skb_orphan_frags(skb, gfp_mask) ||
2183 skb_zerocopy_clone(n, skb, gfp_mask)) {
2184 kfree_skb(n);
2185 n = NULL;
2186 goto out;
2187 }
2188 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2189 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
2190 skb_frag_ref(skb, i);
2191 }
2192 skb_shinfo(n)->nr_frags = i;
2193 }
2194
2195 if (skb_has_frag_list(skb)) {
2196 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
2197 skb_clone_fraglist(n);
2198 }
2199
2200 skb_copy_header(n, skb);
2201 out:
2202 return n;
2203 }
2204 EXPORT_SYMBOL(__pskb_copy_fclone);
2205
2206 /**
2207 * pskb_expand_head - reallocate header of &sk_buff
2208 * @skb: buffer to reallocate
2209 * @nhead: room to add at head
2210 * @ntail: room to add at tail
2211 * @gfp_mask: allocation priority
2212 *
2213 * Expands (or creates identical copy, if @nhead and @ntail are zero)
2214 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
2215 * reference count of 1. Returns zero in the case of success or error,
2216 * if expansion failed. In the last case, &sk_buff is not changed.
2217 *
2218 * All the pointers pointing into skb header may change and must be
2219 * reloaded after call to this function.
2220 */
2221
pskb_expand_head(struct sk_buff * skb,int nhead,int ntail,gfp_t gfp_mask)2222 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
2223 gfp_t gfp_mask)
2224 {
2225 unsigned int osize = skb_end_offset(skb);
2226 unsigned int size = osize + nhead + ntail;
2227 long off;
2228 u8 *data;
2229 int i;
2230
2231 BUG_ON(nhead < 0);
2232
2233 BUG_ON(skb_shared(skb));
2234
2235 skb_zcopy_downgrade_managed(skb);
2236
2237 if (skb_pfmemalloc(skb))
2238 gfp_mask |= __GFP_MEMALLOC;
2239
2240 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
2241 if (!data)
2242 goto nodata;
2243 size = SKB_WITH_OVERHEAD(size);
2244
2245 /* Copy only real data... and, alas, header. This should be
2246 * optimized for the cases when header is void.
2247 */
2248 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
2249
2250 memcpy((struct skb_shared_info *)(data + size),
2251 skb_shinfo(skb),
2252 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
2253
2254 /*
2255 * if shinfo is shared we must drop the old head gracefully, but if it
2256 * is not we can just drop the old head and let the existing refcount
2257 * be since all we did is relocate the values
2258 */
2259 if (skb_cloned(skb)) {
2260 if (skb_orphan_frags(skb, gfp_mask))
2261 goto nofrags;
2262 if (skb_zcopy(skb))
2263 refcount_inc(&skb_uarg(skb)->refcnt);
2264 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2265 skb_frag_ref(skb, i);
2266
2267 if (skb_has_frag_list(skb))
2268 skb_clone_fraglist(skb);
2269
2270 skb_release_data(skb, SKB_CONSUMED);
2271 } else {
2272 skb_free_head(skb);
2273 }
2274 off = (data + nhead) - skb->head;
2275
2276 skb->head = data;
2277 skb->head_frag = 0;
2278 skb->data += off;
2279
2280 skb_set_end_offset(skb, size);
2281 #ifdef NET_SKBUFF_DATA_USES_OFFSET
2282 off = nhead;
2283 #endif
2284 skb->tail += off;
2285 skb_headers_offset_update(skb, nhead);
2286 skb->cloned = 0;
2287 skb->hdr_len = 0;
2288 skb->nohdr = 0;
2289 atomic_set(&skb_shinfo(skb)->dataref, 1);
2290
2291 skb_metadata_clear(skb);
2292
2293 /* It is not generally safe to change skb->truesize.
2294 * For the moment, we really care of rx path, or
2295 * when skb is orphaned (not attached to a socket).
2296 */
2297 if (!skb->sk || skb->destructor == sock_edemux)
2298 skb->truesize += size - osize;
2299
2300 return 0;
2301
2302 nofrags:
2303 skb_kfree_head(data, size);
2304 nodata:
2305 return -ENOMEM;
2306 }
2307 EXPORT_SYMBOL(pskb_expand_head);
2308
2309 /* Make private copy of skb with writable head and some headroom */
2310
skb_realloc_headroom(struct sk_buff * skb,unsigned int headroom)2311 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
2312 {
2313 struct sk_buff *skb2;
2314 int delta = headroom - skb_headroom(skb);
2315
2316 if (delta <= 0)
2317 skb2 = pskb_copy(skb, GFP_ATOMIC);
2318 else {
2319 skb2 = skb_clone(skb, GFP_ATOMIC);
2320 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
2321 GFP_ATOMIC)) {
2322 kfree_skb(skb2);
2323 skb2 = NULL;
2324 }
2325 }
2326 return skb2;
2327 }
2328 EXPORT_SYMBOL(skb_realloc_headroom);
2329
2330 /* Note: We plan to rework this in linux-6.4 */
__skb_unclone_keeptruesize(struct sk_buff * skb,gfp_t pri)2331 int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
2332 {
2333 unsigned int saved_end_offset, saved_truesize;
2334 struct skb_shared_info *shinfo;
2335 int res;
2336
2337 saved_end_offset = skb_end_offset(skb);
2338 saved_truesize = skb->truesize;
2339
2340 res = pskb_expand_head(skb, 0, 0, pri);
2341 if (res)
2342 return res;
2343
2344 skb->truesize = saved_truesize;
2345
2346 if (likely(skb_end_offset(skb) == saved_end_offset))
2347 return 0;
2348
2349 /* We can not change skb->end if the original or new value
2350 * is SKB_SMALL_HEAD_HEADROOM, as it might break skb_kfree_head().
2351 */
2352 if (saved_end_offset == SKB_SMALL_HEAD_HEADROOM ||
2353 skb_end_offset(skb) == SKB_SMALL_HEAD_HEADROOM) {
2354 /* We think this path should not be taken.
2355 * Add a temporary trace to warn us just in case.
2356 */
2357 pr_err_once("__skb_unclone_keeptruesize() skb_end_offset() %u -> %u\n",
2358 saved_end_offset, skb_end_offset(skb));
2359 WARN_ON_ONCE(1);
2360 return 0;
2361 }
2362
2363 shinfo = skb_shinfo(skb);
2364
2365 /* We are about to change back skb->end,
2366 * we need to move skb_shinfo() to its new location.
2367 */
2368 memmove(skb->head + saved_end_offset,
2369 shinfo,
2370 offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
2371
2372 skb_set_end_offset(skb, saved_end_offset);
2373
2374 return 0;
2375 }
2376
2377 /**
2378 * skb_expand_head - reallocate header of &sk_buff
2379 * @skb: buffer to reallocate
2380 * @headroom: needed headroom
2381 *
2382 * Unlike skb_realloc_headroom, this one does not allocate a new skb
2383 * if possible; copies skb->sk to new skb as needed
2384 * and frees original skb in case of failures.
2385 *
2386 * It expect increased headroom and generates warning otherwise.
2387 */
2388
skb_expand_head(struct sk_buff * skb,unsigned int headroom)2389 struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom)
2390 {
2391 int delta = headroom - skb_headroom(skb);
2392 int osize = skb_end_offset(skb);
2393 struct sock *sk = skb->sk;
2394
2395 if (WARN_ONCE(delta <= 0,
2396 "%s is expecting an increase in the headroom", __func__))
2397 return skb;
2398
2399 delta = SKB_DATA_ALIGN(delta);
2400 /* pskb_expand_head() might crash, if skb is shared. */
2401 if (skb_shared(skb) || !is_skb_wmem(skb)) {
2402 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2403
2404 if (unlikely(!nskb))
2405 goto fail;
2406
2407 if (sk)
2408 skb_set_owner_w(nskb, sk);
2409 consume_skb(skb);
2410 skb = nskb;
2411 }
2412 if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC))
2413 goto fail;
2414
2415 if (sk && is_skb_wmem(skb)) {
2416 delta = skb_end_offset(skb) - osize;
2417 refcount_add(delta, &sk->sk_wmem_alloc);
2418 skb->truesize += delta;
2419 }
2420 return skb;
2421
2422 fail:
2423 kfree_skb(skb);
2424 return NULL;
2425 }
2426 EXPORT_SYMBOL(skb_expand_head);
2427
2428 /**
2429 * skb_copy_expand - copy and expand sk_buff
2430 * @skb: buffer to copy
2431 * @newheadroom: new free bytes at head
2432 * @newtailroom: new free bytes at tail
2433 * @gfp_mask: allocation priority
2434 *
2435 * Make a copy of both an &sk_buff and its data and while doing so
2436 * allocate additional space.
2437 *
2438 * This is used when the caller wishes to modify the data and needs a
2439 * private copy of the data to alter as well as more space for new fields.
2440 * Returns %NULL on failure or the pointer to the buffer
2441 * on success. The returned buffer has a reference count of 1.
2442 *
2443 * You must pass %GFP_ATOMIC as the allocation priority if this function
2444 * is called from an interrupt.
2445 */
skb_copy_expand(const struct sk_buff * skb,int newheadroom,int newtailroom,gfp_t gfp_mask)2446 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
2447 int newheadroom, int newtailroom,
2448 gfp_t gfp_mask)
2449 {
2450 /*
2451 * Allocate the copy buffer
2452 */
2453 int head_copy_len, head_copy_off;
2454 struct sk_buff *n;
2455 int oldheadroom;
2456
2457 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
2458 return NULL;
2459
2460 oldheadroom = skb_headroom(skb);
2461 n = __alloc_skb(newheadroom + skb->len + newtailroom,
2462 gfp_mask, skb_alloc_rx_flag(skb),
2463 NUMA_NO_NODE);
2464 if (!n)
2465 return NULL;
2466
2467 skb_reserve(n, newheadroom);
2468
2469 /* Set the tail pointer and length */
2470 skb_put(n, skb->len);
2471
2472 head_copy_len = oldheadroom;
2473 head_copy_off = 0;
2474 if (newheadroom <= head_copy_len)
2475 head_copy_len = newheadroom;
2476 else
2477 head_copy_off = newheadroom - head_copy_len;
2478
2479 /* Copy the linear header and data. */
2480 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
2481 skb->len + head_copy_len));
2482
2483 skb_copy_header(n, skb);
2484
2485 skb_headers_offset_update(n, newheadroom - oldheadroom);
2486
2487 return n;
2488 }
2489 EXPORT_SYMBOL(skb_copy_expand);
2490
2491 /**
2492 * __skb_pad - zero pad the tail of an skb
2493 * @skb: buffer to pad
2494 * @pad: space to pad
2495 * @free_on_error: free buffer on error
2496 *
2497 * Ensure that a buffer is followed by a padding area that is zero
2498 * filled. Used by network drivers which may DMA or transfer data
2499 * beyond the buffer end onto the wire.
2500 *
2501 * May return error in out of memory cases. The skb is freed on error
2502 * if @free_on_error is true.
2503 */
2504
__skb_pad(struct sk_buff * skb,int pad,bool free_on_error)2505 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
2506 {
2507 int err;
2508 int ntail;
2509
2510 /* If the skbuff is non linear tailroom is always zero.. */
2511 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
2512 memset(skb->data+skb->len, 0, pad);
2513 return 0;
2514 }
2515
2516 ntail = skb->data_len + pad - (skb->end - skb->tail);
2517 if (likely(skb_cloned(skb) || ntail > 0)) {
2518 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
2519 if (unlikely(err))
2520 goto free_skb;
2521 }
2522
2523 /* FIXME: The use of this function with non-linear skb's really needs
2524 * to be audited.
2525 */
2526 err = skb_linearize(skb);
2527 if (unlikely(err))
2528 goto free_skb;
2529
2530 memset(skb->data + skb->len, 0, pad);
2531 return 0;
2532
2533 free_skb:
2534 if (free_on_error)
2535 kfree_skb(skb);
2536 return err;
2537 }
2538 EXPORT_SYMBOL(__skb_pad);
2539
2540 /**
2541 * pskb_put - add data to the tail of a potentially fragmented buffer
2542 * @skb: start of the buffer to use
2543 * @tail: tail fragment of the buffer to use
2544 * @len: amount of data to add
2545 *
2546 * This function extends the used data area of the potentially
2547 * fragmented buffer. @tail must be the last fragment of @skb -- or
2548 * @skb itself. If this would exceed the total buffer size the kernel
2549 * will panic. A pointer to the first byte of the extra data is
2550 * returned.
2551 */
2552
pskb_put(struct sk_buff * skb,struct sk_buff * tail,int len)2553 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
2554 {
2555 if (tail != skb) {
2556 skb->data_len += len;
2557 skb->len += len;
2558 }
2559 return skb_put(tail, len);
2560 }
2561 EXPORT_SYMBOL_GPL(pskb_put);
2562
2563 /**
2564 * skb_put - add data to a buffer
2565 * @skb: buffer to use
2566 * @len: amount of data to add
2567 *
2568 * This function extends the used data area of the buffer. If this would
2569 * exceed the total buffer size the kernel will panic. A pointer to the
2570 * first byte of the extra data is returned.
2571 */
skb_put(struct sk_buff * skb,unsigned int len)2572 void *skb_put(struct sk_buff *skb, unsigned int len)
2573 {
2574 void *tmp = skb_tail_pointer(skb);
2575 SKB_LINEAR_ASSERT(skb);
2576 skb->tail += len;
2577 skb->len += len;
2578 if (unlikely(skb->tail > skb->end))
2579 skb_over_panic(skb, len, __builtin_return_address(0));
2580 return tmp;
2581 }
2582 EXPORT_SYMBOL(skb_put);
2583
2584 /**
2585 * skb_push - add data to the start of a buffer
2586 * @skb: buffer to use
2587 * @len: amount of data to add
2588 *
2589 * This function extends the used data area of the buffer at the buffer
2590 * start. If this would exceed the total buffer headroom the kernel will
2591 * panic. A pointer to the first byte of the extra data is returned.
2592 */
skb_push(struct sk_buff * skb,unsigned int len)2593 void *skb_push(struct sk_buff *skb, unsigned int len)
2594 {
2595 skb->data -= len;
2596 skb->len += len;
2597 if (unlikely(skb->data < skb->head))
2598 skb_under_panic(skb, len, __builtin_return_address(0));
2599 return skb->data;
2600 }
2601 EXPORT_SYMBOL(skb_push);
2602
2603 /**
2604 * skb_pull - remove data from the start of a buffer
2605 * @skb: buffer to use
2606 * @len: amount of data to remove
2607 *
2608 * This function removes data from the start of a buffer, returning
2609 * the memory to the headroom. A pointer to the next data in the buffer
2610 * is returned. Once the data has been pulled future pushes will overwrite
2611 * the old data.
2612 */
skb_pull(struct sk_buff * skb,unsigned int len)2613 void *skb_pull(struct sk_buff *skb, unsigned int len)
2614 {
2615 return skb_pull_inline(skb, len);
2616 }
2617 EXPORT_SYMBOL(skb_pull);
2618
2619 /**
2620 * skb_pull_data - remove data from the start of a buffer returning its
2621 * original position.
2622 * @skb: buffer to use
2623 * @len: amount of data to remove
2624 *
2625 * This function removes data from the start of a buffer, returning
2626 * the memory to the headroom. A pointer to the original data in the buffer
2627 * is returned after checking if there is enough data to pull. Once the
2628 * data has been pulled future pushes will overwrite the old data.
2629 */
skb_pull_data(struct sk_buff * skb,size_t len)2630 void *skb_pull_data(struct sk_buff *skb, size_t len)
2631 {
2632 void *data = skb->data;
2633
2634 if (skb->len < len)
2635 return NULL;
2636
2637 skb_pull(skb, len);
2638
2639 return data;
2640 }
2641 EXPORT_SYMBOL(skb_pull_data);
2642
2643 /**
2644 * skb_trim - remove end from a buffer
2645 * @skb: buffer to alter
2646 * @len: new length
2647 *
2648 * Cut the length of a buffer down by removing data from the tail. If
2649 * the buffer is already under the length specified it is not modified.
2650 * The skb must be linear.
2651 */
skb_trim(struct sk_buff * skb,unsigned int len)2652 void skb_trim(struct sk_buff *skb, unsigned int len)
2653 {
2654 if (skb->len > len)
2655 __skb_trim(skb, len);
2656 }
2657 EXPORT_SYMBOL(skb_trim);
2658
2659 /* Trims skb to length len. It can change skb pointers.
2660 */
2661
___pskb_trim(struct sk_buff * skb,unsigned int len)2662 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
2663 {
2664 struct sk_buff **fragp;
2665 struct sk_buff *frag;
2666 int offset = skb_headlen(skb);
2667 int nfrags = skb_shinfo(skb)->nr_frags;
2668 int i;
2669 int err;
2670
2671 if (skb_cloned(skb) &&
2672 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
2673 return err;
2674
2675 i = 0;
2676 if (offset >= len)
2677 goto drop_pages;
2678
2679 for (; i < nfrags; i++) {
2680 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2681
2682 if (end < len) {
2683 offset = end;
2684 continue;
2685 }
2686
2687 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
2688
2689 drop_pages:
2690 skb_shinfo(skb)->nr_frags = i;
2691
2692 for (; i < nfrags; i++)
2693 skb_frag_unref(skb, i);
2694
2695 if (skb_has_frag_list(skb))
2696 skb_drop_fraglist(skb);
2697 goto done;
2698 }
2699
2700 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2701 fragp = &frag->next) {
2702 int end = offset + frag->len;
2703
2704 if (skb_shared(frag)) {
2705 struct sk_buff *nfrag;
2706
2707 nfrag = skb_clone(frag, GFP_ATOMIC);
2708 if (unlikely(!nfrag))
2709 return -ENOMEM;
2710
2711 nfrag->next = frag->next;
2712 consume_skb(frag);
2713 frag = nfrag;
2714 *fragp = frag;
2715 }
2716
2717 if (end < len) {
2718 offset = end;
2719 continue;
2720 }
2721
2722 if (end > len &&
2723 unlikely((err = pskb_trim(frag, len - offset))))
2724 return err;
2725
2726 if (frag->next)
2727 skb_drop_list(&frag->next);
2728 break;
2729 }
2730
2731 done:
2732 if (len > skb_headlen(skb)) {
2733 skb->data_len -= skb->len - len;
2734 skb->len = len;
2735 } else {
2736 skb->len = len;
2737 skb->data_len = 0;
2738 skb_set_tail_pointer(skb, len);
2739 }
2740
2741 if (!skb->sk || skb->destructor == sock_edemux)
2742 skb_condense(skb);
2743 return 0;
2744 }
2745 EXPORT_SYMBOL(___pskb_trim);
2746
2747 /* Note : use pskb_trim_rcsum() instead of calling this directly
2748 */
pskb_trim_rcsum_slow(struct sk_buff * skb,unsigned int len)2749 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2750 {
2751 if (skb->ip_summed == CHECKSUM_COMPLETE) {
2752 int delta = skb->len - len;
2753
2754 skb->csum = csum_block_sub(skb->csum,
2755 skb_checksum(skb, len, delta, 0),
2756 len);
2757 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
2758 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
2759 int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
2760
2761 if (offset + sizeof(__sum16) > hdlen)
2762 return -EINVAL;
2763 }
2764 return __pskb_trim(skb, len);
2765 }
2766 EXPORT_SYMBOL(pskb_trim_rcsum_slow);
2767
2768 /**
2769 * __pskb_pull_tail - advance tail of skb header
2770 * @skb: buffer to reallocate
2771 * @delta: number of bytes to advance tail
2772 *
2773 * The function makes a sense only on a fragmented &sk_buff,
2774 * it expands header moving its tail forward and copying necessary
2775 * data from fragmented part.
2776 *
2777 * &sk_buff MUST have reference count of 1.
2778 *
2779 * Returns %NULL (and &sk_buff does not change) if pull failed
2780 * or value of new tail of skb in the case of success.
2781 *
2782 * All the pointers pointing into skb header may change and must be
2783 * reloaded after call to this function.
2784 */
2785
2786 /* Moves tail of skb head forward, copying data from fragmented part,
2787 * when it is necessary.
2788 * 1. It may fail due to malloc failure.
2789 * 2. It may change skb pointers.
2790 *
2791 * It is pretty complicated. Luckily, it is called only in exceptional cases.
2792 */
__pskb_pull_tail(struct sk_buff * skb,int delta)2793 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
2794 {
2795 /* If skb has not enough free space at tail, get new one
2796 * plus 128 bytes for future expansions. If we have enough
2797 * room at tail, reallocate without expansion only if skb is cloned.
2798 */
2799 int i, k, eat = (skb->tail + delta) - skb->end;
2800
2801 if (eat > 0 || skb_cloned(skb)) {
2802 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2803 GFP_ATOMIC))
2804 return NULL;
2805 }
2806
2807 BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2808 skb_tail_pointer(skb), delta));
2809
2810 /* Optimization: no fragments, no reasons to preestimate
2811 * size of pulled pages. Superb.
2812 */
2813 if (!skb_has_frag_list(skb))
2814 goto pull_pages;
2815
2816 /* Estimate size of pulled pages. */
2817 eat = delta;
2818 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2819 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2820
2821 if (size >= eat)
2822 goto pull_pages;
2823 eat -= size;
2824 }
2825
2826 /* If we need update frag list, we are in troubles.
2827 * Certainly, it is possible to add an offset to skb data,
2828 * but taking into account that pulling is expected to
2829 * be very rare operation, it is worth to fight against
2830 * further bloating skb head and crucify ourselves here instead.
2831 * Pure masohism, indeed. 8)8)
2832 */
2833 if (eat) {
2834 struct sk_buff *list = skb_shinfo(skb)->frag_list;
2835 struct sk_buff *clone = NULL;
2836 struct sk_buff *insp = NULL;
2837
2838 do {
2839 if (list->len <= eat) {
2840 /* Eaten as whole. */
2841 eat -= list->len;
2842 list = list->next;
2843 insp = list;
2844 } else {
2845 /* Eaten partially. */
2846 if (skb_is_gso(skb) && !list->head_frag &&
2847 skb_headlen(list))
2848 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2849
2850 if (skb_shared(list)) {
2851 /* Sucks! We need to fork list. :-( */
2852 clone = skb_clone(list, GFP_ATOMIC);
2853 if (!clone)
2854 return NULL;
2855 insp = list->next;
2856 list = clone;
2857 } else {
2858 /* This may be pulled without
2859 * problems. */
2860 insp = list;
2861 }
2862 if (!pskb_pull(list, eat)) {
2863 kfree_skb(clone);
2864 return NULL;
2865 }
2866 break;
2867 }
2868 } while (eat);
2869
2870 /* Free pulled out fragments. */
2871 while ((list = skb_shinfo(skb)->frag_list) != insp) {
2872 skb_shinfo(skb)->frag_list = list->next;
2873 consume_skb(list);
2874 }
2875 /* And insert new clone at head. */
2876 if (clone) {
2877 clone->next = list;
2878 skb_shinfo(skb)->frag_list = clone;
2879 }
2880 }
2881 /* Success! Now we may commit changes to skb data. */
2882
2883 pull_pages:
2884 eat = delta;
2885 k = 0;
2886 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2887 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2888
2889 if (size <= eat) {
2890 skb_frag_unref(skb, i);
2891 eat -= size;
2892 } else {
2893 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2894
2895 *frag = skb_shinfo(skb)->frags[i];
2896 if (eat) {
2897 skb_frag_off_add(frag, eat);
2898 skb_frag_size_sub(frag, eat);
2899 if (!i)
2900 goto end;
2901 eat = 0;
2902 }
2903 k++;
2904 }
2905 }
2906 skb_shinfo(skb)->nr_frags = k;
2907
2908 end:
2909 skb->tail += delta;
2910 skb->data_len -= delta;
2911
2912 if (!skb->data_len)
2913 skb_zcopy_clear(skb, false);
2914
2915 return skb_tail_pointer(skb);
2916 }
2917 EXPORT_SYMBOL(__pskb_pull_tail);
2918
2919 /**
2920 * skb_copy_bits - copy bits from skb to kernel buffer
2921 * @skb: source skb
2922 * @offset: offset in source
2923 * @to: destination buffer
2924 * @len: number of bytes to copy
2925 *
2926 * Copy the specified number of bytes from the source skb to the
2927 * destination buffer.
2928 *
2929 * CAUTION ! :
2930 * If its prototype is ever changed,
2931 * check arch/{*}/net/{*}.S files,
2932 * since it is called from BPF assembly code.
2933 */
skb_copy_bits(const struct sk_buff * skb,int offset,void * to,int len)2934 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2935 {
2936 int start = skb_headlen(skb);
2937 struct sk_buff *frag_iter;
2938 int i, copy;
2939
2940 if (offset > (int)skb->len - len)
2941 goto fault;
2942
2943 /* Copy header. */
2944 if ((copy = start - offset) > 0) {
2945 if (copy > len)
2946 copy = len;
2947 skb_copy_from_linear_data_offset(skb, offset, to, copy);
2948 if ((len -= copy) == 0)
2949 return 0;
2950 offset += copy;
2951 to += copy;
2952 }
2953
2954 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2955 int end;
2956 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2957
2958 WARN_ON(start > offset + len);
2959
2960 end = start + skb_frag_size(f);
2961 if ((copy = end - offset) > 0) {
2962 u32 p_off, p_len, copied;
2963 struct page *p;
2964 u8 *vaddr;
2965
2966 if (copy > len)
2967 copy = len;
2968
2969 skb_frag_foreach_page(f,
2970 skb_frag_off(f) + offset - start,
2971 copy, p, p_off, p_len, copied) {
2972 vaddr = kmap_atomic(p);
2973 memcpy(to + copied, vaddr + p_off, p_len);
2974 kunmap_atomic(vaddr);
2975 }
2976
2977 if ((len -= copy) == 0)
2978 return 0;
2979 offset += copy;
2980 to += copy;
2981 }
2982 start = end;
2983 }
2984
2985 skb_walk_frags(skb, frag_iter) {
2986 int end;
2987
2988 WARN_ON(start > offset + len);
2989
2990 end = start + frag_iter->len;
2991 if ((copy = end - offset) > 0) {
2992 if (copy > len)
2993 copy = len;
2994 if (skb_copy_bits(frag_iter, offset - start, to, copy))
2995 goto fault;
2996 if ((len -= copy) == 0)
2997 return 0;
2998 offset += copy;
2999 to += copy;
3000 }
3001 start = end;
3002 }
3003
3004 if (!len)
3005 return 0;
3006
3007 fault:
3008 return -EFAULT;
3009 }
3010 EXPORT_SYMBOL(skb_copy_bits);
3011
3012 /*
3013 * Callback from splice_to_pipe(), if we need to release some pages
3014 * at the end of the spd in case we error'ed out in filling the pipe.
3015 */
sock_spd_release(struct splice_pipe_desc * spd,unsigned int i)3016 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3017 {
3018 put_page(spd->pages[i]);
3019 }
3020
linear_to_page(struct page * page,unsigned int * len,unsigned int * offset,struct sock * sk)3021 static struct page *linear_to_page(struct page *page, unsigned int *len,
3022 unsigned int *offset,
3023 struct sock *sk)
3024 {
3025 struct page_frag *pfrag = sk_page_frag(sk);
3026
3027 if (!sk_page_frag_refill(sk, pfrag))
3028 return NULL;
3029
3030 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
3031
3032 memcpy(page_address(pfrag->page) + pfrag->offset,
3033 page_address(page) + *offset, *len);
3034 *offset = pfrag->offset;
3035 pfrag->offset += *len;
3036
3037 return pfrag->page;
3038 }
3039
spd_can_coalesce(const struct splice_pipe_desc * spd,struct page * page,unsigned int offset)3040 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
3041 struct page *page,
3042 unsigned int offset)
3043 {
3044 return spd->nr_pages &&
3045 spd->pages[spd->nr_pages - 1] == page &&
3046 (spd->partial[spd->nr_pages - 1].offset +
3047 spd->partial[spd->nr_pages - 1].len == offset);
3048 }
3049
3050 /*
3051 * Fill page/offset/length into spd, if it can hold more pages.
3052 */
spd_fill_page(struct splice_pipe_desc * spd,struct pipe_inode_info * pipe,struct page * page,unsigned int * len,unsigned int offset,bool linear,struct sock * sk)3053 static bool spd_fill_page(struct splice_pipe_desc *spd,
3054 struct pipe_inode_info *pipe, struct page *page,
3055 unsigned int *len, unsigned int offset,
3056 bool linear,
3057 struct sock *sk)
3058 {
3059 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
3060 return true;
3061
3062 if (linear) {
3063 page = linear_to_page(page, len, &offset, sk);
3064 if (!page)
3065 return true;
3066 }
3067 if (spd_can_coalesce(spd, page, offset)) {
3068 spd->partial[spd->nr_pages - 1].len += *len;
3069 return false;
3070 }
3071 get_page(page);
3072 spd->pages[spd->nr_pages] = page;
3073 spd->partial[spd->nr_pages].len = *len;
3074 spd->partial[spd->nr_pages].offset = offset;
3075 spd->nr_pages++;
3076
3077 return false;
3078 }
3079
__splice_segment(struct page * page,unsigned int poff,unsigned int plen,unsigned int * off,unsigned int * len,struct splice_pipe_desc * spd,bool linear,struct sock * sk,struct pipe_inode_info * pipe)3080 static bool __splice_segment(struct page *page, unsigned int poff,
3081 unsigned int plen, unsigned int *off,
3082 unsigned int *len,
3083 struct splice_pipe_desc *spd, bool linear,
3084 struct sock *sk,
3085 struct pipe_inode_info *pipe)
3086 {
3087 if (!*len)
3088 return true;
3089
3090 /* skip this segment if already processed */
3091 if (*off >= plen) {
3092 *off -= plen;
3093 return false;
3094 }
3095
3096 /* ignore any bits we already processed */
3097 poff += *off;
3098 plen -= *off;
3099 *off = 0;
3100
3101 do {
3102 unsigned int flen = min(*len, plen);
3103
3104 if (spd_fill_page(spd, pipe, page, &flen, poff,
3105 linear, sk))
3106 return true;
3107 poff += flen;
3108 plen -= flen;
3109 *len -= flen;
3110 } while (*len && plen);
3111
3112 return false;
3113 }
3114
3115 /*
3116 * Map linear and fragment data from the skb to spd. It reports true if the
3117 * pipe is full or if we already spliced the requested length.
3118 */
__skb_splice_bits(struct sk_buff * skb,struct pipe_inode_info * pipe,unsigned int * offset,unsigned int * len,struct splice_pipe_desc * spd,struct sock * sk)3119 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
3120 unsigned int *offset, unsigned int *len,
3121 struct splice_pipe_desc *spd, struct sock *sk)
3122 {
3123 int seg;
3124 struct sk_buff *iter;
3125
3126 /* map the linear part :
3127 * If skb->head_frag is set, this 'linear' part is backed by a
3128 * fragment, and if the head is not shared with any clones then
3129 * we can avoid a copy since we own the head portion of this page.
3130 */
3131 if (__splice_segment(virt_to_page(skb->data),
3132 (unsigned long) skb->data & (PAGE_SIZE - 1),
3133 skb_headlen(skb),
3134 offset, len, spd,
3135 skb_head_is_locked(skb),
3136 sk, pipe))
3137 return true;
3138
3139 /*
3140 * then map the fragments
3141 */
3142 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
3143 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
3144
3145 if (__splice_segment(skb_frag_page(f),
3146 skb_frag_off(f), skb_frag_size(f),
3147 offset, len, spd, false, sk, pipe))
3148 return true;
3149 }
3150
3151 skb_walk_frags(skb, iter) {
3152 if (*offset >= iter->len) {
3153 *offset -= iter->len;
3154 continue;
3155 }
3156 /* __skb_splice_bits() only fails if the output has no room
3157 * left, so no point in going over the frag_list for the error
3158 * case.
3159 */
3160 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
3161 return true;
3162 }
3163
3164 return false;
3165 }
3166
3167 /*
3168 * Map data from the skb to a pipe. Should handle both the linear part,
3169 * the fragments, and the frag list.
3170 */
skb_splice_bits(struct sk_buff * skb,struct sock * sk,unsigned int offset,struct pipe_inode_info * pipe,unsigned int tlen,unsigned int flags)3171 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3172 struct pipe_inode_info *pipe, unsigned int tlen,
3173 unsigned int flags)
3174 {
3175 struct partial_page partial[MAX_SKB_FRAGS];
3176 struct page *pages[MAX_SKB_FRAGS];
3177 struct splice_pipe_desc spd = {
3178 .pages = pages,
3179 .partial = partial,
3180 .nr_pages_max = MAX_SKB_FRAGS,
3181 .ops = &nosteal_pipe_buf_ops,
3182 .spd_release = sock_spd_release,
3183 };
3184 int ret = 0;
3185
3186 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
3187
3188 if (spd.nr_pages)
3189 ret = splice_to_pipe(pipe, &spd);
3190
3191 return ret;
3192 }
3193 EXPORT_SYMBOL_GPL(skb_splice_bits);
3194
sendmsg_locked(struct sock * sk,struct msghdr * msg)3195 static int sendmsg_locked(struct sock *sk, struct msghdr *msg)
3196 {
3197 struct socket *sock = sk->sk_socket;
3198 size_t size = msg_data_left(msg);
3199
3200 if (!sock)
3201 return -EINVAL;
3202
3203 if (!sock->ops->sendmsg_locked)
3204 return sock_no_sendmsg_locked(sk, msg, size);
3205
3206 return sock->ops->sendmsg_locked(sk, msg, size);
3207 }
3208
sendmsg_unlocked(struct sock * sk,struct msghdr * msg)3209 static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
3210 {
3211 struct socket *sock = sk->sk_socket;
3212
3213 if (!sock)
3214 return -EINVAL;
3215 return sock_sendmsg(sock, msg);
3216 }
3217
3218 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
__skb_send_sock(struct sock * sk,struct sk_buff * skb,int offset,int len,sendmsg_func sendmsg)3219 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
3220 int len, sendmsg_func sendmsg)
3221 {
3222 unsigned int orig_len = len;
3223 struct sk_buff *head = skb;
3224 unsigned short fragidx;
3225 int slen, ret;
3226
3227 do_frag_list:
3228
3229 /* Deal with head data */
3230 while (offset < skb_headlen(skb) && len) {
3231 struct kvec kv;
3232 struct msghdr msg;
3233
3234 slen = min_t(int, len, skb_headlen(skb) - offset);
3235 kv.iov_base = skb->data + offset;
3236 kv.iov_len = slen;
3237 memset(&msg, 0, sizeof(msg));
3238 msg.msg_flags = MSG_DONTWAIT;
3239
3240 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
3241 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3242 sendmsg_unlocked, sk, &msg);
3243 if (ret <= 0)
3244 goto error;
3245
3246 offset += ret;
3247 len -= ret;
3248 }
3249
3250 /* All the data was skb head? */
3251 if (!len)
3252 goto out;
3253
3254 /* Make offset relative to start of frags */
3255 offset -= skb_headlen(skb);
3256
3257 /* Find where we are in frag list */
3258 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3259 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
3260
3261 if (offset < skb_frag_size(frag))
3262 break;
3263
3264 offset -= skb_frag_size(frag);
3265 }
3266
3267 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
3268 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
3269
3270 slen = min_t(size_t, len, skb_frag_size(frag) - offset);
3271
3272 while (slen) {
3273 struct bio_vec bvec;
3274 struct msghdr msg = {
3275 .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT,
3276 };
3277
3278 bvec_set_page(&bvec, skb_frag_page(frag), slen,
3279 skb_frag_off(frag) + offset);
3280 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
3281 slen);
3282
3283 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
3284 sendmsg_unlocked, sk, &msg);
3285 if (ret <= 0)
3286 goto error;
3287
3288 len -= ret;
3289 offset += ret;
3290 slen -= ret;
3291 }
3292
3293 offset = 0;
3294 }
3295
3296 if (len) {
3297 /* Process any frag lists */
3298
3299 if (skb == head) {
3300 if (skb_has_frag_list(skb)) {
3301 skb = skb_shinfo(skb)->frag_list;
3302 goto do_frag_list;
3303 }
3304 } else if (skb->next) {
3305 skb = skb->next;
3306 goto do_frag_list;
3307 }
3308 }
3309
3310 out:
3311 return orig_len - len;
3312
3313 error:
3314 return orig_len == len ? ret : orig_len - len;
3315 }
3316
3317 /* Send skb data on a socket. Socket must be locked. */
skb_send_sock_locked(struct sock * sk,struct sk_buff * skb,int offset,int len)3318 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3319 int len)
3320 {
3321 return __skb_send_sock(sk, skb, offset, len, sendmsg_locked);
3322 }
3323 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
3324
3325 /* Send skb data on a socket. Socket must be unlocked. */
skb_send_sock(struct sock * sk,struct sk_buff * skb,int offset,int len)3326 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
3327 {
3328 return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked);
3329 }
3330
3331 /**
3332 * skb_store_bits - store bits from kernel buffer to skb
3333 * @skb: destination buffer
3334 * @offset: offset in destination
3335 * @from: source buffer
3336 * @len: number of bytes to copy
3337 *
3338 * Copy the specified number of bytes from the source buffer to the
3339 * destination skb. This function handles all the messy bits of
3340 * traversing fragment lists and such.
3341 */
3342
skb_store_bits(struct sk_buff * skb,int offset,const void * from,int len)3343 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
3344 {
3345 int start = skb_headlen(skb);
3346 struct sk_buff *frag_iter;
3347 int i, copy;
3348
3349 if (offset > (int)skb->len - len)
3350 goto fault;
3351
3352 if ((copy = start - offset) > 0) {
3353 if (copy > len)
3354 copy = len;
3355 skb_copy_to_linear_data_offset(skb, offset, from, copy);
3356 if ((len -= copy) == 0)
3357 return 0;
3358 offset += copy;
3359 from += copy;
3360 }
3361
3362 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3363 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3364 int end;
3365
3366 WARN_ON(start > offset + len);
3367
3368 end = start + skb_frag_size(frag);
3369 if ((copy = end - offset) > 0) {
3370 u32 p_off, p_len, copied;
3371 struct page *p;
3372 u8 *vaddr;
3373
3374 if (copy > len)
3375 copy = len;
3376
3377 skb_frag_foreach_page(frag,
3378 skb_frag_off(frag) + offset - start,
3379 copy, p, p_off, p_len, copied) {
3380 vaddr = kmap_atomic(p);
3381 memcpy(vaddr + p_off, from + copied, p_len);
3382 kunmap_atomic(vaddr);
3383 }
3384
3385 if ((len -= copy) == 0)
3386 return 0;
3387 offset += copy;
3388 from += copy;
3389 }
3390 start = end;
3391 }
3392
3393 skb_walk_frags(skb, frag_iter) {
3394 int end;
3395
3396 WARN_ON(start > offset + len);
3397
3398 end = start + frag_iter->len;
3399 if ((copy = end - offset) > 0) {
3400 if (copy > len)
3401 copy = len;
3402 if (skb_store_bits(frag_iter, offset - start,
3403 from, copy))
3404 goto fault;
3405 if ((len -= copy) == 0)
3406 return 0;
3407 offset += copy;
3408 from += copy;
3409 }
3410 start = end;
3411 }
3412 if (!len)
3413 return 0;
3414
3415 fault:
3416 return -EFAULT;
3417 }
3418 EXPORT_SYMBOL(skb_store_bits);
3419
3420 /* Checksum skb data. */
__skb_checksum(const struct sk_buff * skb,int offset,int len,__wsum csum,const struct skb_checksum_ops * ops)3421 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
3422 __wsum csum, const struct skb_checksum_ops *ops)
3423 {
3424 int start = skb_headlen(skb);
3425 int i, copy = start - offset;
3426 struct sk_buff *frag_iter;
3427 int pos = 0;
3428
3429 /* Checksum header. */
3430 if (copy > 0) {
3431 if (copy > len)
3432 copy = len;
3433 csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
3434 skb->data + offset, copy, csum);
3435 if ((len -= copy) == 0)
3436 return csum;
3437 offset += copy;
3438 pos = copy;
3439 }
3440
3441 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3442 int end;
3443 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3444
3445 WARN_ON(start > offset + len);
3446
3447 end = start + skb_frag_size(frag);
3448 if ((copy = end - offset) > 0) {
3449 u32 p_off, p_len, copied;
3450 struct page *p;
3451 __wsum csum2;
3452 u8 *vaddr;
3453
3454 if (copy > len)
3455 copy = len;
3456
3457 skb_frag_foreach_page(frag,
3458 skb_frag_off(frag) + offset - start,
3459 copy, p, p_off, p_len, copied) {
3460 vaddr = kmap_atomic(p);
3461 csum2 = INDIRECT_CALL_1(ops->update,
3462 csum_partial_ext,
3463 vaddr + p_off, p_len, 0);
3464 kunmap_atomic(vaddr);
3465 csum = INDIRECT_CALL_1(ops->combine,
3466 csum_block_add_ext, csum,
3467 csum2, pos, p_len);
3468 pos += p_len;
3469 }
3470
3471 if (!(len -= copy))
3472 return csum;
3473 offset += copy;
3474 }
3475 start = end;
3476 }
3477
3478 skb_walk_frags(skb, frag_iter) {
3479 int end;
3480
3481 WARN_ON(start > offset + len);
3482
3483 end = start + frag_iter->len;
3484 if ((copy = end - offset) > 0) {
3485 __wsum csum2;
3486 if (copy > len)
3487 copy = len;
3488 csum2 = __skb_checksum(frag_iter, offset - start,
3489 copy, 0, ops);
3490 csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
3491 csum, csum2, pos, copy);
3492 if ((len -= copy) == 0)
3493 return csum;
3494 offset += copy;
3495 pos += copy;
3496 }
3497 start = end;
3498 }
3499 BUG_ON(len);
3500
3501 return csum;
3502 }
3503 EXPORT_SYMBOL(__skb_checksum);
3504
skb_checksum(const struct sk_buff * skb,int offset,int len,__wsum csum)3505 __wsum skb_checksum(const struct sk_buff *skb, int offset,
3506 int len, __wsum csum)
3507 {
3508 const struct skb_checksum_ops ops = {
3509 .update = csum_partial_ext,
3510 .combine = csum_block_add_ext,
3511 };
3512
3513 return __skb_checksum(skb, offset, len, csum, &ops);
3514 }
3515 EXPORT_SYMBOL(skb_checksum);
3516
3517 /* Both of above in one bottle. */
3518
skb_copy_and_csum_bits(const struct sk_buff * skb,int offset,u8 * to,int len)3519 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
3520 u8 *to, int len)
3521 {
3522 int start = skb_headlen(skb);
3523 int i, copy = start - offset;
3524 struct sk_buff *frag_iter;
3525 int pos = 0;
3526 __wsum csum = 0;
3527
3528 /* Copy header. */
3529 if (copy > 0) {
3530 if (copy > len)
3531 copy = len;
3532 csum = csum_partial_copy_nocheck(skb->data + offset, to,
3533 copy);
3534 if ((len -= copy) == 0)
3535 return csum;
3536 offset += copy;
3537 to += copy;
3538 pos = copy;
3539 }
3540
3541 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3542 int end;
3543
3544 WARN_ON(start > offset + len);
3545
3546 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3547 if ((copy = end - offset) > 0) {
3548 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3549 u32 p_off, p_len, copied;
3550 struct page *p;
3551 __wsum csum2;
3552 u8 *vaddr;
3553
3554 if (copy > len)
3555 copy = len;
3556
3557 skb_frag_foreach_page(frag,
3558 skb_frag_off(frag) + offset - start,
3559 copy, p, p_off, p_len, copied) {
3560 vaddr = kmap_atomic(p);
3561 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
3562 to + copied,
3563 p_len);
3564 kunmap_atomic(vaddr);
3565 csum = csum_block_add(csum, csum2, pos);
3566 pos += p_len;
3567 }
3568
3569 if (!(len -= copy))
3570 return csum;
3571 offset += copy;
3572 to += copy;
3573 }
3574 start = end;
3575 }
3576
3577 skb_walk_frags(skb, frag_iter) {
3578 __wsum csum2;
3579 int end;
3580
3581 WARN_ON(start > offset + len);
3582
3583 end = start + frag_iter->len;
3584 if ((copy = end - offset) > 0) {
3585 if (copy > len)
3586 copy = len;
3587 csum2 = skb_copy_and_csum_bits(frag_iter,
3588 offset - start,
3589 to, copy);
3590 csum = csum_block_add(csum, csum2, pos);
3591 if ((len -= copy) == 0)
3592 return csum;
3593 offset += copy;
3594 to += copy;
3595 pos += copy;
3596 }
3597 start = end;
3598 }
3599 BUG_ON(len);
3600 return csum;
3601 }
3602 EXPORT_SYMBOL(skb_copy_and_csum_bits);
3603
__skb_checksum_complete_head(struct sk_buff * skb,int len)3604 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
3605 {
3606 __sum16 sum;
3607
3608 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
3609 /* See comments in __skb_checksum_complete(). */
3610 if (likely(!sum)) {
3611 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3612 !skb->csum_complete_sw)
3613 netdev_rx_csum_fault(skb->dev, skb);
3614 }
3615 if (!skb_shared(skb))
3616 skb->csum_valid = !sum;
3617 return sum;
3618 }
3619 EXPORT_SYMBOL(__skb_checksum_complete_head);
3620
3621 /* This function assumes skb->csum already holds pseudo header's checksum,
3622 * which has been changed from the hardware checksum, for example, by
3623 * __skb_checksum_validate_complete(). And, the original skb->csum must
3624 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
3625 *
3626 * It returns non-zero if the recomputed checksum is still invalid, otherwise
3627 * zero. The new checksum is stored back into skb->csum unless the skb is
3628 * shared.
3629 */
__skb_checksum_complete(struct sk_buff * skb)3630 __sum16 __skb_checksum_complete(struct sk_buff *skb)
3631 {
3632 __wsum csum;
3633 __sum16 sum;
3634
3635 csum = skb_checksum(skb, 0, skb->len, 0);
3636
3637 sum = csum_fold(csum_add(skb->csum, csum));
3638 /* This check is inverted, because we already knew the hardware
3639 * checksum is invalid before calling this function. So, if the
3640 * re-computed checksum is valid instead, then we have a mismatch
3641 * between the original skb->csum and skb_checksum(). This means either
3642 * the original hardware checksum is incorrect or we screw up skb->csum
3643 * when moving skb->data around.
3644 */
3645 if (likely(!sum)) {
3646 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
3647 !skb->csum_complete_sw)
3648 netdev_rx_csum_fault(skb->dev, skb);
3649 }
3650
3651 if (!skb_shared(skb)) {
3652 /* Save full packet checksum */
3653 skb->csum = csum;
3654 skb->ip_summed = CHECKSUM_COMPLETE;
3655 skb->csum_complete_sw = 1;
3656 skb->csum_valid = !sum;
3657 }
3658
3659 return sum;
3660 }
3661 EXPORT_SYMBOL(__skb_checksum_complete);
3662
warn_crc32c_csum_update(const void * buff,int len,__wsum sum)3663 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
3664 {
3665 net_warn_ratelimited(
3666 "%s: attempt to compute crc32c without libcrc32c.ko\n",
3667 __func__);
3668 return 0;
3669 }
3670
warn_crc32c_csum_combine(__wsum csum,__wsum csum2,int offset,int len)3671 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
3672 int offset, int len)
3673 {
3674 net_warn_ratelimited(
3675 "%s: attempt to compute crc32c without libcrc32c.ko\n",
3676 __func__);
3677 return 0;
3678 }
3679
3680 static const struct skb_checksum_ops default_crc32c_ops = {
3681 .update = warn_crc32c_csum_update,
3682 .combine = warn_crc32c_csum_combine,
3683 };
3684
3685 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
3686 &default_crc32c_ops;
3687 EXPORT_SYMBOL(crc32c_csum_stub);
3688
3689 /**
3690 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
3691 * @from: source buffer
3692 *
3693 * Calculates the amount of linear headroom needed in the 'to' skb passed
3694 * into skb_zerocopy().
3695 */
3696 unsigned int
skb_zerocopy_headlen(const struct sk_buff * from)3697 skb_zerocopy_headlen(const struct sk_buff *from)
3698 {
3699 unsigned int hlen = 0;
3700
3701 if (!from->head_frag ||
3702 skb_headlen(from) < L1_CACHE_BYTES ||
3703 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) {
3704 hlen = skb_headlen(from);
3705 if (!hlen)
3706 hlen = from->len;
3707 }
3708
3709 if (skb_has_frag_list(from))
3710 hlen = from->len;
3711
3712 return hlen;
3713 }
3714 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
3715
3716 /**
3717 * skb_zerocopy - Zero copy skb to skb
3718 * @to: destination buffer
3719 * @from: source buffer
3720 * @len: number of bytes to copy from source buffer
3721 * @hlen: size of linear headroom in destination buffer
3722 *
3723 * Copies up to `len` bytes from `from` to `to` by creating references
3724 * to the frags in the source buffer.
3725 *
3726 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the
3727 * headroom in the `to` buffer.
3728 *
3729 * Return value:
3730 * 0: everything is OK
3731 * -ENOMEM: couldn't orphan frags of @from due to lack of memory
3732 * -EFAULT: skb_copy_bits() found some problem with skb geometry
3733 */
3734 int
skb_zerocopy(struct sk_buff * to,struct sk_buff * from,int len,int hlen)3735 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
3736 {
3737 int i, j = 0;
3738 int plen = 0; /* length of skb->head fragment */
3739 int ret;
3740 struct page *page;
3741 unsigned int offset;
3742
3743 BUG_ON(!from->head_frag && !hlen);
3744
3745 /* dont bother with small payloads */
3746 if (len <= skb_tailroom(to))
3747 return skb_copy_bits(from, 0, skb_put(to, len), len);
3748
3749 if (hlen) {
3750 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
3751 if (unlikely(ret))
3752 return ret;
3753 len -= hlen;
3754 } else {
3755 plen = min_t(int, skb_headlen(from), len);
3756 if (plen) {
3757 page = virt_to_head_page(from->head);
3758 offset = from->data - (unsigned char *)page_address(page);
3759 __skb_fill_netmem_desc(to, 0, page_to_netmem(page),
3760 offset, plen);
3761 get_page(page);
3762 j = 1;
3763 len -= plen;
3764 }
3765 }
3766
3767 skb_len_add(to, len + plen);
3768
3769 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
3770 skb_tx_error(from);
3771 return -ENOMEM;
3772 }
3773 skb_zerocopy_clone(to, from, GFP_ATOMIC);
3774
3775 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
3776 int size;
3777
3778 if (!len)
3779 break;
3780 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
3781 size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
3782 len);
3783 skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
3784 len -= size;
3785 skb_frag_ref(to, j);
3786 j++;
3787 }
3788 skb_shinfo(to)->nr_frags = j;
3789
3790 return 0;
3791 }
3792 EXPORT_SYMBOL_GPL(skb_zerocopy);
3793
skb_copy_and_csum_dev(const struct sk_buff * skb,u8 * to)3794 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3795 {
3796 __wsum csum;
3797 long csstart;
3798
3799 if (skb->ip_summed == CHECKSUM_PARTIAL)
3800 csstart = skb_checksum_start_offset(skb);
3801 else
3802 csstart = skb_headlen(skb);
3803
3804 BUG_ON(csstart > skb_headlen(skb));
3805
3806 skb_copy_from_linear_data(skb, to, csstart);
3807
3808 csum = 0;
3809 if (csstart != skb->len)
3810 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3811 skb->len - csstart);
3812
3813 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3814 long csstuff = csstart + skb->csum_offset;
3815
3816 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
3817 }
3818 }
3819 EXPORT_SYMBOL(skb_copy_and_csum_dev);
3820
3821 /**
3822 * skb_dequeue - remove from the head of the queue
3823 * @list: list to dequeue from
3824 *
3825 * Remove the head of the list. The list lock is taken so the function
3826 * may be used safely with other locking list functions. The head item is
3827 * returned or %NULL if the list is empty.
3828 */
3829
skb_dequeue(struct sk_buff_head * list)3830 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
3831 {
3832 unsigned long flags;
3833 struct sk_buff *result;
3834
3835 spin_lock_irqsave(&list->lock, flags);
3836 result = __skb_dequeue(list);
3837 spin_unlock_irqrestore(&list->lock, flags);
3838 return result;
3839 }
3840 EXPORT_SYMBOL(skb_dequeue);
3841
3842 /**
3843 * skb_dequeue_tail - remove from the tail of the queue
3844 * @list: list to dequeue from
3845 *
3846 * Remove the tail of the list. The list lock is taken so the function
3847 * may be used safely with other locking list functions. The tail item is
3848 * returned or %NULL if the list is empty.
3849 */
skb_dequeue_tail(struct sk_buff_head * list)3850 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
3851 {
3852 unsigned long flags;
3853 struct sk_buff *result;
3854
3855 spin_lock_irqsave(&list->lock, flags);
3856 result = __skb_dequeue_tail(list);
3857 spin_unlock_irqrestore(&list->lock, flags);
3858 return result;
3859 }
3860 EXPORT_SYMBOL(skb_dequeue_tail);
3861
3862 /**
3863 * skb_queue_purge_reason - empty a list
3864 * @list: list to empty
3865 * @reason: drop reason
3866 *
3867 * Delete all buffers on an &sk_buff list. Each buffer is removed from
3868 * the list and one reference dropped. This function takes the list
3869 * lock and is atomic with respect to other list locking functions.
3870 */
skb_queue_purge_reason(struct sk_buff_head * list,enum skb_drop_reason reason)3871 void skb_queue_purge_reason(struct sk_buff_head *list,
3872 enum skb_drop_reason reason)
3873 {
3874 struct sk_buff_head tmp;
3875 unsigned long flags;
3876
3877 if (skb_queue_empty_lockless(list))
3878 return;
3879
3880 __skb_queue_head_init(&tmp);
3881
3882 spin_lock_irqsave(&list->lock, flags);
3883 skb_queue_splice_init(list, &tmp);
3884 spin_unlock_irqrestore(&list->lock, flags);
3885
3886 __skb_queue_purge_reason(&tmp, reason);
3887 }
3888 EXPORT_SYMBOL(skb_queue_purge_reason);
3889
3890 /**
3891 * skb_rbtree_purge - empty a skb rbtree
3892 * @root: root of the rbtree to empty
3893 * Return value: the sum of truesizes of all purged skbs.
3894 *
3895 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
3896 * the list and one reference dropped. This function does not take
3897 * any lock. Synchronization should be handled by the caller (e.g., TCP
3898 * out-of-order queue is protected by the socket lock).
3899 */
skb_rbtree_purge(struct rb_root * root)3900 unsigned int skb_rbtree_purge(struct rb_root *root)
3901 {
3902 struct rb_node *p = rb_first(root);
3903 unsigned int sum = 0;
3904
3905 while (p) {
3906 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3907
3908 p = rb_next(p);
3909 rb_erase(&skb->rbnode, root);
3910 sum += skb->truesize;
3911 kfree_skb(skb);
3912 }
3913 return sum;
3914 }
3915
skb_errqueue_purge(struct sk_buff_head * list)3916 void skb_errqueue_purge(struct sk_buff_head *list)
3917 {
3918 struct sk_buff *skb, *next;
3919 struct sk_buff_head kill;
3920 unsigned long flags;
3921
3922 __skb_queue_head_init(&kill);
3923
3924 spin_lock_irqsave(&list->lock, flags);
3925 skb_queue_walk_safe(list, skb, next) {
3926 if (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ZEROCOPY ||
3927 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING)
3928 continue;
3929 __skb_unlink(skb, list);
3930 __skb_queue_tail(&kill, skb);
3931 }
3932 spin_unlock_irqrestore(&list->lock, flags);
3933 __skb_queue_purge(&kill);
3934 }
3935 EXPORT_SYMBOL(skb_errqueue_purge);
3936
3937 /**
3938 * skb_queue_head - queue a buffer at the list head
3939 * @list: list to use
3940 * @newsk: buffer to queue
3941 *
3942 * Queue a buffer at the start of the list. This function takes the
3943 * list lock and can be used safely with other locking &sk_buff functions
3944 * safely.
3945 *
3946 * A buffer cannot be placed on two lists at the same time.
3947 */
skb_queue_head(struct sk_buff_head * list,struct sk_buff * newsk)3948 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
3949 {
3950 unsigned long flags;
3951
3952 spin_lock_irqsave(&list->lock, flags);
3953 __skb_queue_head(list, newsk);
3954 spin_unlock_irqrestore(&list->lock, flags);
3955 }
3956 EXPORT_SYMBOL(skb_queue_head);
3957
3958 /**
3959 * skb_queue_tail - queue a buffer at the list tail
3960 * @list: list to use
3961 * @newsk: buffer to queue
3962 *
3963 * Queue a buffer at the tail of the list. This function takes the
3964 * list lock and can be used safely with other locking &sk_buff functions
3965 * safely.
3966 *
3967 * A buffer cannot be placed on two lists at the same time.
3968 */
skb_queue_tail(struct sk_buff_head * list,struct sk_buff * newsk)3969 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
3970 {
3971 unsigned long flags;
3972
3973 spin_lock_irqsave(&list->lock, flags);
3974 __skb_queue_tail(list, newsk);
3975 spin_unlock_irqrestore(&list->lock, flags);
3976 }
3977 EXPORT_SYMBOL(skb_queue_tail);
3978
3979 /**
3980 * skb_unlink - remove a buffer from a list
3981 * @skb: buffer to remove
3982 * @list: list to use
3983 *
3984 * Remove a packet from a list. The list locks are taken and this
3985 * function is atomic with respect to other list locked calls
3986 *
3987 * You must know what list the SKB is on.
3988 */
skb_unlink(struct sk_buff * skb,struct sk_buff_head * list)3989 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
3990 {
3991 unsigned long flags;
3992
3993 spin_lock_irqsave(&list->lock, flags);
3994 __skb_unlink(skb, list);
3995 spin_unlock_irqrestore(&list->lock, flags);
3996 }
3997 EXPORT_SYMBOL(skb_unlink);
3998
3999 /**
4000 * skb_append - append a buffer
4001 * @old: buffer to insert after
4002 * @newsk: buffer to insert
4003 * @list: list to use
4004 *
4005 * Place a packet after a given packet in a list. The list locks are taken
4006 * and this function is atomic with respect to other list locked calls.
4007 * A buffer cannot be placed on two lists at the same time.
4008 */
skb_append(struct sk_buff * old,struct sk_buff * newsk,struct sk_buff_head * list)4009 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
4010 {
4011 unsigned long flags;
4012
4013 spin_lock_irqsave(&list->lock, flags);
4014 __skb_queue_after(list, old, newsk);
4015 spin_unlock_irqrestore(&list->lock, flags);
4016 }
4017 EXPORT_SYMBOL(skb_append);
4018
skb_split_inside_header(struct sk_buff * skb,struct sk_buff * skb1,const u32 len,const int pos)4019 static inline void skb_split_inside_header(struct sk_buff *skb,
4020 struct sk_buff* skb1,
4021 const u32 len, const int pos)
4022 {
4023 int i;
4024
4025 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
4026 pos - len);
4027 /* And move data appendix as is. */
4028 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
4029 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
4030
4031 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
4032 skb_shinfo(skb)->nr_frags = 0;
4033 skb1->data_len = skb->data_len;
4034 skb1->len += skb1->data_len;
4035 skb->data_len = 0;
4036 skb->len = len;
4037 skb_set_tail_pointer(skb, len);
4038 }
4039
skb_split_no_header(struct sk_buff * skb,struct sk_buff * skb1,const u32 len,int pos)4040 static inline void skb_split_no_header(struct sk_buff *skb,
4041 struct sk_buff* skb1,
4042 const u32 len, int pos)
4043 {
4044 int i, k = 0;
4045 const int nfrags = skb_shinfo(skb)->nr_frags;
4046
4047 skb_shinfo(skb)->nr_frags = 0;
4048 skb1->len = skb1->data_len = skb->len - len;
4049 skb->len = len;
4050 skb->data_len = len - pos;
4051
4052 for (i = 0; i < nfrags; i++) {
4053 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
4054
4055 if (pos + size > len) {
4056 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
4057
4058 if (pos < len) {
4059 /* Split frag.
4060 * We have two variants in this case:
4061 * 1. Move all the frag to the second
4062 * part, if it is possible. F.e.
4063 * this approach is mandatory for TUX,
4064 * where splitting is expensive.
4065 * 2. Split is accurately. We make this.
4066 */
4067 skb_frag_ref(skb, i);
4068 skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
4069 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
4070 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
4071 skb_shinfo(skb)->nr_frags++;
4072 }
4073 k++;
4074 } else
4075 skb_shinfo(skb)->nr_frags++;
4076 pos += size;
4077 }
4078 skb_shinfo(skb1)->nr_frags = k;
4079 }
4080
4081 /**
4082 * skb_split - Split fragmented skb to two parts at length len.
4083 * @skb: the buffer to split
4084 * @skb1: the buffer to receive the second part
4085 * @len: new length for skb
4086 */
skb_split(struct sk_buff * skb,struct sk_buff * skb1,const u32 len)4087 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
4088 {
4089 int pos = skb_headlen(skb);
4090 const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
4091
4092 skb_zcopy_downgrade_managed(skb);
4093
4094 skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
4095 skb_zerocopy_clone(skb1, skb, 0);
4096 if (len < pos) /* Split line is inside header. */
4097 skb_split_inside_header(skb, skb1, len, pos);
4098 else /* Second chunk has no header, nothing to copy. */
4099 skb_split_no_header(skb, skb1, len, pos);
4100 }
4101 EXPORT_SYMBOL(skb_split);
4102
4103 /* Shifting from/to a cloned skb is a no-go.
4104 *
4105 * Caller cannot keep skb_shinfo related pointers past calling here!
4106 */
skb_prepare_for_shift(struct sk_buff * skb)4107 static int skb_prepare_for_shift(struct sk_buff *skb)
4108 {
4109 return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
4110 }
4111
4112 /**
4113 * skb_shift - Shifts paged data partially from skb to another
4114 * @tgt: buffer into which tail data gets added
4115 * @skb: buffer from which the paged data comes from
4116 * @shiftlen: shift up to this many bytes
4117 *
4118 * Attempts to shift up to shiftlen worth of bytes, which may be less than
4119 * the length of the skb, from skb to tgt. Returns number bytes shifted.
4120 * It's up to caller to free skb if everything was shifted.
4121 *
4122 * If @tgt runs out of frags, the whole operation is aborted.
4123 *
4124 * Skb cannot include anything else but paged data while tgt is allowed
4125 * to have non-paged data as well.
4126 *
4127 * TODO: full sized shift could be optimized but that would need
4128 * specialized skb free'er to handle frags without up-to-date nr_frags.
4129 */
skb_shift(struct sk_buff * tgt,struct sk_buff * skb,int shiftlen)4130 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
4131 {
4132 int from, to, merge, todo;
4133 skb_frag_t *fragfrom, *fragto;
4134
4135 BUG_ON(shiftlen > skb->len);
4136
4137 if (skb_headlen(skb))
4138 return 0;
4139 if (skb_zcopy(tgt) || skb_zcopy(skb))
4140 return 0;
4141
4142 todo = shiftlen;
4143 from = 0;
4144 to = skb_shinfo(tgt)->nr_frags;
4145 fragfrom = &skb_shinfo(skb)->frags[from];
4146
4147 /* Actual merge is delayed until the point when we know we can
4148 * commit all, so that we don't have to undo partial changes
4149 */
4150 if (!to ||
4151 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
4152 skb_frag_off(fragfrom))) {
4153 merge = -1;
4154 } else {
4155 merge = to - 1;
4156
4157 todo -= skb_frag_size(fragfrom);
4158 if (todo < 0) {
4159 if (skb_prepare_for_shift(skb) ||
4160 skb_prepare_for_shift(tgt))
4161 return 0;
4162
4163 /* All previous frag pointers might be stale! */
4164 fragfrom = &skb_shinfo(skb)->frags[from];
4165 fragto = &skb_shinfo(tgt)->frags[merge];
4166
4167 skb_frag_size_add(fragto, shiftlen);
4168 skb_frag_size_sub(fragfrom, shiftlen);
4169 skb_frag_off_add(fragfrom, shiftlen);
4170
4171 goto onlymerged;
4172 }
4173
4174 from++;
4175 }
4176
4177 /* Skip full, not-fitting skb to avoid expensive operations */
4178 if ((shiftlen == skb->len) &&
4179 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
4180 return 0;
4181
4182 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
4183 return 0;
4184
4185 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
4186 if (to == MAX_SKB_FRAGS)
4187 return 0;
4188
4189 fragfrom = &skb_shinfo(skb)->frags[from];
4190 fragto = &skb_shinfo(tgt)->frags[to];
4191
4192 if (todo >= skb_frag_size(fragfrom)) {
4193 *fragto = *fragfrom;
4194 todo -= skb_frag_size(fragfrom);
4195 from++;
4196 to++;
4197
4198 } else {
4199 __skb_frag_ref(fragfrom);
4200 skb_frag_page_copy(fragto, fragfrom);
4201 skb_frag_off_copy(fragto, fragfrom);
4202 skb_frag_size_set(fragto, todo);
4203
4204 skb_frag_off_add(fragfrom, todo);
4205 skb_frag_size_sub(fragfrom, todo);
4206 todo = 0;
4207
4208 to++;
4209 break;
4210 }
4211 }
4212
4213 /* Ready to "commit" this state change to tgt */
4214 skb_shinfo(tgt)->nr_frags = to;
4215
4216 if (merge >= 0) {
4217 fragfrom = &skb_shinfo(skb)->frags[0];
4218 fragto = &skb_shinfo(tgt)->frags[merge];
4219
4220 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
4221 __skb_frag_unref(fragfrom, skb->pp_recycle);
4222 }
4223
4224 /* Reposition in the original skb */
4225 to = 0;
4226 while (from < skb_shinfo(skb)->nr_frags)
4227 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
4228 skb_shinfo(skb)->nr_frags = to;
4229
4230 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
4231
4232 onlymerged:
4233 /* Most likely the tgt won't ever need its checksum anymore, skb on
4234 * the other hand might need it if it needs to be resent
4235 */
4236 tgt->ip_summed = CHECKSUM_PARTIAL;
4237 skb->ip_summed = CHECKSUM_PARTIAL;
4238
4239 skb_len_add(skb, -shiftlen);
4240 skb_len_add(tgt, shiftlen);
4241
4242 return shiftlen;
4243 }
4244
4245 /**
4246 * skb_prepare_seq_read - Prepare a sequential read of skb data
4247 * @skb: the buffer to read
4248 * @from: lower offset of data to be read
4249 * @to: upper offset of data to be read
4250 * @st: state variable
4251 *
4252 * Initializes the specified state variable. Must be called before
4253 * invoking skb_seq_read() for the first time.
4254 */
skb_prepare_seq_read(struct sk_buff * skb,unsigned int from,unsigned int to,struct skb_seq_state * st)4255 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
4256 unsigned int to, struct skb_seq_state *st)
4257 {
4258 st->lower_offset = from;
4259 st->upper_offset = to;
4260 st->root_skb = st->cur_skb = skb;
4261 st->frag_idx = st->stepped_offset = 0;
4262 st->frag_data = NULL;
4263 st->frag_off = 0;
4264 }
4265 EXPORT_SYMBOL(skb_prepare_seq_read);
4266
4267 /**
4268 * skb_seq_read - Sequentially read skb data
4269 * @consumed: number of bytes consumed by the caller so far
4270 * @data: destination pointer for data to be returned
4271 * @st: state variable
4272 *
4273 * Reads a block of skb data at @consumed relative to the
4274 * lower offset specified to skb_prepare_seq_read(). Assigns
4275 * the head of the data block to @data and returns the length
4276 * of the block or 0 if the end of the skb data or the upper
4277 * offset has been reached.
4278 *
4279 * The caller is not required to consume all of the data
4280 * returned, i.e. @consumed is typically set to the number
4281 * of bytes already consumed and the next call to
4282 * skb_seq_read() will return the remaining part of the block.
4283 *
4284 * Note 1: The size of each block of data returned can be arbitrary,
4285 * this limitation is the cost for zerocopy sequential
4286 * reads of potentially non linear data.
4287 *
4288 * Note 2: Fragment lists within fragments are not implemented
4289 * at the moment, state->root_skb could be replaced with
4290 * a stack for this purpose.
4291 */
skb_seq_read(unsigned int consumed,const u8 ** data,struct skb_seq_state * st)4292 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
4293 struct skb_seq_state *st)
4294 {
4295 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
4296 skb_frag_t *frag;
4297
4298 if (unlikely(abs_offset >= st->upper_offset)) {
4299 if (st->frag_data) {
4300 kunmap_atomic(st->frag_data);
4301 st->frag_data = NULL;
4302 }
4303 return 0;
4304 }
4305
4306 next_skb:
4307 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
4308
4309 if (abs_offset < block_limit && !st->frag_data) {
4310 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
4311 return block_limit - abs_offset;
4312 }
4313
4314 if (st->frag_idx == 0 && !st->frag_data)
4315 st->stepped_offset += skb_headlen(st->cur_skb);
4316
4317 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
4318 unsigned int pg_idx, pg_off, pg_sz;
4319
4320 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
4321
4322 pg_idx = 0;
4323 pg_off = skb_frag_off(frag);
4324 pg_sz = skb_frag_size(frag);
4325
4326 if (skb_frag_must_loop(skb_frag_page(frag))) {
4327 pg_idx = (pg_off + st->frag_off) >> PAGE_SHIFT;
4328 pg_off = offset_in_page(pg_off + st->frag_off);
4329 pg_sz = min_t(unsigned int, pg_sz - st->frag_off,
4330 PAGE_SIZE - pg_off);
4331 }
4332
4333 block_limit = pg_sz + st->stepped_offset;
4334 if (abs_offset < block_limit) {
4335 if (!st->frag_data)
4336 st->frag_data = kmap_atomic(skb_frag_page(frag) + pg_idx);
4337
4338 *data = (u8 *)st->frag_data + pg_off +
4339 (abs_offset - st->stepped_offset);
4340
4341 return block_limit - abs_offset;
4342 }
4343
4344 if (st->frag_data) {
4345 kunmap_atomic(st->frag_data);
4346 st->frag_data = NULL;
4347 }
4348
4349 st->stepped_offset += pg_sz;
4350 st->frag_off += pg_sz;
4351 if (st->frag_off == skb_frag_size(frag)) {
4352 st->frag_off = 0;
4353 st->frag_idx++;
4354 }
4355 }
4356
4357 if (st->frag_data) {
4358 kunmap_atomic(st->frag_data);
4359 st->frag_data = NULL;
4360 }
4361
4362 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
4363 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
4364 st->frag_idx = 0;
4365 goto next_skb;
4366 } else if (st->cur_skb->next) {
4367 st->cur_skb = st->cur_skb->next;
4368 st->frag_idx = 0;
4369 goto next_skb;
4370 }
4371
4372 return 0;
4373 }
4374 EXPORT_SYMBOL(skb_seq_read);
4375
4376 /**
4377 * skb_abort_seq_read - Abort a sequential read of skb data
4378 * @st: state variable
4379 *
4380 * Must be called if skb_seq_read() was not called until it
4381 * returned 0.
4382 */
skb_abort_seq_read(struct skb_seq_state * st)4383 void skb_abort_seq_read(struct skb_seq_state *st)
4384 {
4385 if (st->frag_data)
4386 kunmap_atomic(st->frag_data);
4387 }
4388 EXPORT_SYMBOL(skb_abort_seq_read);
4389
4390 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
4391
skb_ts_get_next_block(unsigned int offset,const u8 ** text,struct ts_config * conf,struct ts_state * state)4392 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
4393 struct ts_config *conf,
4394 struct ts_state *state)
4395 {
4396 return skb_seq_read(offset, text, TS_SKB_CB(state));
4397 }
4398
skb_ts_finish(struct ts_config * conf,struct ts_state * state)4399 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
4400 {
4401 skb_abort_seq_read(TS_SKB_CB(state));
4402 }
4403
4404 /**
4405 * skb_find_text - Find a text pattern in skb data
4406 * @skb: the buffer to look in
4407 * @from: search offset
4408 * @to: search limit
4409 * @config: textsearch configuration
4410 *
4411 * Finds a pattern in the skb data according to the specified
4412 * textsearch configuration. Use textsearch_next() to retrieve
4413 * subsequent occurrences of the pattern. Returns the offset
4414 * to the first occurrence or UINT_MAX if no match was found.
4415 */
skb_find_text(struct sk_buff * skb,unsigned int from,unsigned int to,struct ts_config * config)4416 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
4417 unsigned int to, struct ts_config *config)
4418 {
4419 unsigned int patlen = config->ops->get_pattern_len(config);
4420 struct ts_state state;
4421 unsigned int ret;
4422
4423 BUILD_BUG_ON(sizeof(struct skb_seq_state) > sizeof(state.cb));
4424
4425 config->get_next_block = skb_ts_get_next_block;
4426 config->finish = skb_ts_finish;
4427
4428 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
4429
4430 ret = textsearch_find(config, &state);
4431 return (ret + patlen <= to - from ? ret : UINT_MAX);
4432 }
4433 EXPORT_SYMBOL(skb_find_text);
4434
skb_append_pagefrags(struct sk_buff * skb,struct page * page,int offset,size_t size,size_t max_frags)4435 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
4436 int offset, size_t size, size_t max_frags)
4437 {
4438 int i = skb_shinfo(skb)->nr_frags;
4439
4440 if (skb_can_coalesce(skb, i, page, offset)) {
4441 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
4442 } else if (i < max_frags) {
4443 skb_zcopy_downgrade_managed(skb);
4444 get_page(page);
4445 skb_fill_page_desc_noacc(skb, i, page, offset, size);
4446 } else {
4447 return -EMSGSIZE;
4448 }
4449
4450 return 0;
4451 }
4452 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
4453
4454 /**
4455 * skb_pull_rcsum - pull skb and update receive checksum
4456 * @skb: buffer to update
4457 * @len: length of data pulled
4458 *
4459 * This function performs an skb_pull on the packet and updates
4460 * the CHECKSUM_COMPLETE checksum. It should be used on
4461 * receive path processing instead of skb_pull unless you know
4462 * that the checksum difference is zero (e.g., a valid IP header)
4463 * or you are setting ip_summed to CHECKSUM_NONE.
4464 */
skb_pull_rcsum(struct sk_buff * skb,unsigned int len)4465 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
4466 {
4467 unsigned char *data = skb->data;
4468
4469 BUG_ON(len > skb->len);
4470 __skb_pull(skb, len);
4471 skb_postpull_rcsum(skb, data, len);
4472 return skb->data;
4473 }
4474 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
4475
skb_head_frag_to_page_desc(struct sk_buff * frag_skb)4476 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
4477 {
4478 skb_frag_t head_frag;
4479 struct page *page;
4480
4481 page = virt_to_head_page(frag_skb->head);
4482 skb_frag_fill_page_desc(&head_frag, page, frag_skb->data -
4483 (unsigned char *)page_address(page),
4484 skb_headlen(frag_skb));
4485 return head_frag;
4486 }
4487
skb_segment_list(struct sk_buff * skb,netdev_features_t features,unsigned int offset)4488 struct sk_buff *skb_segment_list(struct sk_buff *skb,
4489 netdev_features_t features,
4490 unsigned int offset)
4491 {
4492 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
4493 unsigned int tnl_hlen = skb_tnl_header_len(skb);
4494 unsigned int delta_truesize = 0;
4495 unsigned int delta_len = 0;
4496 struct sk_buff *tail = NULL;
4497 struct sk_buff *nskb, *tmp;
4498 int len_diff, err;
4499
4500 skb_push(skb, -skb_network_offset(skb) + offset);
4501
4502 /* Ensure the head is writeable before touching the shared info */
4503 err = skb_unclone(skb, GFP_ATOMIC);
4504 if (err)
4505 goto err_linearize;
4506
4507 skb_shinfo(skb)->frag_list = NULL;
4508
4509 while (list_skb) {
4510 nskb = list_skb;
4511 list_skb = list_skb->next;
4512
4513 err = 0;
4514 delta_truesize += nskb->truesize;
4515 if (skb_shared(nskb)) {
4516 tmp = skb_clone(nskb, GFP_ATOMIC);
4517 if (tmp) {
4518 consume_skb(nskb);
4519 nskb = tmp;
4520 err = skb_unclone(nskb, GFP_ATOMIC);
4521 } else {
4522 err = -ENOMEM;
4523 }
4524 }
4525
4526 if (!tail)
4527 skb->next = nskb;
4528 else
4529 tail->next = nskb;
4530
4531 if (unlikely(err)) {
4532 nskb->next = list_skb;
4533 goto err_linearize;
4534 }
4535
4536 tail = nskb;
4537
4538 delta_len += nskb->len;
4539
4540 skb_push(nskb, -skb_network_offset(nskb) + offset);
4541
4542 skb_release_head_state(nskb);
4543 len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
4544 __copy_skb_header(nskb, skb);
4545
4546 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
4547 nskb->transport_header += len_diff;
4548 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
4549 nskb->data - tnl_hlen,
4550 offset + tnl_hlen);
4551
4552 if (skb_needs_linearize(nskb, features) &&
4553 __skb_linearize(nskb))
4554 goto err_linearize;
4555 }
4556
4557 skb->truesize = skb->truesize - delta_truesize;
4558 skb->data_len = skb->data_len - delta_len;
4559 skb->len = skb->len - delta_len;
4560
4561 skb_gso_reset(skb);
4562
4563 skb->prev = tail;
4564
4565 if (skb_needs_linearize(skb, features) &&
4566 __skb_linearize(skb))
4567 goto err_linearize;
4568
4569 skb_get(skb);
4570
4571 return skb;
4572
4573 err_linearize:
4574 kfree_skb_list(skb->next);
4575 skb->next = NULL;
4576 return ERR_PTR(-ENOMEM);
4577 }
4578 EXPORT_SYMBOL_GPL(skb_segment_list);
4579
4580 /**
4581 * skb_segment - Perform protocol segmentation on skb.
4582 * @head_skb: buffer to segment
4583 * @features: features for the output path (see dev->features)
4584 *
4585 * This function performs segmentation on the given skb. It returns
4586 * a pointer to the first in a list of new skbs for the segments.
4587 * In case of error it returns ERR_PTR(err).
4588 */
skb_segment(struct sk_buff * head_skb,netdev_features_t features)4589 struct sk_buff *skb_segment(struct sk_buff *head_skb,
4590 netdev_features_t features)
4591 {
4592 struct sk_buff *segs = NULL;
4593 struct sk_buff *tail = NULL;
4594 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
4595 unsigned int mss = skb_shinfo(head_skb)->gso_size;
4596 unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
4597 unsigned int offset = doffset;
4598 unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
4599 unsigned int partial_segs = 0;
4600 unsigned int headroom;
4601 unsigned int len = head_skb->len;
4602 struct sk_buff *frag_skb;
4603 skb_frag_t *frag;
4604 __be16 proto;
4605 bool csum, sg;
4606 int err = -ENOMEM;
4607 int i = 0;
4608 int nfrags, pos;
4609
4610 if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
4611 mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) {
4612 struct sk_buff *check_skb;
4613
4614 for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
4615 if (skb_headlen(check_skb) && !check_skb->head_frag) {
4616 /* gso_size is untrusted, and we have a frag_list with
4617 * a linear non head_frag item.
4618 *
4619 * If head_skb's headlen does not fit requested gso_size,
4620 * it means that the frag_list members do NOT terminate
4621 * on exact gso_size boundaries. Hence we cannot perform
4622 * skb_frag_t page sharing. Therefore we must fallback to
4623 * copying the frag_list skbs; we do so by disabling SG.
4624 */
4625 features &= ~NETIF_F_SG;
4626 break;
4627 }
4628 }
4629 }
4630
4631 __skb_push(head_skb, doffset);
4632 proto = skb_network_protocol(head_skb, NULL);
4633 if (unlikely(!proto))
4634 return ERR_PTR(-EINVAL);
4635
4636 sg = !!(features & NETIF_F_SG);
4637 csum = !!can_checksum_protocol(features, proto);
4638
4639 if (sg && csum && (mss != GSO_BY_FRAGS)) {
4640 if (!(features & NETIF_F_GSO_PARTIAL)) {
4641 struct sk_buff *iter;
4642 unsigned int frag_len;
4643
4644 if (!list_skb ||
4645 !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
4646 goto normal;
4647
4648 /* If we get here then all the required
4649 * GSO features except frag_list are supported.
4650 * Try to split the SKB to multiple GSO SKBs
4651 * with no frag_list.
4652 * Currently we can do that only when the buffers don't
4653 * have a linear part and all the buffers except
4654 * the last are of the same length.
4655 */
4656 frag_len = list_skb->len;
4657 skb_walk_frags(head_skb, iter) {
4658 if (frag_len != iter->len && iter->next)
4659 goto normal;
4660 if (skb_headlen(iter) && !iter->head_frag)
4661 goto normal;
4662
4663 len -= iter->len;
4664 }
4665
4666 if (len != frag_len)
4667 goto normal;
4668 }
4669
4670 /* GSO partial only requires that we trim off any excess that
4671 * doesn't fit into an MSS sized block, so take care of that
4672 * now.
4673 * Cap len to not accidentally hit GSO_BY_FRAGS.
4674 */
4675 partial_segs = min(len, GSO_BY_FRAGS - 1) / mss;
4676 if (partial_segs > 1)
4677 mss *= partial_segs;
4678 else
4679 partial_segs = 0;
4680 }
4681
4682 normal:
4683 headroom = skb_headroom(head_skb);
4684 pos = skb_headlen(head_skb);
4685
4686 if (skb_orphan_frags(head_skb, GFP_ATOMIC))
4687 return ERR_PTR(-ENOMEM);
4688
4689 nfrags = skb_shinfo(head_skb)->nr_frags;
4690 frag = skb_shinfo(head_skb)->frags;
4691 frag_skb = head_skb;
4692
4693 do {
4694 struct sk_buff *nskb;
4695 skb_frag_t *nskb_frag;
4696 int hsize;
4697 int size;
4698
4699 if (unlikely(mss == GSO_BY_FRAGS)) {
4700 len = list_skb->len;
4701 } else {
4702 len = head_skb->len - offset;
4703 if (len > mss)
4704 len = mss;
4705 }
4706
4707 hsize = skb_headlen(head_skb) - offset;
4708
4709 if (hsize <= 0 && i >= nfrags && skb_headlen(list_skb) &&
4710 (skb_headlen(list_skb) == len || sg)) {
4711 BUG_ON(skb_headlen(list_skb) > len);
4712
4713 nskb = skb_clone(list_skb, GFP_ATOMIC);
4714 if (unlikely(!nskb))
4715 goto err;
4716
4717 i = 0;
4718 nfrags = skb_shinfo(list_skb)->nr_frags;
4719 frag = skb_shinfo(list_skb)->frags;
4720 frag_skb = list_skb;
4721 pos += skb_headlen(list_skb);
4722
4723 while (pos < offset + len) {
4724 BUG_ON(i >= nfrags);
4725
4726 size = skb_frag_size(frag);
4727 if (pos + size > offset + len)
4728 break;
4729
4730 i++;
4731 pos += size;
4732 frag++;
4733 }
4734
4735 list_skb = list_skb->next;
4736
4737 if (unlikely(pskb_trim(nskb, len))) {
4738 kfree_skb(nskb);
4739 goto err;
4740 }
4741
4742 hsize = skb_end_offset(nskb);
4743 if (skb_cow_head(nskb, doffset + headroom)) {
4744 kfree_skb(nskb);
4745 goto err;
4746 }
4747
4748 nskb->truesize += skb_end_offset(nskb) - hsize;
4749 skb_release_head_state(nskb);
4750 __skb_push(nskb, doffset);
4751 } else {
4752 if (hsize < 0)
4753 hsize = 0;
4754 if (hsize > len || !sg)
4755 hsize = len;
4756
4757 nskb = __alloc_skb(hsize + doffset + headroom,
4758 GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
4759 NUMA_NO_NODE);
4760
4761 if (unlikely(!nskb))
4762 goto err;
4763
4764 skb_reserve(nskb, headroom);
4765 __skb_put(nskb, doffset);
4766 }
4767
4768 if (segs)
4769 tail->next = nskb;
4770 else
4771 segs = nskb;
4772 tail = nskb;
4773
4774 __copy_skb_header(nskb, head_skb);
4775
4776 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
4777 skb_reset_mac_len(nskb);
4778
4779 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
4780 nskb->data - tnl_hlen,
4781 doffset + tnl_hlen);
4782
4783 if (nskb->len == len + doffset)
4784 goto perform_csum_check;
4785
4786 if (!sg) {
4787 if (!csum) {
4788 if (!nskb->remcsum_offload)
4789 nskb->ip_summed = CHECKSUM_NONE;
4790 SKB_GSO_CB(nskb)->csum =
4791 skb_copy_and_csum_bits(head_skb, offset,
4792 skb_put(nskb,
4793 len),
4794 len);
4795 SKB_GSO_CB(nskb)->csum_start =
4796 skb_headroom(nskb) + doffset;
4797 } else {
4798 if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
4799 goto err;
4800 }
4801 continue;
4802 }
4803
4804 nskb_frag = skb_shinfo(nskb)->frags;
4805
4806 skb_copy_from_linear_data_offset(head_skb, offset,
4807 skb_put(nskb, hsize), hsize);
4808
4809 skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags &
4810 SKBFL_SHARED_FRAG;
4811
4812 if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
4813 goto err;
4814
4815 while (pos < offset + len) {
4816 if (i >= nfrags) {
4817 if (skb_orphan_frags(list_skb, GFP_ATOMIC) ||
4818 skb_zerocopy_clone(nskb, list_skb,
4819 GFP_ATOMIC))
4820 goto err;
4821
4822 i = 0;
4823 nfrags = skb_shinfo(list_skb)->nr_frags;
4824 frag = skb_shinfo(list_skb)->frags;
4825 frag_skb = list_skb;
4826 if (!skb_headlen(list_skb)) {
4827 BUG_ON(!nfrags);
4828 } else {
4829 BUG_ON(!list_skb->head_frag);
4830
4831 /* to make room for head_frag. */
4832 i--;
4833 frag--;
4834 }
4835
4836 list_skb = list_skb->next;
4837 }
4838
4839 if (unlikely(skb_shinfo(nskb)->nr_frags >=
4840 MAX_SKB_FRAGS)) {
4841 net_warn_ratelimited(
4842 "skb_segment: too many frags: %u %u\n",
4843 pos, mss);
4844 err = -EINVAL;
4845 goto err;
4846 }
4847
4848 *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
4849 __skb_frag_ref(nskb_frag);
4850 size = skb_frag_size(nskb_frag);
4851
4852 if (pos < offset) {
4853 skb_frag_off_add(nskb_frag, offset - pos);
4854 skb_frag_size_sub(nskb_frag, offset - pos);
4855 }
4856
4857 skb_shinfo(nskb)->nr_frags++;
4858
4859 if (pos + size <= offset + len) {
4860 i++;
4861 frag++;
4862 pos += size;
4863 } else {
4864 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
4865 goto skip_fraglist;
4866 }
4867
4868 nskb_frag++;
4869 }
4870
4871 skip_fraglist:
4872 nskb->data_len = len - hsize;
4873 nskb->len += nskb->data_len;
4874 nskb->truesize += nskb->data_len;
4875
4876 perform_csum_check:
4877 if (!csum) {
4878 if (skb_has_shared_frag(nskb) &&
4879 __skb_linearize(nskb))
4880 goto err;
4881
4882 if (!nskb->remcsum_offload)
4883 nskb->ip_summed = CHECKSUM_NONE;
4884 SKB_GSO_CB(nskb)->csum =
4885 skb_checksum(nskb, doffset,
4886 nskb->len - doffset, 0);
4887 SKB_GSO_CB(nskb)->csum_start =
4888 skb_headroom(nskb) + doffset;
4889 }
4890 } while ((offset += len) < head_skb->len);
4891
4892 /* Some callers want to get the end of the list.
4893 * Put it in segs->prev to avoid walking the list.
4894 * (see validate_xmit_skb_list() for example)
4895 */
4896 segs->prev = tail;
4897
4898 if (partial_segs) {
4899 struct sk_buff *iter;
4900 int type = skb_shinfo(head_skb)->gso_type;
4901 unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
4902
4903 /* Update type to add partial and then remove dodgy if set */
4904 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
4905 type &= ~SKB_GSO_DODGY;
4906
4907 /* Update GSO info and prepare to start updating headers on
4908 * our way back down the stack of protocols.
4909 */
4910 for (iter = segs; iter; iter = iter->next) {
4911 skb_shinfo(iter)->gso_size = gso_size;
4912 skb_shinfo(iter)->gso_segs = partial_segs;
4913 skb_shinfo(iter)->gso_type = type;
4914 SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
4915 }
4916
4917 if (tail->len - doffset <= gso_size)
4918 skb_shinfo(tail)->gso_size = 0;
4919 else if (tail != segs)
4920 skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
4921 }
4922
4923 /* Following permits correct backpressure, for protocols
4924 * using skb_set_owner_w().
4925 * Idea is to tranfert ownership from head_skb to last segment.
4926 */
4927 if (head_skb->destructor == sock_wfree) {
4928 swap(tail->truesize, head_skb->truesize);
4929 swap(tail->destructor, head_skb->destructor);
4930 swap(tail->sk, head_skb->sk);
4931 }
4932 return segs;
4933
4934 err:
4935 kfree_skb_list(segs);
4936 return ERR_PTR(err);
4937 }
4938 EXPORT_SYMBOL_GPL(skb_segment);
4939
4940 #ifdef CONFIG_SKB_EXTENSIONS
4941 #define SKB_EXT_ALIGN_VALUE 8
4942 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
4943
4944 static const u8 skb_ext_type_len[] = {
4945 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4946 [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
4947 #endif
4948 #ifdef CONFIG_XFRM
4949 [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
4950 #endif
4951 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4952 [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
4953 #endif
4954 #if IS_ENABLED(CONFIG_MPTCP)
4955 [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
4956 #endif
4957 #if IS_ENABLED(CONFIG_MCTP_FLOWS)
4958 [SKB_EXT_MCTP] = SKB_EXT_CHUNKSIZEOF(struct mctp_flow),
4959 #endif
4960 };
4961
skb_ext_total_length(void)4962 static __always_inline unsigned int skb_ext_total_length(void)
4963 {
4964 unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext);
4965 int i;
4966
4967 for (i = 0; i < ARRAY_SIZE(skb_ext_type_len); i++)
4968 l += skb_ext_type_len[i];
4969
4970 return l;
4971 }
4972
skb_extensions_init(void)4973 static void skb_extensions_init(void)
4974 {
4975 BUILD_BUG_ON(SKB_EXT_NUM >= 8);
4976 #if !IS_ENABLED(CONFIG_KCOV_INSTRUMENT_ALL)
4977 BUILD_BUG_ON(skb_ext_total_length() > 255);
4978 #endif
4979
4980 skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
4981 SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
4982 0,
4983 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4984 NULL);
4985 }
4986 #else
skb_extensions_init(void)4987 static void skb_extensions_init(void) {}
4988 #endif
4989
4990 /* The SKB kmem_cache slab is critical for network performance. Never
4991 * merge/alias the slab with similar sized objects. This avoids fragmentation
4992 * that hurts performance of kmem_cache_{alloc,free}_bulk APIs.
4993 */
4994 #ifndef CONFIG_SLUB_TINY
4995 #define FLAG_SKB_NO_MERGE SLAB_NO_MERGE
4996 #else /* CONFIG_SLUB_TINY - simple loop in kmem_cache_alloc_bulk */
4997 #define FLAG_SKB_NO_MERGE 0
4998 #endif
4999
skb_init(void)5000 void __init skb_init(void)
5001 {
5002 net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
5003 sizeof(struct sk_buff),
5004 0,
5005 SLAB_HWCACHE_ALIGN|SLAB_PANIC|
5006 FLAG_SKB_NO_MERGE,
5007 offsetof(struct sk_buff, cb),
5008 sizeof_field(struct sk_buff, cb),
5009 NULL);
5010 net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
5011 sizeof(struct sk_buff_fclones),
5012 0,
5013 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
5014 NULL);
5015 /* usercopy should only access first SKB_SMALL_HEAD_HEADROOM bytes.
5016 * struct skb_shared_info is located at the end of skb->head,
5017 * and should not be copied to/from user.
5018 */
5019 net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
5020 SKB_SMALL_HEAD_CACHE_SIZE,
5021 0,
5022 SLAB_HWCACHE_ALIGN | SLAB_PANIC,
5023 0,
5024 SKB_SMALL_HEAD_HEADROOM,
5025 NULL);
5026 skb_extensions_init();
5027 }
5028
5029 static int
__skb_to_sgvec(struct sk_buff * skb,struct scatterlist * sg,int offset,int len,unsigned int recursion_level)5030 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
5031 unsigned int recursion_level)
5032 {
5033 int start = skb_headlen(skb);
5034 int i, copy = start - offset;
5035 struct sk_buff *frag_iter;
5036 int elt = 0;
5037
5038 if (unlikely(recursion_level >= 24))
5039 return -EMSGSIZE;
5040
5041 if (copy > 0) {
5042 if (copy > len)
5043 copy = len;
5044 sg_set_buf(sg, skb->data + offset, copy);
5045 elt++;
5046 if ((len -= copy) == 0)
5047 return elt;
5048 offset += copy;
5049 }
5050
5051 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
5052 int end;
5053
5054 WARN_ON(start > offset + len);
5055
5056 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
5057 if ((copy = end - offset) > 0) {
5058 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5059 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
5060 return -EMSGSIZE;
5061
5062 if (copy > len)
5063 copy = len;
5064 sg_set_page(&sg[elt], skb_frag_page(frag), copy,
5065 skb_frag_off(frag) + offset - start);
5066 elt++;
5067 if (!(len -= copy))
5068 return elt;
5069 offset += copy;
5070 }
5071 start = end;
5072 }
5073
5074 skb_walk_frags(skb, frag_iter) {
5075 int end, ret;
5076
5077 WARN_ON(start > offset + len);
5078
5079 end = start + frag_iter->len;
5080 if ((copy = end - offset) > 0) {
5081 if (unlikely(elt && sg_is_last(&sg[elt - 1])))
5082 return -EMSGSIZE;
5083
5084 if (copy > len)
5085 copy = len;
5086 ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
5087 copy, recursion_level + 1);
5088 if (unlikely(ret < 0))
5089 return ret;
5090 elt += ret;
5091 if ((len -= copy) == 0)
5092 return elt;
5093 offset += copy;
5094 }
5095 start = end;
5096 }
5097 BUG_ON(len);
5098 return elt;
5099 }
5100
5101 /**
5102 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
5103 * @skb: Socket buffer containing the buffers to be mapped
5104 * @sg: The scatter-gather list to map into
5105 * @offset: The offset into the buffer's contents to start mapping
5106 * @len: Length of buffer space to be mapped
5107 *
5108 * Fill the specified scatter-gather list with mappings/pointers into a
5109 * region of the buffer space attached to a socket buffer. Returns either
5110 * the number of scatterlist items used, or -EMSGSIZE if the contents
5111 * could not fit.
5112 */
skb_to_sgvec(struct sk_buff * skb,struct scatterlist * sg,int offset,int len)5113 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
5114 {
5115 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
5116
5117 if (nsg <= 0)
5118 return nsg;
5119
5120 sg_mark_end(&sg[nsg - 1]);
5121
5122 return nsg;
5123 }
5124 EXPORT_SYMBOL_GPL(skb_to_sgvec);
5125
5126 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
5127 * sglist without mark the sg which contain last skb data as the end.
5128 * So the caller can mannipulate sg list as will when padding new data after
5129 * the first call without calling sg_unmark_end to expend sg list.
5130 *
5131 * Scenario to use skb_to_sgvec_nomark:
5132 * 1. sg_init_table
5133 * 2. skb_to_sgvec_nomark(payload1)
5134 * 3. skb_to_sgvec_nomark(payload2)
5135 *
5136 * This is equivalent to:
5137 * 1. sg_init_table
5138 * 2. skb_to_sgvec(payload1)
5139 * 3. sg_unmark_end
5140 * 4. skb_to_sgvec(payload2)
5141 *
5142 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
5143 * is more preferable.
5144 */
skb_to_sgvec_nomark(struct sk_buff * skb,struct scatterlist * sg,int offset,int len)5145 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
5146 int offset, int len)
5147 {
5148 return __skb_to_sgvec(skb, sg, offset, len, 0);
5149 }
5150 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
5151
5152
5153
5154 /**
5155 * skb_cow_data - Check that a socket buffer's data buffers are writable
5156 * @skb: The socket buffer to check.
5157 * @tailbits: Amount of trailing space to be added
5158 * @trailer: Returned pointer to the skb where the @tailbits space begins
5159 *
5160 * Make sure that the data buffers attached to a socket buffer are
5161 * writable. If they are not, private copies are made of the data buffers
5162 * and the socket buffer is set to use these instead.
5163 *
5164 * If @tailbits is given, make sure that there is space to write @tailbits
5165 * bytes of data beyond current end of socket buffer. @trailer will be
5166 * set to point to the skb in which this space begins.
5167 *
5168 * The number of scatterlist elements required to completely map the
5169 * COW'd and extended socket buffer will be returned.
5170 */
skb_cow_data(struct sk_buff * skb,int tailbits,struct sk_buff ** trailer)5171 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
5172 {
5173 int copyflag;
5174 int elt;
5175 struct sk_buff *skb1, **skb_p;
5176
5177 /* If skb is cloned or its head is paged, reallocate
5178 * head pulling out all the pages (pages are considered not writable
5179 * at the moment even if they are anonymous).
5180 */
5181 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
5182 !__pskb_pull_tail(skb, __skb_pagelen(skb)))
5183 return -ENOMEM;
5184
5185 /* Easy case. Most of packets will go this way. */
5186 if (!skb_has_frag_list(skb)) {
5187 /* A little of trouble, not enough of space for trailer.
5188 * This should not happen, when stack is tuned to generate
5189 * good frames. OK, on miss we reallocate and reserve even more
5190 * space, 128 bytes is fair. */
5191
5192 if (skb_tailroom(skb) < tailbits &&
5193 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
5194 return -ENOMEM;
5195
5196 /* Voila! */
5197 *trailer = skb;
5198 return 1;
5199 }
5200
5201 /* Misery. We are in troubles, going to mincer fragments... */
5202
5203 elt = 1;
5204 skb_p = &skb_shinfo(skb)->frag_list;
5205 copyflag = 0;
5206
5207 while ((skb1 = *skb_p) != NULL) {
5208 int ntail = 0;
5209
5210 /* The fragment is partially pulled by someone,
5211 * this can happen on input. Copy it and everything
5212 * after it. */
5213
5214 if (skb_shared(skb1))
5215 copyflag = 1;
5216
5217 /* If the skb is the last, worry about trailer. */
5218
5219 if (skb1->next == NULL && tailbits) {
5220 if (skb_shinfo(skb1)->nr_frags ||
5221 skb_has_frag_list(skb1) ||
5222 skb_tailroom(skb1) < tailbits)
5223 ntail = tailbits + 128;
5224 }
5225
5226 if (copyflag ||
5227 skb_cloned(skb1) ||
5228 ntail ||
5229 skb_shinfo(skb1)->nr_frags ||
5230 skb_has_frag_list(skb1)) {
5231 struct sk_buff *skb2;
5232
5233 /* Fuck, we are miserable poor guys... */
5234 if (ntail == 0)
5235 skb2 = skb_copy(skb1, GFP_ATOMIC);
5236 else
5237 skb2 = skb_copy_expand(skb1,
5238 skb_headroom(skb1),
5239 ntail,
5240 GFP_ATOMIC);
5241 if (unlikely(skb2 == NULL))
5242 return -ENOMEM;
5243
5244 if (skb1->sk)
5245 skb_set_owner_w(skb2, skb1->sk);
5246
5247 /* Looking around. Are we still alive?
5248 * OK, link new skb, drop old one */
5249
5250 skb2->next = skb1->next;
5251 *skb_p = skb2;
5252 kfree_skb(skb1);
5253 skb1 = skb2;
5254 }
5255 elt++;
5256 *trailer = skb1;
5257 skb_p = &skb1->next;
5258 }
5259
5260 return elt;
5261 }
5262 EXPORT_SYMBOL_GPL(skb_cow_data);
5263
sock_rmem_free(struct sk_buff * skb)5264 static void sock_rmem_free(struct sk_buff *skb)
5265 {
5266 struct sock *sk = skb->sk;
5267
5268 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
5269 }
5270
skb_set_err_queue(struct sk_buff * skb)5271 static void skb_set_err_queue(struct sk_buff *skb)
5272 {
5273 /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
5274 * So, it is safe to (mis)use it to mark skbs on the error queue.
5275 */
5276 skb->pkt_type = PACKET_OUTGOING;
5277 BUILD_BUG_ON(PACKET_OUTGOING == 0);
5278 }
5279
5280 /*
5281 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
5282 */
sock_queue_err_skb(struct sock * sk,struct sk_buff * skb)5283 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
5284 {
5285 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
5286 (unsigned int)READ_ONCE(sk->sk_rcvbuf))
5287 return -ENOMEM;
5288
5289 skb_orphan(skb);
5290 skb->sk = sk;
5291 skb->destructor = sock_rmem_free;
5292 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
5293 skb_set_err_queue(skb);
5294
5295 /* before exiting rcu section, make sure dst is refcounted */
5296 skb_dst_force(skb);
5297
5298 skb_queue_tail(&sk->sk_error_queue, skb);
5299 if (!sock_flag(sk, SOCK_DEAD))
5300 sk_error_report(sk);
5301 return 0;
5302 }
5303 EXPORT_SYMBOL(sock_queue_err_skb);
5304
is_icmp_err_skb(const struct sk_buff * skb)5305 static bool is_icmp_err_skb(const struct sk_buff *skb)
5306 {
5307 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
5308 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
5309 }
5310
sock_dequeue_err_skb(struct sock * sk)5311 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
5312 {
5313 struct sk_buff_head *q = &sk->sk_error_queue;
5314 struct sk_buff *skb, *skb_next = NULL;
5315 bool icmp_next = false;
5316 unsigned long flags;
5317
5318 if (skb_queue_empty_lockless(q))
5319 return NULL;
5320
5321 spin_lock_irqsave(&q->lock, flags);
5322 skb = __skb_dequeue(q);
5323 if (skb && (skb_next = skb_peek(q))) {
5324 icmp_next = is_icmp_err_skb(skb_next);
5325 if (icmp_next)
5326 sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_errno;
5327 }
5328 spin_unlock_irqrestore(&q->lock, flags);
5329
5330 if (is_icmp_err_skb(skb) && !icmp_next)
5331 sk->sk_err = 0;
5332
5333 if (skb_next)
5334 sk_error_report(sk);
5335
5336 return skb;
5337 }
5338 EXPORT_SYMBOL(sock_dequeue_err_skb);
5339
5340 /**
5341 * skb_clone_sk - create clone of skb, and take reference to socket
5342 * @skb: the skb to clone
5343 *
5344 * This function creates a clone of a buffer that holds a reference on
5345 * sk_refcnt. Buffers created via this function are meant to be
5346 * returned using sock_queue_err_skb, or free via kfree_skb.
5347 *
5348 * When passing buffers allocated with this function to sock_queue_err_skb
5349 * it is necessary to wrap the call with sock_hold/sock_put in order to
5350 * prevent the socket from being released prior to being enqueued on
5351 * the sk_error_queue.
5352 */
skb_clone_sk(struct sk_buff * skb)5353 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
5354 {
5355 struct sock *sk = skb->sk;
5356 struct sk_buff *clone;
5357
5358 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
5359 return NULL;
5360
5361 clone = skb_clone(skb, GFP_ATOMIC);
5362 if (!clone) {
5363 sock_put(sk);
5364 return NULL;
5365 }
5366
5367 clone->sk = sk;
5368 clone->destructor = sock_efree;
5369
5370 return clone;
5371 }
5372 EXPORT_SYMBOL(skb_clone_sk);
5373
__skb_complete_tx_timestamp(struct sk_buff * skb,struct sock * sk,int tstype,bool opt_stats)5374 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
5375 struct sock *sk,
5376 int tstype,
5377 bool opt_stats)
5378 {
5379 struct sock_exterr_skb *serr;
5380 int err;
5381
5382 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
5383
5384 serr = SKB_EXT_ERR(skb);
5385 memset(serr, 0, sizeof(*serr));
5386 serr->ee.ee_errno = ENOMSG;
5387 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
5388 serr->ee.ee_info = tstype;
5389 serr->opt_stats = opt_stats;
5390 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
5391 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
5392 serr->ee.ee_data = skb_shinfo(skb)->tskey;
5393 if (sk_is_tcp(sk))
5394 serr->ee.ee_data -= atomic_read(&sk->sk_tskey);
5395 }
5396
5397 err = sock_queue_err_skb(sk, skb);
5398
5399 if (err)
5400 kfree_skb(skb);
5401 }
5402
skb_may_tx_timestamp(struct sock * sk,bool tsonly)5403 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
5404 {
5405 bool ret;
5406
5407 if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly))
5408 return true;
5409
5410 read_lock_bh(&sk->sk_callback_lock);
5411 ret = sk->sk_socket && sk->sk_socket->file &&
5412 file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
5413 read_unlock_bh(&sk->sk_callback_lock);
5414 return ret;
5415 }
5416
skb_complete_tx_timestamp(struct sk_buff * skb,struct skb_shared_hwtstamps * hwtstamps)5417 void skb_complete_tx_timestamp(struct sk_buff *skb,
5418 struct skb_shared_hwtstamps *hwtstamps)
5419 {
5420 struct sock *sk = skb->sk;
5421
5422 if (!skb_may_tx_timestamp(sk, false))
5423 goto err;
5424
5425 /* Take a reference to prevent skb_orphan() from freeing the socket,
5426 * but only if the socket refcount is not zero.
5427 */
5428 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5429 *skb_hwtstamps(skb) = *hwtstamps;
5430 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
5431 sock_put(sk);
5432 return;
5433 }
5434
5435 err:
5436 kfree_skb(skb);
5437 }
5438 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
5439
__skb_tstamp_tx(struct sk_buff * orig_skb,const struct sk_buff * ack_skb,struct skb_shared_hwtstamps * hwtstamps,struct sock * sk,int tstype)5440 void __skb_tstamp_tx(struct sk_buff *orig_skb,
5441 const struct sk_buff *ack_skb,
5442 struct skb_shared_hwtstamps *hwtstamps,
5443 struct sock *sk, int tstype)
5444 {
5445 struct sk_buff *skb;
5446 bool tsonly, opt_stats = false;
5447 u32 tsflags;
5448
5449 if (!sk)
5450 return;
5451
5452 tsflags = READ_ONCE(sk->sk_tsflags);
5453 if (!hwtstamps && !(tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
5454 skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
5455 return;
5456
5457 tsonly = tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
5458 if (!skb_may_tx_timestamp(sk, tsonly))
5459 return;
5460
5461 if (tsonly) {
5462 #ifdef CONFIG_INET
5463 if ((tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
5464 sk_is_tcp(sk)) {
5465 skb = tcp_get_timestamping_opt_stats(sk, orig_skb,
5466 ack_skb);
5467 opt_stats = true;
5468 } else
5469 #endif
5470 skb = alloc_skb(0, GFP_ATOMIC);
5471 } else {
5472 skb = skb_clone(orig_skb, GFP_ATOMIC);
5473
5474 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
5475 kfree_skb(skb);
5476 return;
5477 }
5478 }
5479 if (!skb)
5480 return;
5481
5482 if (tsonly) {
5483 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
5484 SKBTX_ANY_TSTAMP;
5485 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
5486 }
5487
5488 if (hwtstamps)
5489 *skb_hwtstamps(skb) = *hwtstamps;
5490 else
5491 __net_timestamp(skb);
5492
5493 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
5494 }
5495 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
5496
skb_tstamp_tx(struct sk_buff * orig_skb,struct skb_shared_hwtstamps * hwtstamps)5497 void skb_tstamp_tx(struct sk_buff *orig_skb,
5498 struct skb_shared_hwtstamps *hwtstamps)
5499 {
5500 return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk,
5501 SCM_TSTAMP_SND);
5502 }
5503 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
5504
5505 #ifdef CONFIG_WIRELESS
skb_complete_wifi_ack(struct sk_buff * skb,bool acked)5506 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
5507 {
5508 struct sock *sk = skb->sk;
5509 struct sock_exterr_skb *serr;
5510 int err = 1;
5511
5512 skb->wifi_acked_valid = 1;
5513 skb->wifi_acked = acked;
5514
5515 serr = SKB_EXT_ERR(skb);
5516 memset(serr, 0, sizeof(*serr));
5517 serr->ee.ee_errno = ENOMSG;
5518 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
5519
5520 /* Take a reference to prevent skb_orphan() from freeing the socket,
5521 * but only if the socket refcount is not zero.
5522 */
5523 if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
5524 err = sock_queue_err_skb(sk, skb);
5525 sock_put(sk);
5526 }
5527 if (err)
5528 kfree_skb(skb);
5529 }
5530 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
5531 #endif /* CONFIG_WIRELESS */
5532
5533 /**
5534 * skb_partial_csum_set - set up and verify partial csum values for packet
5535 * @skb: the skb to set
5536 * @start: the number of bytes after skb->data to start checksumming.
5537 * @off: the offset from start to place the checksum.
5538 *
5539 * For untrusted partially-checksummed packets, we need to make sure the values
5540 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
5541 *
5542 * This function checks and sets those values and skb->ip_summed: if this
5543 * returns false you should drop the packet.
5544 */
skb_partial_csum_set(struct sk_buff * skb,u16 start,u16 off)5545 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
5546 {
5547 u32 csum_end = (u32)start + (u32)off + sizeof(__sum16);
5548 u32 csum_start = skb_headroom(skb) + (u32)start;
5549
5550 if (unlikely(csum_start >= U16_MAX || csum_end > skb_headlen(skb))) {
5551 net_warn_ratelimited("bad partial csum: csum=%u/%u headroom=%u headlen=%u\n",
5552 start, off, skb_headroom(skb), skb_headlen(skb));
5553 return false;
5554 }
5555 skb->ip_summed = CHECKSUM_PARTIAL;
5556 skb->csum_start = csum_start;
5557 skb->csum_offset = off;
5558 skb->transport_header = csum_start;
5559 return true;
5560 }
5561 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
5562
skb_maybe_pull_tail(struct sk_buff * skb,unsigned int len,unsigned int max)5563 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
5564 unsigned int max)
5565 {
5566 if (skb_headlen(skb) >= len)
5567 return 0;
5568
5569 /* If we need to pullup then pullup to the max, so we
5570 * won't need to do it again.
5571 */
5572 if (max > skb->len)
5573 max = skb->len;
5574
5575 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
5576 return -ENOMEM;
5577
5578 if (skb_headlen(skb) < len)
5579 return -EPROTO;
5580
5581 return 0;
5582 }
5583
5584 #define MAX_TCP_HDR_LEN (15 * 4)
5585
skb_checksum_setup_ip(struct sk_buff * skb,typeof (IPPROTO_IP)proto,unsigned int off)5586 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
5587 typeof(IPPROTO_IP) proto,
5588 unsigned int off)
5589 {
5590 int err;
5591
5592 switch (proto) {
5593 case IPPROTO_TCP:
5594 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
5595 off + MAX_TCP_HDR_LEN);
5596 if (!err && !skb_partial_csum_set(skb, off,
5597 offsetof(struct tcphdr,
5598 check)))
5599 err = -EPROTO;
5600 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
5601
5602 case IPPROTO_UDP:
5603 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
5604 off + sizeof(struct udphdr));
5605 if (!err && !skb_partial_csum_set(skb, off,
5606 offsetof(struct udphdr,
5607 check)))
5608 err = -EPROTO;
5609 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
5610 }
5611
5612 return ERR_PTR(-EPROTO);
5613 }
5614
5615 /* This value should be large enough to cover a tagged ethernet header plus
5616 * maximally sized IP and TCP or UDP headers.
5617 */
5618 #define MAX_IP_HDR_LEN 128
5619
skb_checksum_setup_ipv4(struct sk_buff * skb,bool recalculate)5620 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
5621 {
5622 unsigned int off;
5623 bool fragment;
5624 __sum16 *csum;
5625 int err;
5626
5627 fragment = false;
5628
5629 err = skb_maybe_pull_tail(skb,
5630 sizeof(struct iphdr),
5631 MAX_IP_HDR_LEN);
5632 if (err < 0)
5633 goto out;
5634
5635 if (ip_is_fragment(ip_hdr(skb)))
5636 fragment = true;
5637
5638 off = ip_hdrlen(skb);
5639
5640 err = -EPROTO;
5641
5642 if (fragment)
5643 goto out;
5644
5645 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
5646 if (IS_ERR(csum))
5647 return PTR_ERR(csum);
5648
5649 if (recalculate)
5650 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
5651 ip_hdr(skb)->daddr,
5652 skb->len - off,
5653 ip_hdr(skb)->protocol, 0);
5654 err = 0;
5655
5656 out:
5657 return err;
5658 }
5659
5660 /* This value should be large enough to cover a tagged ethernet header plus
5661 * an IPv6 header, all options, and a maximal TCP or UDP header.
5662 */
5663 #define MAX_IPV6_HDR_LEN 256
5664
5665 #define OPT_HDR(type, skb, off) \
5666 (type *)(skb_network_header(skb) + (off))
5667
skb_checksum_setup_ipv6(struct sk_buff * skb,bool recalculate)5668 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
5669 {
5670 int err;
5671 u8 nexthdr;
5672 unsigned int off;
5673 unsigned int len;
5674 bool fragment;
5675 bool done;
5676 __sum16 *csum;
5677
5678 fragment = false;
5679 done = false;
5680
5681 off = sizeof(struct ipv6hdr);
5682
5683 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
5684 if (err < 0)
5685 goto out;
5686
5687 nexthdr = ipv6_hdr(skb)->nexthdr;
5688
5689 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
5690 while (off <= len && !done) {
5691 switch (nexthdr) {
5692 case IPPROTO_DSTOPTS:
5693 case IPPROTO_HOPOPTS:
5694 case IPPROTO_ROUTING: {
5695 struct ipv6_opt_hdr *hp;
5696
5697 err = skb_maybe_pull_tail(skb,
5698 off +
5699 sizeof(struct ipv6_opt_hdr),
5700 MAX_IPV6_HDR_LEN);
5701 if (err < 0)
5702 goto out;
5703
5704 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
5705 nexthdr = hp->nexthdr;
5706 off += ipv6_optlen(hp);
5707 break;
5708 }
5709 case IPPROTO_AH: {
5710 struct ip_auth_hdr *hp;
5711
5712 err = skb_maybe_pull_tail(skb,
5713 off +
5714 sizeof(struct ip_auth_hdr),
5715 MAX_IPV6_HDR_LEN);
5716 if (err < 0)
5717 goto out;
5718
5719 hp = OPT_HDR(struct ip_auth_hdr, skb, off);
5720 nexthdr = hp->nexthdr;
5721 off += ipv6_authlen(hp);
5722 break;
5723 }
5724 case IPPROTO_FRAGMENT: {
5725 struct frag_hdr *hp;
5726
5727 err = skb_maybe_pull_tail(skb,
5728 off +
5729 sizeof(struct frag_hdr),
5730 MAX_IPV6_HDR_LEN);
5731 if (err < 0)
5732 goto out;
5733
5734 hp = OPT_HDR(struct frag_hdr, skb, off);
5735
5736 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
5737 fragment = true;
5738
5739 nexthdr = hp->nexthdr;
5740 off += sizeof(struct frag_hdr);
5741 break;
5742 }
5743 default:
5744 done = true;
5745 break;
5746 }
5747 }
5748
5749 err = -EPROTO;
5750
5751 if (!done || fragment)
5752 goto out;
5753
5754 csum = skb_checksum_setup_ip(skb, nexthdr, off);
5755 if (IS_ERR(csum))
5756 return PTR_ERR(csum);
5757
5758 if (recalculate)
5759 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
5760 &ipv6_hdr(skb)->daddr,
5761 skb->len - off, nexthdr, 0);
5762 err = 0;
5763
5764 out:
5765 return err;
5766 }
5767
5768 /**
5769 * skb_checksum_setup - set up partial checksum offset
5770 * @skb: the skb to set up
5771 * @recalculate: if true the pseudo-header checksum will be recalculated
5772 */
skb_checksum_setup(struct sk_buff * skb,bool recalculate)5773 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
5774 {
5775 int err;
5776
5777 switch (skb->protocol) {
5778 case htons(ETH_P_IP):
5779 err = skb_checksum_setup_ipv4(skb, recalculate);
5780 break;
5781
5782 case htons(ETH_P_IPV6):
5783 err = skb_checksum_setup_ipv6(skb, recalculate);
5784 break;
5785
5786 default:
5787 err = -EPROTO;
5788 break;
5789 }
5790
5791 return err;
5792 }
5793 EXPORT_SYMBOL(skb_checksum_setup);
5794
5795 /**
5796 * skb_checksum_maybe_trim - maybe trims the given skb
5797 * @skb: the skb to check
5798 * @transport_len: the data length beyond the network header
5799 *
5800 * Checks whether the given skb has data beyond the given transport length.
5801 * If so, returns a cloned skb trimmed to this transport length.
5802 * Otherwise returns the provided skb. Returns NULL in error cases
5803 * (e.g. transport_len exceeds skb length or out-of-memory).
5804 *
5805 * Caller needs to set the skb transport header and free any returned skb if it
5806 * differs from the provided skb.
5807 */
skb_checksum_maybe_trim(struct sk_buff * skb,unsigned int transport_len)5808 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
5809 unsigned int transport_len)
5810 {
5811 struct sk_buff *skb_chk;
5812 unsigned int len = skb_transport_offset(skb) + transport_len;
5813 int ret;
5814
5815 if (skb->len < len)
5816 return NULL;
5817 else if (skb->len == len)
5818 return skb;
5819
5820 skb_chk = skb_clone(skb, GFP_ATOMIC);
5821 if (!skb_chk)
5822 return NULL;
5823
5824 ret = pskb_trim_rcsum(skb_chk, len);
5825 if (ret) {
5826 kfree_skb(skb_chk);
5827 return NULL;
5828 }
5829
5830 return skb_chk;
5831 }
5832
5833 /**
5834 * skb_checksum_trimmed - validate checksum of an skb
5835 * @skb: the skb to check
5836 * @transport_len: the data length beyond the network header
5837 * @skb_chkf: checksum function to use
5838 *
5839 * Applies the given checksum function skb_chkf to the provided skb.
5840 * Returns a checked and maybe trimmed skb. Returns NULL on error.
5841 *
5842 * If the skb has data beyond the given transport length, then a
5843 * trimmed & cloned skb is checked and returned.
5844 *
5845 * Caller needs to set the skb transport header and free any returned skb if it
5846 * differs from the provided skb.
5847 */
skb_checksum_trimmed(struct sk_buff * skb,unsigned int transport_len,__sum16 (* skb_chkf)(struct sk_buff * skb))5848 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5849 unsigned int transport_len,
5850 __sum16(*skb_chkf)(struct sk_buff *skb))
5851 {
5852 struct sk_buff *skb_chk;
5853 unsigned int offset = skb_transport_offset(skb);
5854 __sum16 ret;
5855
5856 skb_chk = skb_checksum_maybe_trim(skb, transport_len);
5857 if (!skb_chk)
5858 goto err;
5859
5860 if (!pskb_may_pull(skb_chk, offset))
5861 goto err;
5862
5863 skb_pull_rcsum(skb_chk, offset);
5864 ret = skb_chkf(skb_chk);
5865 skb_push_rcsum(skb_chk, offset);
5866
5867 if (ret)
5868 goto err;
5869
5870 return skb_chk;
5871
5872 err:
5873 if (skb_chk && skb_chk != skb)
5874 kfree_skb(skb_chk);
5875
5876 return NULL;
5877
5878 }
5879 EXPORT_SYMBOL(skb_checksum_trimmed);
5880
__skb_warn_lro_forwarding(const struct sk_buff * skb)5881 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
5882 {
5883 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
5884 skb->dev->name);
5885 }
5886 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
5887
kfree_skb_partial(struct sk_buff * skb,bool head_stolen)5888 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
5889 {
5890 if (head_stolen) {
5891 skb_release_head_state(skb);
5892 kmem_cache_free(net_hotdata.skbuff_cache, skb);
5893 } else {
5894 __kfree_skb(skb);
5895 }
5896 }
5897 EXPORT_SYMBOL(kfree_skb_partial);
5898
5899 /**
5900 * skb_try_coalesce - try to merge skb to prior one
5901 * @to: prior buffer
5902 * @from: buffer to add
5903 * @fragstolen: pointer to boolean
5904 * @delta_truesize: how much more was allocated than was requested
5905 */
skb_try_coalesce(struct sk_buff * to,struct sk_buff * from,bool * fragstolen,int * delta_truesize)5906 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
5907 bool *fragstolen, int *delta_truesize)
5908 {
5909 struct skb_shared_info *to_shinfo, *from_shinfo;
5910 int i, delta, len = from->len;
5911
5912 *fragstolen = false;
5913
5914 if (skb_cloned(to))
5915 return false;
5916
5917 /* In general, avoid mixing page_pool and non-page_pool allocated
5918 * pages within the same SKB. In theory we could take full
5919 * references if @from is cloned and !@to->pp_recycle but its
5920 * tricky (due to potential race with the clone disappearing) and
5921 * rare, so not worth dealing with.
5922 */
5923 if (to->pp_recycle != from->pp_recycle)
5924 return false;
5925
5926 if (len <= skb_tailroom(to)) {
5927 if (len)
5928 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
5929 *delta_truesize = 0;
5930 return true;
5931 }
5932
5933 to_shinfo = skb_shinfo(to);
5934 from_shinfo = skb_shinfo(from);
5935 if (to_shinfo->frag_list || from_shinfo->frag_list)
5936 return false;
5937 if (skb_zcopy(to) || skb_zcopy(from))
5938 return false;
5939
5940 if (skb_headlen(from) != 0) {
5941 struct page *page;
5942 unsigned int offset;
5943
5944 if (to_shinfo->nr_frags +
5945 from_shinfo->nr_frags >= MAX_SKB_FRAGS)
5946 return false;
5947
5948 if (skb_head_is_locked(from))
5949 return false;
5950
5951 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
5952
5953 page = virt_to_head_page(from->head);
5954 offset = from->data - (unsigned char *)page_address(page);
5955
5956 skb_fill_page_desc(to, to_shinfo->nr_frags,
5957 page, offset, skb_headlen(from));
5958 *fragstolen = true;
5959 } else {
5960 if (to_shinfo->nr_frags +
5961 from_shinfo->nr_frags > MAX_SKB_FRAGS)
5962 return false;
5963
5964 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
5965 }
5966
5967 WARN_ON_ONCE(delta < len);
5968
5969 memcpy(to_shinfo->frags + to_shinfo->nr_frags,
5970 from_shinfo->frags,
5971 from_shinfo->nr_frags * sizeof(skb_frag_t));
5972 to_shinfo->nr_frags += from_shinfo->nr_frags;
5973
5974 if (!skb_cloned(from))
5975 from_shinfo->nr_frags = 0;
5976
5977 /* if the skb is not cloned this does nothing
5978 * since we set nr_frags to 0.
5979 */
5980 if (skb_pp_frag_ref(from)) {
5981 for (i = 0; i < from_shinfo->nr_frags; i++)
5982 __skb_frag_ref(&from_shinfo->frags[i]);
5983 }
5984
5985 to->truesize += delta;
5986 to->len += len;
5987 to->data_len += len;
5988
5989 *delta_truesize = delta;
5990 return true;
5991 }
5992 EXPORT_SYMBOL(skb_try_coalesce);
5993
5994 /**
5995 * skb_scrub_packet - scrub an skb
5996 *
5997 * @skb: buffer to clean
5998 * @xnet: packet is crossing netns
5999 *
6000 * skb_scrub_packet can be used after encapsulating or decapsulting a packet
6001 * into/from a tunnel. Some information have to be cleared during these
6002 * operations.
6003 * skb_scrub_packet can also be used to clean a skb before injecting it in
6004 * another namespace (@xnet == true). We have to clear all information in the
6005 * skb that could impact namespace isolation.
6006 */
skb_scrub_packet(struct sk_buff * skb,bool xnet)6007 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
6008 {
6009 skb->pkt_type = PACKET_HOST;
6010 skb->skb_iif = 0;
6011 skb->ignore_df = 0;
6012 skb_dst_drop(skb);
6013 skb_ext_reset(skb);
6014 nf_reset_ct(skb);
6015 nf_reset_trace(skb);
6016
6017 #ifdef CONFIG_NET_SWITCHDEV
6018 skb->offload_fwd_mark = 0;
6019 skb->offload_l3_fwd_mark = 0;
6020 #endif
6021
6022 if (!xnet)
6023 return;
6024
6025 ipvs_reset(skb);
6026 skb->mark = 0;
6027 skb_clear_tstamp(skb);
6028 }
6029 EXPORT_SYMBOL_GPL(skb_scrub_packet);
6030
skb_reorder_vlan_header(struct sk_buff * skb)6031 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
6032 {
6033 int mac_len, meta_len;
6034 void *meta;
6035
6036 if (skb_cow(skb, skb_headroom(skb)) < 0) {
6037 kfree_skb(skb);
6038 return NULL;
6039 }
6040
6041 mac_len = skb->data - skb_mac_header(skb);
6042 if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
6043 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
6044 mac_len - VLAN_HLEN - ETH_TLEN);
6045 }
6046
6047 meta_len = skb_metadata_len(skb);
6048 if (meta_len) {
6049 meta = skb_metadata_end(skb) - meta_len;
6050 memmove(meta + VLAN_HLEN, meta, meta_len);
6051 }
6052
6053 skb->mac_header += VLAN_HLEN;
6054 return skb;
6055 }
6056
skb_vlan_untag(struct sk_buff * skb)6057 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
6058 {
6059 struct vlan_hdr *vhdr;
6060 u16 vlan_tci;
6061
6062 if (unlikely(skb_vlan_tag_present(skb))) {
6063 /* vlan_tci is already set-up so leave this for another time */
6064 return skb;
6065 }
6066
6067 skb = skb_share_check(skb, GFP_ATOMIC);
6068 if (unlikely(!skb))
6069 goto err_free;
6070 /* We may access the two bytes after vlan_hdr in vlan_set_encap_proto(). */
6071 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
6072 goto err_free;
6073
6074 vhdr = (struct vlan_hdr *)skb->data;
6075 vlan_tci = ntohs(vhdr->h_vlan_TCI);
6076 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
6077
6078 skb_pull_rcsum(skb, VLAN_HLEN);
6079 vlan_set_encap_proto(skb, vhdr);
6080
6081 skb = skb_reorder_vlan_header(skb);
6082 if (unlikely(!skb))
6083 goto err_free;
6084
6085 skb_reset_network_header(skb);
6086 if (!skb_transport_header_was_set(skb))
6087 skb_reset_transport_header(skb);
6088 skb_reset_mac_len(skb);
6089
6090 return skb;
6091
6092 err_free:
6093 kfree_skb(skb);
6094 return NULL;
6095 }
6096 EXPORT_SYMBOL(skb_vlan_untag);
6097
skb_ensure_writable(struct sk_buff * skb,unsigned int write_len)6098 int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len)
6099 {
6100 if (!pskb_may_pull(skb, write_len))
6101 return -ENOMEM;
6102
6103 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
6104 return 0;
6105
6106 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
6107 }
6108 EXPORT_SYMBOL(skb_ensure_writable);
6109
skb_ensure_writable_head_tail(struct sk_buff * skb,struct net_device * dev)6110 int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev)
6111 {
6112 int needed_headroom = dev->needed_headroom;
6113 int needed_tailroom = dev->needed_tailroom;
6114
6115 /* For tail taggers, we need to pad short frames ourselves, to ensure
6116 * that the tail tag does not fail at its role of being at the end of
6117 * the packet, once the conduit interface pads the frame. Account for
6118 * that pad length here, and pad later.
6119 */
6120 if (unlikely(needed_tailroom && skb->len < ETH_ZLEN))
6121 needed_tailroom += ETH_ZLEN - skb->len;
6122 /* skb_headroom() returns unsigned int... */
6123 needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0);
6124 needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0);
6125
6126 if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb)))
6127 /* No reallocation needed, yay! */
6128 return 0;
6129
6130 return pskb_expand_head(skb, needed_headroom, needed_tailroom,
6131 GFP_ATOMIC);
6132 }
6133 EXPORT_SYMBOL(skb_ensure_writable_head_tail);
6134
6135 /* remove VLAN header from packet and update csum accordingly.
6136 * expects a non skb_vlan_tag_present skb with a vlan tag payload
6137 */
__skb_vlan_pop(struct sk_buff * skb,u16 * vlan_tci)6138 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
6139 {
6140 int offset = skb->data - skb_mac_header(skb);
6141 int err;
6142
6143 if (WARN_ONCE(offset,
6144 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
6145 offset)) {
6146 return -EINVAL;
6147 }
6148
6149 err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
6150 if (unlikely(err))
6151 return err;
6152
6153 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6154
6155 vlan_remove_tag(skb, vlan_tci);
6156
6157 skb->mac_header += VLAN_HLEN;
6158
6159 if (skb_network_offset(skb) < ETH_HLEN)
6160 skb_set_network_header(skb, ETH_HLEN);
6161
6162 skb_reset_mac_len(skb);
6163
6164 return err;
6165 }
6166 EXPORT_SYMBOL(__skb_vlan_pop);
6167
6168 /* Pop a vlan tag either from hwaccel or from payload.
6169 * Expects skb->data at mac header.
6170 */
skb_vlan_pop(struct sk_buff * skb)6171 int skb_vlan_pop(struct sk_buff *skb)
6172 {
6173 u16 vlan_tci;
6174 __be16 vlan_proto;
6175 int err;
6176
6177 if (likely(skb_vlan_tag_present(skb))) {
6178 __vlan_hwaccel_clear_tag(skb);
6179 } else {
6180 if (unlikely(!eth_type_vlan(skb->protocol)))
6181 return 0;
6182
6183 err = __skb_vlan_pop(skb, &vlan_tci);
6184 if (err)
6185 return err;
6186 }
6187 /* move next vlan tag to hw accel tag */
6188 if (likely(!eth_type_vlan(skb->protocol)))
6189 return 0;
6190
6191 vlan_proto = skb->protocol;
6192 err = __skb_vlan_pop(skb, &vlan_tci);
6193 if (unlikely(err))
6194 return err;
6195
6196 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6197 return 0;
6198 }
6199 EXPORT_SYMBOL(skb_vlan_pop);
6200
6201 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
6202 * Expects skb->data at mac header.
6203 */
skb_vlan_push(struct sk_buff * skb,__be16 vlan_proto,u16 vlan_tci)6204 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
6205 {
6206 if (skb_vlan_tag_present(skb)) {
6207 int offset = skb->data - skb_mac_header(skb);
6208 int err;
6209
6210 if (WARN_ONCE(offset,
6211 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
6212 offset)) {
6213 return -EINVAL;
6214 }
6215
6216 err = __vlan_insert_tag(skb, skb->vlan_proto,
6217 skb_vlan_tag_get(skb));
6218 if (err)
6219 return err;
6220
6221 skb->protocol = skb->vlan_proto;
6222 skb->mac_len += VLAN_HLEN;
6223
6224 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
6225 }
6226 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
6227 return 0;
6228 }
6229 EXPORT_SYMBOL(skb_vlan_push);
6230
6231 /**
6232 * skb_eth_pop() - Drop the Ethernet header at the head of a packet
6233 *
6234 * @skb: Socket buffer to modify
6235 *
6236 * Drop the Ethernet header of @skb.
6237 *
6238 * Expects that skb->data points to the mac header and that no VLAN tags are
6239 * present.
6240 *
6241 * Returns 0 on success, -errno otherwise.
6242 */
skb_eth_pop(struct sk_buff * skb)6243 int skb_eth_pop(struct sk_buff *skb)
6244 {
6245 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
6246 skb_network_offset(skb) < ETH_HLEN)
6247 return -EPROTO;
6248
6249 skb_pull_rcsum(skb, ETH_HLEN);
6250 skb_reset_mac_header(skb);
6251 skb_reset_mac_len(skb);
6252
6253 return 0;
6254 }
6255 EXPORT_SYMBOL(skb_eth_pop);
6256
6257 /**
6258 * skb_eth_push() - Add a new Ethernet header at the head of a packet
6259 *
6260 * @skb: Socket buffer to modify
6261 * @dst: Destination MAC address of the new header
6262 * @src: Source MAC address of the new header
6263 *
6264 * Prepend @skb with a new Ethernet header.
6265 *
6266 * Expects that skb->data points to the mac header, which must be empty.
6267 *
6268 * Returns 0 on success, -errno otherwise.
6269 */
skb_eth_push(struct sk_buff * skb,const unsigned char * dst,const unsigned char * src)6270 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
6271 const unsigned char *src)
6272 {
6273 struct ethhdr *eth;
6274 int err;
6275
6276 if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
6277 return -EPROTO;
6278
6279 err = skb_cow_head(skb, sizeof(*eth));
6280 if (err < 0)
6281 return err;
6282
6283 skb_push(skb, sizeof(*eth));
6284 skb_reset_mac_header(skb);
6285 skb_reset_mac_len(skb);
6286
6287 eth = eth_hdr(skb);
6288 ether_addr_copy(eth->h_dest, dst);
6289 ether_addr_copy(eth->h_source, src);
6290 eth->h_proto = skb->protocol;
6291
6292 skb_postpush_rcsum(skb, eth, sizeof(*eth));
6293
6294 return 0;
6295 }
6296 EXPORT_SYMBOL(skb_eth_push);
6297
6298 /* Update the ethertype of hdr and the skb csum value if required. */
skb_mod_eth_type(struct sk_buff * skb,struct ethhdr * hdr,__be16 ethertype)6299 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
6300 __be16 ethertype)
6301 {
6302 if (skb->ip_summed == CHECKSUM_COMPLETE) {
6303 __be16 diff[] = { ~hdr->h_proto, ethertype };
6304
6305 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6306 }
6307
6308 hdr->h_proto = ethertype;
6309 }
6310
6311 /**
6312 * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
6313 * the packet
6314 *
6315 * @skb: buffer
6316 * @mpls_lse: MPLS label stack entry to push
6317 * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
6318 * @mac_len: length of the MAC header
6319 * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
6320 * ethernet
6321 *
6322 * Expects skb->data at mac header.
6323 *
6324 * Returns 0 on success, -errno otherwise.
6325 */
skb_mpls_push(struct sk_buff * skb,__be32 mpls_lse,__be16 mpls_proto,int mac_len,bool ethernet)6326 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
6327 int mac_len, bool ethernet)
6328 {
6329 struct mpls_shim_hdr *lse;
6330 int err;
6331
6332 if (unlikely(!eth_p_mpls(mpls_proto)))
6333 return -EINVAL;
6334
6335 /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
6336 if (skb->encapsulation)
6337 return -EINVAL;
6338
6339 err = skb_cow_head(skb, MPLS_HLEN);
6340 if (unlikely(err))
6341 return err;
6342
6343 if (!skb->inner_protocol) {
6344 skb_set_inner_network_header(skb, skb_network_offset(skb));
6345 skb_set_inner_protocol(skb, skb->protocol);
6346 }
6347
6348 skb_push(skb, MPLS_HLEN);
6349 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
6350 mac_len);
6351 skb_reset_mac_header(skb);
6352 skb_set_network_header(skb, mac_len);
6353 skb_reset_mac_len(skb);
6354
6355 lse = mpls_hdr(skb);
6356 lse->label_stack_entry = mpls_lse;
6357 skb_postpush_rcsum(skb, lse, MPLS_HLEN);
6358
6359 if (ethernet && mac_len >= ETH_HLEN)
6360 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
6361 skb->protocol = mpls_proto;
6362
6363 return 0;
6364 }
6365 EXPORT_SYMBOL_GPL(skb_mpls_push);
6366
6367 /**
6368 * skb_mpls_pop() - pop the outermost MPLS header
6369 *
6370 * @skb: buffer
6371 * @next_proto: ethertype of header after popped MPLS header
6372 * @mac_len: length of the MAC header
6373 * @ethernet: flag to indicate if the packet is ethernet
6374 *
6375 * Expects skb->data at mac header.
6376 *
6377 * Returns 0 on success, -errno otherwise.
6378 */
skb_mpls_pop(struct sk_buff * skb,__be16 next_proto,int mac_len,bool ethernet)6379 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
6380 bool ethernet)
6381 {
6382 int err;
6383
6384 if (unlikely(!eth_p_mpls(skb->protocol)))
6385 return 0;
6386
6387 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
6388 if (unlikely(err))
6389 return err;
6390
6391 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
6392 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
6393 mac_len);
6394
6395 __skb_pull(skb, MPLS_HLEN);
6396 skb_reset_mac_header(skb);
6397 skb_set_network_header(skb, mac_len);
6398
6399 if (ethernet && mac_len >= ETH_HLEN) {
6400 struct ethhdr *hdr;
6401
6402 /* use mpls_hdr() to get ethertype to account for VLANs. */
6403 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
6404 skb_mod_eth_type(skb, hdr, next_proto);
6405 }
6406 skb->protocol = next_proto;
6407
6408 return 0;
6409 }
6410 EXPORT_SYMBOL_GPL(skb_mpls_pop);
6411
6412 /**
6413 * skb_mpls_update_lse() - modify outermost MPLS header and update csum
6414 *
6415 * @skb: buffer
6416 * @mpls_lse: new MPLS label stack entry to update to
6417 *
6418 * Expects skb->data at mac header.
6419 *
6420 * Returns 0 on success, -errno otherwise.
6421 */
skb_mpls_update_lse(struct sk_buff * skb,__be32 mpls_lse)6422 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
6423 {
6424 int err;
6425
6426 if (unlikely(!eth_p_mpls(skb->protocol)))
6427 return -EINVAL;
6428
6429 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
6430 if (unlikely(err))
6431 return err;
6432
6433 if (skb->ip_summed == CHECKSUM_COMPLETE) {
6434 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
6435
6436 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
6437 }
6438
6439 mpls_hdr(skb)->label_stack_entry = mpls_lse;
6440
6441 return 0;
6442 }
6443 EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
6444
6445 /**
6446 * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
6447 *
6448 * @skb: buffer
6449 *
6450 * Expects skb->data at mac header.
6451 *
6452 * Returns 0 on success, -errno otherwise.
6453 */
skb_mpls_dec_ttl(struct sk_buff * skb)6454 int skb_mpls_dec_ttl(struct sk_buff *skb)
6455 {
6456 u32 lse;
6457 u8 ttl;
6458
6459 if (unlikely(!eth_p_mpls(skb->protocol)))
6460 return -EINVAL;
6461
6462 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
6463 return -ENOMEM;
6464
6465 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
6466 ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
6467 if (!--ttl)
6468 return -EINVAL;
6469
6470 lse &= ~MPLS_LS_TTL_MASK;
6471 lse |= ttl << MPLS_LS_TTL_SHIFT;
6472
6473 return skb_mpls_update_lse(skb, cpu_to_be32(lse));
6474 }
6475 EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
6476
6477 /**
6478 * alloc_skb_with_frags - allocate skb with page frags
6479 *
6480 * @header_len: size of linear part
6481 * @data_len: needed length in frags
6482 * @order: max page order desired.
6483 * @errcode: pointer to error code if any
6484 * @gfp_mask: allocation mask
6485 *
6486 * This can be used to allocate a paged skb, given a maximal order for frags.
6487 */
alloc_skb_with_frags(unsigned long header_len,unsigned long data_len,int order,int * errcode,gfp_t gfp_mask)6488 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
6489 unsigned long data_len,
6490 int order,
6491 int *errcode,
6492 gfp_t gfp_mask)
6493 {
6494 unsigned long chunk;
6495 struct sk_buff *skb;
6496 struct page *page;
6497 int nr_frags = 0;
6498
6499 *errcode = -EMSGSIZE;
6500 if (unlikely(data_len > MAX_SKB_FRAGS * (PAGE_SIZE << order)))
6501 return NULL;
6502
6503 *errcode = -ENOBUFS;
6504 skb = alloc_skb(header_len, gfp_mask);
6505 if (!skb)
6506 return NULL;
6507
6508 while (data_len) {
6509 if (nr_frags == MAX_SKB_FRAGS - 1)
6510 goto failure;
6511 while (order && PAGE_ALIGN(data_len) < (PAGE_SIZE << order))
6512 order--;
6513
6514 if (order) {
6515 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
6516 __GFP_COMP |
6517 __GFP_NOWARN,
6518 order);
6519 if (!page) {
6520 order--;
6521 continue;
6522 }
6523 } else {
6524 page = alloc_page(gfp_mask);
6525 if (!page)
6526 goto failure;
6527 }
6528 chunk = min_t(unsigned long, data_len,
6529 PAGE_SIZE << order);
6530 skb_fill_page_desc(skb, nr_frags, page, 0, chunk);
6531 nr_frags++;
6532 skb->truesize += (PAGE_SIZE << order);
6533 data_len -= chunk;
6534 }
6535 return skb;
6536
6537 failure:
6538 kfree_skb(skb);
6539 return NULL;
6540 }
6541 EXPORT_SYMBOL(alloc_skb_with_frags);
6542
6543 /* carve out the first off bytes from skb when off < headlen */
pskb_carve_inside_header(struct sk_buff * skb,const u32 off,const int headlen,gfp_t gfp_mask)6544 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
6545 const int headlen, gfp_t gfp_mask)
6546 {
6547 int i;
6548 unsigned int size = skb_end_offset(skb);
6549 int new_hlen = headlen - off;
6550 u8 *data;
6551
6552 if (skb_pfmemalloc(skb))
6553 gfp_mask |= __GFP_MEMALLOC;
6554
6555 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
6556 if (!data)
6557 return -ENOMEM;
6558 size = SKB_WITH_OVERHEAD(size);
6559
6560 /* Copy real data, and all frags */
6561 skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
6562 skb->len -= off;
6563
6564 memcpy((struct skb_shared_info *)(data + size),
6565 skb_shinfo(skb),
6566 offsetof(struct skb_shared_info,
6567 frags[skb_shinfo(skb)->nr_frags]));
6568 if (skb_cloned(skb)) {
6569 /* drop the old head gracefully */
6570 if (skb_orphan_frags(skb, gfp_mask)) {
6571 skb_kfree_head(data, size);
6572 return -ENOMEM;
6573 }
6574 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
6575 skb_frag_ref(skb, i);
6576 if (skb_has_frag_list(skb))
6577 skb_clone_fraglist(skb);
6578 skb_release_data(skb, SKB_CONSUMED);
6579 } else {
6580 /* we can reuse existing recount- all we did was
6581 * relocate values
6582 */
6583 skb_free_head(skb);
6584 }
6585
6586 skb->head = data;
6587 skb->data = data;
6588 skb->head_frag = 0;
6589 skb_set_end_offset(skb, size);
6590 skb_set_tail_pointer(skb, skb_headlen(skb));
6591 skb_headers_offset_update(skb, 0);
6592 skb->cloned = 0;
6593 skb->hdr_len = 0;
6594 skb->nohdr = 0;
6595 atomic_set(&skb_shinfo(skb)->dataref, 1);
6596
6597 return 0;
6598 }
6599
6600 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
6601
6602 /* carve out the first eat bytes from skb's frag_list. May recurse into
6603 * pskb_carve()
6604 */
pskb_carve_frag_list(struct sk_buff * skb,struct skb_shared_info * shinfo,int eat,gfp_t gfp_mask)6605 static int pskb_carve_frag_list(struct sk_buff *skb,
6606 struct skb_shared_info *shinfo, int eat,
6607 gfp_t gfp_mask)
6608 {
6609 struct sk_buff *list = shinfo->frag_list;
6610 struct sk_buff *clone = NULL;
6611 struct sk_buff *insp = NULL;
6612
6613 do {
6614 if (!list) {
6615 pr_err("Not enough bytes to eat. Want %d\n", eat);
6616 return -EFAULT;
6617 }
6618 if (list->len <= eat) {
6619 /* Eaten as whole. */
6620 eat -= list->len;
6621 list = list->next;
6622 insp = list;
6623 } else {
6624 /* Eaten partially. */
6625 if (skb_shared(list)) {
6626 clone = skb_clone(list, gfp_mask);
6627 if (!clone)
6628 return -ENOMEM;
6629 insp = list->next;
6630 list = clone;
6631 } else {
6632 /* This may be pulled without problems. */
6633 insp = list;
6634 }
6635 if (pskb_carve(list, eat, gfp_mask) < 0) {
6636 kfree_skb(clone);
6637 return -ENOMEM;
6638 }
6639 break;
6640 }
6641 } while (eat);
6642
6643 /* Free pulled out fragments. */
6644 while ((list = shinfo->frag_list) != insp) {
6645 shinfo->frag_list = list->next;
6646 consume_skb(list);
6647 }
6648 /* And insert new clone at head. */
6649 if (clone) {
6650 clone->next = list;
6651 shinfo->frag_list = clone;
6652 }
6653 return 0;
6654 }
6655
6656 /* carve off first len bytes from skb. Split line (off) is in the
6657 * non-linear part of skb
6658 */
pskb_carve_inside_nonlinear(struct sk_buff * skb,const u32 off,int pos,gfp_t gfp_mask)6659 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
6660 int pos, gfp_t gfp_mask)
6661 {
6662 int i, k = 0;
6663 unsigned int size = skb_end_offset(skb);
6664 u8 *data;
6665 const int nfrags = skb_shinfo(skb)->nr_frags;
6666 struct skb_shared_info *shinfo;
6667
6668 if (skb_pfmemalloc(skb))
6669 gfp_mask |= __GFP_MEMALLOC;
6670
6671 data = kmalloc_reserve(&size, gfp_mask, NUMA_NO_NODE, NULL);
6672 if (!data)
6673 return -ENOMEM;
6674 size = SKB_WITH_OVERHEAD(size);
6675
6676 memcpy((struct skb_shared_info *)(data + size),
6677 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
6678 if (skb_orphan_frags(skb, gfp_mask)) {
6679 skb_kfree_head(data, size);
6680 return -ENOMEM;
6681 }
6682 shinfo = (struct skb_shared_info *)(data + size);
6683 for (i = 0; i < nfrags; i++) {
6684 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
6685
6686 if (pos + fsize > off) {
6687 shinfo->frags[k] = skb_shinfo(skb)->frags[i];
6688
6689 if (pos < off) {
6690 /* Split frag.
6691 * We have two variants in this case:
6692 * 1. Move all the frag to the second
6693 * part, if it is possible. F.e.
6694 * this approach is mandatory for TUX,
6695 * where splitting is expensive.
6696 * 2. Split is accurately. We make this.
6697 */
6698 skb_frag_off_add(&shinfo->frags[0], off - pos);
6699 skb_frag_size_sub(&shinfo->frags[0], off - pos);
6700 }
6701 skb_frag_ref(skb, i);
6702 k++;
6703 }
6704 pos += fsize;
6705 }
6706 shinfo->nr_frags = k;
6707 if (skb_has_frag_list(skb))
6708 skb_clone_fraglist(skb);
6709
6710 /* split line is in frag list */
6711 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
6712 /* skb_frag_unref() is not needed here as shinfo->nr_frags = 0. */
6713 if (skb_has_frag_list(skb))
6714 kfree_skb_list(skb_shinfo(skb)->frag_list);
6715 skb_kfree_head(data, size);
6716 return -ENOMEM;
6717 }
6718 skb_release_data(skb, SKB_CONSUMED);
6719
6720 skb->head = data;
6721 skb->head_frag = 0;
6722 skb->data = data;
6723 skb_set_end_offset(skb, size);
6724 skb_reset_tail_pointer(skb);
6725 skb_headers_offset_update(skb, 0);
6726 skb->cloned = 0;
6727 skb->hdr_len = 0;
6728 skb->nohdr = 0;
6729 skb->len -= off;
6730 skb->data_len = skb->len;
6731 atomic_set(&skb_shinfo(skb)->dataref, 1);
6732 return 0;
6733 }
6734
6735 /* remove len bytes from the beginning of the skb */
pskb_carve(struct sk_buff * skb,const u32 len,gfp_t gfp)6736 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
6737 {
6738 int headlen = skb_headlen(skb);
6739
6740 if (len < headlen)
6741 return pskb_carve_inside_header(skb, len, headlen, gfp);
6742 else
6743 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
6744 }
6745
6746 /* Extract to_copy bytes starting at off from skb, and return this in
6747 * a new skb
6748 */
pskb_extract(struct sk_buff * skb,int off,int to_copy,gfp_t gfp)6749 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
6750 int to_copy, gfp_t gfp)
6751 {
6752 struct sk_buff *clone = skb_clone(skb, gfp);
6753
6754 if (!clone)
6755 return NULL;
6756
6757 if (pskb_carve(clone, off, gfp) < 0 ||
6758 pskb_trim(clone, to_copy)) {
6759 kfree_skb(clone);
6760 return NULL;
6761 }
6762 return clone;
6763 }
6764 EXPORT_SYMBOL(pskb_extract);
6765
6766 /**
6767 * skb_condense - try to get rid of fragments/frag_list if possible
6768 * @skb: buffer
6769 *
6770 * Can be used to save memory before skb is added to a busy queue.
6771 * If packet has bytes in frags and enough tail room in skb->head,
6772 * pull all of them, so that we can free the frags right now and adjust
6773 * truesize.
6774 * Notes:
6775 * We do not reallocate skb->head thus can not fail.
6776 * Caller must re-evaluate skb->truesize if needed.
6777 */
skb_condense(struct sk_buff * skb)6778 void skb_condense(struct sk_buff *skb)
6779 {
6780 if (skb->data_len) {
6781 if (skb->data_len > skb->end - skb->tail ||
6782 skb_cloned(skb))
6783 return;
6784
6785 /* Nice, we can free page frag(s) right now */
6786 __pskb_pull_tail(skb, skb->data_len);
6787 }
6788 /* At this point, skb->truesize might be over estimated,
6789 * because skb had a fragment, and fragments do not tell
6790 * their truesize.
6791 * When we pulled its content into skb->head, fragment
6792 * was freed, but __pskb_pull_tail() could not possibly
6793 * adjust skb->truesize, not knowing the frag truesize.
6794 */
6795 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
6796 }
6797 EXPORT_SYMBOL(skb_condense);
6798
6799 #ifdef CONFIG_SKB_EXTENSIONS
skb_ext_get_ptr(struct skb_ext * ext,enum skb_ext_id id)6800 static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)
6801 {
6802 return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);
6803 }
6804
6805 /**
6806 * __skb_ext_alloc - allocate a new skb extensions storage
6807 *
6808 * @flags: See kmalloc().
6809 *
6810 * Returns the newly allocated pointer. The pointer can later attached to a
6811 * skb via __skb_ext_set().
6812 * Note: caller must handle the skb_ext as an opaque data.
6813 */
__skb_ext_alloc(gfp_t flags)6814 struct skb_ext *__skb_ext_alloc(gfp_t flags)
6815 {
6816 struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
6817
6818 if (new) {
6819 memset(new->offset, 0, sizeof(new->offset));
6820 refcount_set(&new->refcnt, 1);
6821 }
6822
6823 return new;
6824 }
6825
skb_ext_maybe_cow(struct skb_ext * old,unsigned int old_active)6826 static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,
6827 unsigned int old_active)
6828 {
6829 struct skb_ext *new;
6830
6831 if (refcount_read(&old->refcnt) == 1)
6832 return old;
6833
6834 new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
6835 if (!new)
6836 return NULL;
6837
6838 memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
6839 refcount_set(&new->refcnt, 1);
6840
6841 #ifdef CONFIG_XFRM
6842 if (old_active & (1 << SKB_EXT_SEC_PATH)) {
6843 struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
6844 unsigned int i;
6845
6846 for (i = 0; i < sp->len; i++)
6847 xfrm_state_hold(sp->xvec[i]);
6848 }
6849 #endif
6850 #ifdef CONFIG_MCTP_FLOWS
6851 if (old_active & (1 << SKB_EXT_MCTP)) {
6852 struct mctp_flow *flow = skb_ext_get_ptr(old, SKB_EXT_MCTP);
6853
6854 if (flow->key)
6855 refcount_inc(&flow->key->refs);
6856 }
6857 #endif
6858 __skb_ext_put(old);
6859 return new;
6860 }
6861
6862 /**
6863 * __skb_ext_set - attach the specified extension storage to this skb
6864 * @skb: buffer
6865 * @id: extension id
6866 * @ext: extension storage previously allocated via __skb_ext_alloc()
6867 *
6868 * Existing extensions, if any, are cleared.
6869 *
6870 * Returns the pointer to the extension.
6871 */
__skb_ext_set(struct sk_buff * skb,enum skb_ext_id id,struct skb_ext * ext)6872 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
6873 struct skb_ext *ext)
6874 {
6875 unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
6876
6877 skb_ext_put(skb);
6878 newlen = newoff + skb_ext_type_len[id];
6879 ext->chunks = newlen;
6880 ext->offset[id] = newoff;
6881 skb->extensions = ext;
6882 skb->active_extensions = 1 << id;
6883 return skb_ext_get_ptr(ext, id);
6884 }
6885
6886 /**
6887 * skb_ext_add - allocate space for given extension, COW if needed
6888 * @skb: buffer
6889 * @id: extension to allocate space for
6890 *
6891 * Allocates enough space for the given extension.
6892 * If the extension is already present, a pointer to that extension
6893 * is returned.
6894 *
6895 * If the skb was cloned, COW applies and the returned memory can be
6896 * modified without changing the extension space of clones buffers.
6897 *
6898 * Returns pointer to the extension or NULL on allocation failure.
6899 */
skb_ext_add(struct sk_buff * skb,enum skb_ext_id id)6900 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
6901 {
6902 struct skb_ext *new, *old = NULL;
6903 unsigned int newlen, newoff;
6904
6905 if (skb->active_extensions) {
6906 old = skb->extensions;
6907
6908 new = skb_ext_maybe_cow(old, skb->active_extensions);
6909 if (!new)
6910 return NULL;
6911
6912 if (__skb_ext_exist(new, id))
6913 goto set_active;
6914
6915 newoff = new->chunks;
6916 } else {
6917 newoff = SKB_EXT_CHUNKSIZEOF(*new);
6918
6919 new = __skb_ext_alloc(GFP_ATOMIC);
6920 if (!new)
6921 return NULL;
6922 }
6923
6924 newlen = newoff + skb_ext_type_len[id];
6925 new->chunks = newlen;
6926 new->offset[id] = newoff;
6927 set_active:
6928 skb->slow_gro = 1;
6929 skb->extensions = new;
6930 skb->active_extensions |= 1 << id;
6931 return skb_ext_get_ptr(new, id);
6932 }
6933 EXPORT_SYMBOL(skb_ext_add);
6934
6935 #ifdef CONFIG_XFRM
skb_ext_put_sp(struct sec_path * sp)6936 static void skb_ext_put_sp(struct sec_path *sp)
6937 {
6938 unsigned int i;
6939
6940 for (i = 0; i < sp->len; i++)
6941 xfrm_state_put(sp->xvec[i]);
6942 }
6943 #endif
6944
6945 #ifdef CONFIG_MCTP_FLOWS
skb_ext_put_mctp(struct mctp_flow * flow)6946 static void skb_ext_put_mctp(struct mctp_flow *flow)
6947 {
6948 if (flow->key)
6949 mctp_key_unref(flow->key);
6950 }
6951 #endif
6952
__skb_ext_del(struct sk_buff * skb,enum skb_ext_id id)6953 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
6954 {
6955 struct skb_ext *ext = skb->extensions;
6956
6957 skb->active_extensions &= ~(1 << id);
6958 if (skb->active_extensions == 0) {
6959 skb->extensions = NULL;
6960 __skb_ext_put(ext);
6961 #ifdef CONFIG_XFRM
6962 } else if (id == SKB_EXT_SEC_PATH &&
6963 refcount_read(&ext->refcnt) == 1) {
6964 struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
6965
6966 skb_ext_put_sp(sp);
6967 sp->len = 0;
6968 #endif
6969 }
6970 }
6971 EXPORT_SYMBOL(__skb_ext_del);
6972
__skb_ext_put(struct skb_ext * ext)6973 void __skb_ext_put(struct skb_ext *ext)
6974 {
6975 /* If this is last clone, nothing can increment
6976 * it after check passes. Avoids one atomic op.
6977 */
6978 if (refcount_read(&ext->refcnt) == 1)
6979 goto free_now;
6980
6981 if (!refcount_dec_and_test(&ext->refcnt))
6982 return;
6983 free_now:
6984 #ifdef CONFIG_XFRM
6985 if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
6986 skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
6987 #endif
6988 #ifdef CONFIG_MCTP_FLOWS
6989 if (__skb_ext_exist(ext, SKB_EXT_MCTP))
6990 skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
6991 #endif
6992
6993 kmem_cache_free(skbuff_ext_cache, ext);
6994 }
6995 EXPORT_SYMBOL(__skb_ext_put);
6996 #endif /* CONFIG_SKB_EXTENSIONS */
6997
kfree_skb_napi_cache(struct sk_buff * skb)6998 static void kfree_skb_napi_cache(struct sk_buff *skb)
6999 {
7000 /* if SKB is a clone, don't handle this case */
7001 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
7002 __kfree_skb(skb);
7003 return;
7004 }
7005
7006 local_bh_disable();
7007 __napi_kfree_skb(skb, SKB_CONSUMED);
7008 local_bh_enable();
7009 }
7010
7011 /**
7012 * skb_attempt_defer_free - queue skb for remote freeing
7013 * @skb: buffer
7014 *
7015 * Put @skb in a per-cpu list, using the cpu which
7016 * allocated the skb/pages to reduce false sharing
7017 * and memory zone spinlock contention.
7018 */
skb_attempt_defer_free(struct sk_buff * skb)7019 void skb_attempt_defer_free(struct sk_buff *skb)
7020 {
7021 int cpu = skb->alloc_cpu;
7022 struct softnet_data *sd;
7023 unsigned int defer_max;
7024 bool kick;
7025
7026 if (cpu == raw_smp_processor_id() ||
7027 WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
7028 !cpu_online(cpu)) {
7029 nodefer: kfree_skb_napi_cache(skb);
7030 return;
7031 }
7032
7033 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb));
7034 DEBUG_NET_WARN_ON_ONCE(skb->destructor);
7035
7036 sd = &per_cpu(softnet_data, cpu);
7037 defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
7038 if (READ_ONCE(sd->defer_count) >= defer_max)
7039 goto nodefer;
7040
7041 spin_lock_bh(&sd->defer_lock);
7042 /* Send an IPI every time queue reaches half capacity. */
7043 kick = sd->defer_count == (defer_max >> 1);
7044 /* Paired with the READ_ONCE() few lines above */
7045 WRITE_ONCE(sd->defer_count, sd->defer_count + 1);
7046
7047 skb->next = sd->defer_list;
7048 /* Paired with READ_ONCE() in skb_defer_free_flush() */
7049 WRITE_ONCE(sd->defer_list, skb);
7050 spin_unlock_bh(&sd->defer_lock);
7051
7052 /* Make sure to trigger NET_RX_SOFTIRQ on the remote CPU
7053 * if we are unlucky enough (this seems very unlikely).
7054 */
7055 if (unlikely(kick))
7056 kick_defer_list_purge(sd, cpu);
7057 }
7058
skb_splice_csum_page(struct sk_buff * skb,struct page * page,size_t offset,size_t len)7059 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
7060 size_t offset, size_t len)
7061 {
7062 const char *kaddr;
7063 __wsum csum;
7064
7065 kaddr = kmap_local_page(page);
7066 csum = csum_partial(kaddr + offset, len, 0);
7067 kunmap_local(kaddr);
7068 skb->csum = csum_block_add(skb->csum, csum, skb->len);
7069 }
7070
7071 /**
7072 * skb_splice_from_iter - Splice (or copy) pages to skbuff
7073 * @skb: The buffer to add pages to
7074 * @iter: Iterator representing the pages to be added
7075 * @maxsize: Maximum amount of pages to be added
7076 * @gfp: Allocation flags
7077 *
7078 * This is a common helper function for supporting MSG_SPLICE_PAGES. It
7079 * extracts pages from an iterator and adds them to the socket buffer if
7080 * possible, copying them to fragments if not possible (such as if they're slab
7081 * pages).
7082 *
7083 * Returns the amount of data spliced/copied or -EMSGSIZE if there's
7084 * insufficient space in the buffer to transfer anything.
7085 */
skb_splice_from_iter(struct sk_buff * skb,struct iov_iter * iter,ssize_t maxsize,gfp_t gfp)7086 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
7087 ssize_t maxsize, gfp_t gfp)
7088 {
7089 size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags);
7090 struct page *pages[8], **ppages = pages;
7091 ssize_t spliced = 0, ret = 0;
7092 unsigned int i;
7093
7094 while (iter->count > 0) {
7095 ssize_t space, nr, len;
7096 size_t off;
7097
7098 ret = -EMSGSIZE;
7099 space = frag_limit - skb_shinfo(skb)->nr_frags;
7100 if (space < 0)
7101 break;
7102
7103 /* We might be able to coalesce without increasing nr_frags */
7104 nr = clamp_t(size_t, space, 1, ARRAY_SIZE(pages));
7105
7106 len = iov_iter_extract_pages(iter, &ppages, maxsize, nr, 0, &off);
7107 if (len <= 0) {
7108 ret = len ?: -EIO;
7109 break;
7110 }
7111
7112 i = 0;
7113 do {
7114 struct page *page = pages[i++];
7115 size_t part = min_t(size_t, PAGE_SIZE - off, len);
7116
7117 ret = -EIO;
7118 if (WARN_ON_ONCE(!sendpage_ok(page)))
7119 goto out;
7120
7121 ret = skb_append_pagefrags(skb, page, off, part,
7122 frag_limit);
7123 if (ret < 0) {
7124 iov_iter_revert(iter, len);
7125 goto out;
7126 }
7127
7128 if (skb->ip_summed == CHECKSUM_NONE)
7129 skb_splice_csum_page(skb, page, off, part);
7130
7131 off = 0;
7132 spliced += part;
7133 maxsize -= part;
7134 len -= part;
7135 } while (len > 0);
7136
7137 if (maxsize <= 0)
7138 break;
7139 }
7140
7141 out:
7142 skb_len_add(skb, spliced);
7143 return spliced ?: ret;
7144 }
7145 EXPORT_SYMBOL(skb_splice_from_iter);
7146
7147 static __always_inline
memcpy_from_iter_csum(void * iter_from,size_t progress,size_t len,void * to,void * priv2)7148 size_t memcpy_from_iter_csum(void *iter_from, size_t progress,
7149 size_t len, void *to, void *priv2)
7150 {
7151 __wsum *csum = priv2;
7152 __wsum next = csum_partial_copy_nocheck(iter_from, to + progress, len);
7153
7154 *csum = csum_block_add(*csum, next, progress);
7155 return 0;
7156 }
7157
7158 static __always_inline
copy_from_user_iter_csum(void __user * iter_from,size_t progress,size_t len,void * to,void * priv2)7159 size_t copy_from_user_iter_csum(void __user *iter_from, size_t progress,
7160 size_t len, void *to, void *priv2)
7161 {
7162 __wsum next, *csum = priv2;
7163
7164 next = csum_and_copy_from_user(iter_from, to + progress, len);
7165 *csum = csum_block_add(*csum, next, progress);
7166 return next ? 0 : len;
7167 }
7168
csum_and_copy_from_iter_full(void * addr,size_t bytes,__wsum * csum,struct iov_iter * i)7169 bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
7170 __wsum *csum, struct iov_iter *i)
7171 {
7172 size_t copied;
7173
7174 if (WARN_ON_ONCE(!i->data_source))
7175 return false;
7176 copied = iterate_and_advance2(i, bytes, addr, csum,
7177 copy_from_user_iter_csum,
7178 memcpy_from_iter_csum);
7179 if (likely(copied == bytes))
7180 return true;
7181 iov_iter_revert(i, copied);
7182 return false;
7183 }
7184 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
7185