1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk). 4 * 5 * (C) SGI 2006, Christoph Lameter 6 * Cleaned up and restructured to ease the addition of alternative 7 * implementations of SLAB allocators. 8 * (C) Linux Foundation 2008-2013 9 * Unified interface for all slab allocators 10 */ 11 12 #ifndef _LINUX_SLAB_H 13 #define _LINUX_SLAB_H 14 15 #include <linux/gfp.h> 16 #include <linux/overflow.h> 17 #include <linux/types.h> 18 #include <linux/workqueue.h> 19 #include <linux/percpu-refcount.h> 20 21 22 /* 23 * Flags to pass to kmem_cache_create(). 24 * The ones marked DEBUG are only valid if CONFIG_DEBUG_SLAB is set. 25 */ 26 /* DEBUG: Perform (expensive) checks on alloc/free */ 27 #define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) 28 /* DEBUG: Red zone objs in a cache */ 29 #define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) 30 /* DEBUG: Poison objects */ 31 #define SLAB_POISON ((slab_flags_t __force)0x00000800U) 32 /* Indicate a kmalloc slab */ 33 #define SLAB_KMALLOC ((slab_flags_t __force)0x00001000U) 34 /* Align objs on cache lines */ 35 #define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) 36 /* Use GFP_DMA memory */ 37 #define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) 38 /* Use GFP_DMA32 memory */ 39 #define SLAB_CACHE_DMA32 ((slab_flags_t __force)0x00008000U) 40 /* DEBUG: Store the last owner for bug hunting */ 41 #define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) 42 /* Panic if kmem_cache_create() fails */ 43 #define SLAB_PANIC ((slab_flags_t __force)0x00040000U) 44 /* 45 * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 46 * 47 * This delays freeing the SLAB page by a grace period, it does _NOT_ 48 * delay object freeing. This means that if you do kmem_cache_free() 49 * that memory location is free to be reused at any time. Thus it may 50 * be possible to see another object there in the same RCU grace period. 51 * 52 * This feature only ensures the memory location backing the object 53 * stays valid, the trick to using this is relying on an independent 54 * object validation pass. Something like: 55 * 56 * rcu_read_lock() 57 * again: 58 * obj = lockless_lookup(key); 59 * if (obj) { 60 * if (!try_get_ref(obj)) // might fail for free objects 61 * goto again; 62 * 63 * if (obj->key != key) { // not the object we expected 64 * put_ref(obj); 65 * goto again; 66 * } 67 * } 68 * rcu_read_unlock(); 69 * 70 * This is useful if we need to approach a kernel structure obliquely, 71 * from its address obtained without the usual locking. We can lock 72 * the structure to stabilize it and check it's still at the given address, 73 * only if we can be sure that the memory has not been meanwhile reused 74 * for some other kind of object (which our subsystem's lock might corrupt). 75 * 76 * rcu_read_lock before reading the address, then rcu_read_unlock after 77 * taking the spinlock within the structure expected at that address. 78 * 79 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. 80 */ 81 /* Defer freeing slabs to RCU */ 82 #define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) 83 /* Spread some memory over cpuset */ 84 #define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) 85 /* Trace allocations and frees */ 86 #define SLAB_TRACE ((slab_flags_t __force)0x00200000U) 87 88 /* Flag to prevent checks on free */ 89 #ifdef CONFIG_DEBUG_OBJECTS 90 # define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) 91 #else 92 # define SLAB_DEBUG_OBJECTS 0 93 #endif 94 95 /* Avoid kmemleak tracing */ 96 #define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) 97 98 /* Fault injection mark */ 99 #ifdef CONFIG_FAILSLAB 100 # define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) 101 #else 102 # define SLAB_FAILSLAB 0 103 #endif 104 /* Account to memcg */ 105 #ifdef CONFIG_MEMCG_KMEM 106 # define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) 107 #else 108 # define SLAB_ACCOUNT 0 109 #endif 110 111 #ifdef CONFIG_KASAN_GENERIC 112 #define SLAB_KASAN ((slab_flags_t __force)0x08000000U) 113 #else 114 #define SLAB_KASAN 0 115 #endif 116 117 /* 118 * Ignore user specified debugging flags. 119 * Intended for caches created for self-tests so they have only flags 120 * specified in the code and other flags are ignored. 121 */ 122 #define SLAB_NO_USER_FLAGS ((slab_flags_t __force)0x10000000U) 123 124 #ifdef CONFIG_KFENCE 125 #define SLAB_SKIP_KFENCE ((slab_flags_t __force)0x20000000U) 126 #else 127 #define SLAB_SKIP_KFENCE 0 128 #endif 129 130 /* The following flags affect the page allocator grouping pages by mobility */ 131 /* Objects are reclaimable */ 132 #define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) 133 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ 134 135 /* 136 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. 137 * 138 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault. 139 * 140 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can. 141 * Both make kfree a no-op. 142 */ 143 #define ZERO_SIZE_PTR ((void *)16) 144 145 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ 146 (unsigned long)ZERO_SIZE_PTR) 147 148 #include <linux/kasan.h> 149 150 struct list_lru; 151 struct mem_cgroup; 152 /* 153 * struct kmem_cache related prototypes 154 */ 155 void __init kmem_cache_init(void); 156 bool slab_is_available(void); 157 158 struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, 159 unsigned int align, slab_flags_t flags, 160 void (*ctor)(void *)); 161 struct kmem_cache *kmem_cache_create_usercopy(const char *name, 162 unsigned int size, unsigned int align, 163 slab_flags_t flags, 164 unsigned int useroffset, unsigned int usersize, 165 void (*ctor)(void *)); 166 void kmem_cache_destroy(struct kmem_cache *s); 167 int kmem_cache_shrink(struct kmem_cache *s); 168 169 /* 170 * Please use this macro to create slab caches. Simply specify the 171 * name of the structure and maybe some flags that are listed above. 172 * 173 * The alignment of the struct determines object alignment. If you 174 * f.e. add ____cacheline_aligned_in_smp to the struct declaration 175 * then the objects will be properly aligned in SMP configurations. 176 */ 177 #define KMEM_CACHE(__struct, __flags) \ 178 kmem_cache_create(#__struct, sizeof(struct __struct), \ 179 __alignof__(struct __struct), (__flags), NULL) 180 181 /* 182 * To whitelist a single field for copying to/from usercopy, use this 183 * macro instead for KMEM_CACHE() above. 184 */ 185 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \ 186 kmem_cache_create_usercopy(#__struct, \ 187 sizeof(struct __struct), \ 188 __alignof__(struct __struct), (__flags), \ 189 offsetof(struct __struct, __field), \ 190 sizeof_field(struct __struct, __field), NULL) 191 192 /* 193 * Common kmalloc functions provided by all allocators 194 */ 195 void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2); 196 void kfree(const void *objp); 197 void kfree_sensitive(const void *objp); 198 size_t __ksize(const void *objp); 199 200 /** 201 * ksize - Report actual allocation size of associated object 202 * 203 * @objp: Pointer returned from a prior kmalloc()-family allocation. 204 * 205 * This should not be used for writing beyond the originally requested 206 * allocation size. Either use krealloc() or round up the allocation size 207 * with kmalloc_size_roundup() prior to allocation. If this is used to 208 * access beyond the originally requested allocation size, UBSAN_BOUNDS 209 * and/or FORTIFY_SOURCE may trip, since they only know about the 210 * originally allocated size via the __alloc_size attribute. 211 */ 212 size_t ksize(const void *objp); 213 214 #ifdef CONFIG_PRINTK 215 bool kmem_valid_obj(void *object); 216 void kmem_dump_obj(void *object); 217 #endif 218 219 /* 220 * Some archs want to perform DMA into kmalloc caches and need a guaranteed 221 * alignment larger than the alignment of a 64-bit integer. 222 * Setting ARCH_DMA_MINALIGN in arch headers allows that. 223 */ 224 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 225 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN 226 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN 227 #define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) 228 #else 229 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) 230 #endif 231 232 /* 233 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. 234 * Intended for arches that get misalignment faults even for 64 bit integer 235 * aligned buffers. 236 */ 237 #ifndef ARCH_SLAB_MINALIGN 238 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 239 #endif 240 241 /* 242 * Arches can define this function if they want to decide the minimum slab 243 * alignment at runtime. The value returned by the function must be a power 244 * of two and >= ARCH_SLAB_MINALIGN. 245 */ 246 #ifndef arch_slab_minalign 247 static inline unsigned int arch_slab_minalign(void) 248 { 249 return ARCH_SLAB_MINALIGN; 250 } 251 #endif 252 253 /* 254 * kmem_cache_alloc and friends return pointers aligned to ARCH_SLAB_MINALIGN. 255 * kmalloc and friends return pointers aligned to both ARCH_KMALLOC_MINALIGN 256 * and ARCH_SLAB_MINALIGN, but here we only assume the former alignment. 257 */ 258 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN) 259 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN) 260 #define __assume_page_alignment __assume_aligned(PAGE_SIZE) 261 262 /* 263 * Kmalloc array related definitions 264 */ 265 266 #ifdef CONFIG_SLAB 267 /* 268 * SLAB and SLUB directly allocates requests fitting in to an order-1 page 269 * (PAGE_SIZE*2). Larger requests are passed to the page allocator. 270 */ 271 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 272 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 273 #ifndef KMALLOC_SHIFT_LOW 274 #define KMALLOC_SHIFT_LOW 5 275 #endif 276 #endif 277 278 #ifdef CONFIG_SLUB 279 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 280 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 281 #ifndef KMALLOC_SHIFT_LOW 282 #define KMALLOC_SHIFT_LOW 3 283 #endif 284 #endif 285 286 #ifdef CONFIG_SLOB 287 /* 288 * SLOB passes all requests larger than one page to the page allocator. 289 * No kmalloc array is necessary since objects of different sizes can 290 * be allocated from the same page. 291 */ 292 #define KMALLOC_SHIFT_HIGH PAGE_SHIFT 293 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 294 #ifndef KMALLOC_SHIFT_LOW 295 #define KMALLOC_SHIFT_LOW 3 296 #endif 297 #endif 298 299 /* Maximum allocatable size */ 300 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) 301 /* Maximum size for which we actually use a slab cache */ 302 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) 303 /* Maximum order allocatable via the slab allocator */ 304 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) 305 306 /* 307 * Kmalloc subsystem. 308 */ 309 #ifndef KMALLOC_MIN_SIZE 310 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) 311 #endif 312 313 /* 314 * This restriction comes from byte sized index implementation. 315 * Page size is normally 2^12 bytes and, in this case, if we want to use 316 * byte sized index which can represent 2^8 entries, the size of the object 317 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16. 318 * If minimum size of kmalloc is less than 16, we use it as minimum object 319 * size and give up to use byte sized index. 320 */ 321 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ 322 (KMALLOC_MIN_SIZE) : 16) 323 324 /* 325 * Whenever changing this, take care of that kmalloc_type() and 326 * create_kmalloc_caches() still work as intended. 327 * 328 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP 329 * is for accounted but unreclaimable and non-dma objects. All the other 330 * kmem caches can have both accounted and unaccounted objects. 331 */ 332 enum kmalloc_cache_type { 333 KMALLOC_NORMAL = 0, 334 #ifndef CONFIG_ZONE_DMA 335 KMALLOC_DMA = KMALLOC_NORMAL, 336 #endif 337 #ifndef CONFIG_MEMCG_KMEM 338 KMALLOC_CGROUP = KMALLOC_NORMAL, 339 #else 340 KMALLOC_CGROUP, 341 #endif 342 KMALLOC_RECLAIM, 343 #ifdef CONFIG_ZONE_DMA 344 KMALLOC_DMA, 345 #endif 346 NR_KMALLOC_TYPES 347 }; 348 349 #ifndef CONFIG_SLOB 350 extern struct kmem_cache * 351 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1]; 352 353 /* 354 * Define gfp bits that should not be set for KMALLOC_NORMAL. 355 */ 356 #define KMALLOC_NOT_NORMAL_BITS \ 357 (__GFP_RECLAIMABLE | \ 358 (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \ 359 (IS_ENABLED(CONFIG_MEMCG_KMEM) ? __GFP_ACCOUNT : 0)) 360 361 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags) 362 { 363 /* 364 * The most common case is KMALLOC_NORMAL, so test for it 365 * with a single branch for all the relevant flags. 366 */ 367 if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0)) 368 return KMALLOC_NORMAL; 369 370 /* 371 * At least one of the flags has to be set. Their priorities in 372 * decreasing order are: 373 * 1) __GFP_DMA 374 * 2) __GFP_RECLAIMABLE 375 * 3) __GFP_ACCOUNT 376 */ 377 if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA)) 378 return KMALLOC_DMA; 379 if (!IS_ENABLED(CONFIG_MEMCG_KMEM) || (flags & __GFP_RECLAIMABLE)) 380 return KMALLOC_RECLAIM; 381 else 382 return KMALLOC_CGROUP; 383 } 384 385 /* 386 * Figure out which kmalloc slab an allocation of a certain size 387 * belongs to. 388 * 0 = zero alloc 389 * 1 = 65 .. 96 bytes 390 * 2 = 129 .. 192 bytes 391 * n = 2^(n-1)+1 .. 2^n 392 * 393 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized; 394 * typical usage is via kmalloc_index() and therefore evaluated at compile-time. 395 * Callers where !size_is_constant should only be test modules, where runtime 396 * overheads of __kmalloc_index() can be tolerated. Also see kmalloc_slab(). 397 */ 398 static __always_inline unsigned int __kmalloc_index(size_t size, 399 bool size_is_constant) 400 { 401 if (!size) 402 return 0; 403 404 if (size <= KMALLOC_MIN_SIZE) 405 return KMALLOC_SHIFT_LOW; 406 407 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) 408 return 1; 409 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) 410 return 2; 411 if (size <= 8) return 3; 412 if (size <= 16) return 4; 413 if (size <= 32) return 5; 414 if (size <= 64) return 6; 415 if (size <= 128) return 7; 416 if (size <= 256) return 8; 417 if (size <= 512) return 9; 418 if (size <= 1024) return 10; 419 if (size <= 2 * 1024) return 11; 420 if (size <= 4 * 1024) return 12; 421 if (size <= 8 * 1024) return 13; 422 if (size <= 16 * 1024) return 14; 423 if (size <= 32 * 1024) return 15; 424 if (size <= 64 * 1024) return 16; 425 if (size <= 128 * 1024) return 17; 426 if (size <= 256 * 1024) return 18; 427 if (size <= 512 * 1024) return 19; 428 if (size <= 1024 * 1024) return 20; 429 if (size <= 2 * 1024 * 1024) return 21; 430 431 if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) 432 BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()"); 433 else 434 BUG(); 435 436 /* Will never be reached. Needed because the compiler may complain */ 437 return -1; 438 } 439 static_assert(PAGE_SHIFT <= 20); 440 #define kmalloc_index(s) __kmalloc_index(s, true) 441 #endif /* !CONFIG_SLOB */ 442 443 void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); 444 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t flags) __assume_slab_alignment __malloc; 445 void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, 446 gfp_t gfpflags) __assume_slab_alignment __malloc; 447 void kmem_cache_free(struct kmem_cache *s, void *objp); 448 449 /* 450 * Bulk allocation and freeing operations. These are accelerated in an 451 * allocator specific way to avoid taking locks repeatedly or building 452 * metadata structures unnecessarily. 453 * 454 * Note that interrupts must be enabled when calling these functions. 455 */ 456 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); 457 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p); 458 459 /* 460 * Caller must not use kfree_bulk() on memory not originally allocated 461 * by kmalloc(), because the SLOB allocator cannot handle this. 462 */ 463 static __always_inline void kfree_bulk(size_t size, void **p) 464 { 465 kmem_cache_free_bulk(NULL, size, p); 466 } 467 468 void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment 469 __alloc_size(1); 470 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment 471 __malloc; 472 473 #ifdef CONFIG_TRACING 474 void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) 475 __assume_kmalloc_alignment __alloc_size(3); 476 477 void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, 478 int node, size_t size) __assume_kmalloc_alignment 479 __alloc_size(4); 480 #else /* CONFIG_TRACING */ 481 /* Save a function call when CONFIG_TRACING=n */ 482 static __always_inline __alloc_size(3) 483 void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) 484 { 485 void *ret = kmem_cache_alloc(s, flags); 486 487 ret = kasan_kmalloc(s, ret, size, flags); 488 return ret; 489 } 490 491 static __always_inline __alloc_size(4) 492 void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, 493 int node, size_t size) 494 { 495 void *ret = kmem_cache_alloc_node(s, gfpflags, node); 496 497 ret = kasan_kmalloc(s, ret, size, gfpflags); 498 return ret; 499 } 500 #endif /* CONFIG_TRACING */ 501 502 void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment 503 __alloc_size(1); 504 505 void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment 506 __alloc_size(1); 507 508 /** 509 * kmalloc - allocate memory 510 * @size: how many bytes of memory are required. 511 * @flags: the type of memory to allocate. 512 * 513 * kmalloc is the normal method of allocating memory 514 * for objects smaller than page size in the kernel. 515 * 516 * The allocated object address is aligned to at least ARCH_KMALLOC_MINALIGN 517 * bytes. For @size of power of two bytes, the alignment is also guaranteed 518 * to be at least to the size. 519 * 520 * The @flags argument may be one of the GFP flags defined at 521 * include/linux/gfp.h and described at 522 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>` 523 * 524 * The recommended usage of the @flags is described at 525 * :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>` 526 * 527 * Below is a brief outline of the most useful GFP flags 528 * 529 * %GFP_KERNEL 530 * Allocate normal kernel ram. May sleep. 531 * 532 * %GFP_NOWAIT 533 * Allocation will not sleep. 534 * 535 * %GFP_ATOMIC 536 * Allocation will not sleep. May use emergency pools. 537 * 538 * %GFP_HIGHUSER 539 * Allocate memory from high memory on behalf of user. 540 * 541 * Also it is possible to set different flags by OR'ing 542 * in one or more of the following additional @flags: 543 * 544 * %__GFP_HIGH 545 * This allocation has high priority and may use emergency pools. 546 * 547 * %__GFP_NOFAIL 548 * Indicate that this allocation is in no way allowed to fail 549 * (think twice before using). 550 * 551 * %__GFP_NORETRY 552 * If memory is not immediately available, 553 * then give up at once. 554 * 555 * %__GFP_NOWARN 556 * If allocation fails, don't issue any warnings. 557 * 558 * %__GFP_RETRY_MAYFAIL 559 * Try really hard to succeed the allocation but fail 560 * eventually. 561 */ 562 static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) 563 { 564 if (__builtin_constant_p(size)) { 565 #ifndef CONFIG_SLOB 566 unsigned int index; 567 #endif 568 if (size > KMALLOC_MAX_CACHE_SIZE) 569 return kmalloc_large(size, flags); 570 #ifndef CONFIG_SLOB 571 index = kmalloc_index(size); 572 573 if (!index) 574 return ZERO_SIZE_PTR; 575 576 return kmalloc_trace( 577 kmalloc_caches[kmalloc_type(flags)][index], 578 flags, size); 579 #endif 580 } 581 return __kmalloc(size, flags); 582 } 583 584 #ifndef CONFIG_SLOB 585 static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) 586 { 587 if (__builtin_constant_p(size)) { 588 unsigned int index; 589 590 if (size > KMALLOC_MAX_CACHE_SIZE) 591 return kmalloc_large_node(size, flags, node); 592 593 index = kmalloc_index(size); 594 595 if (!index) 596 return ZERO_SIZE_PTR; 597 598 return kmalloc_node_trace( 599 kmalloc_caches[kmalloc_type(flags)][index], 600 flags, node, size); 601 } 602 return __kmalloc_node(size, flags, node); 603 } 604 #else 605 static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) 606 { 607 if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE) 608 return kmalloc_large_node(size, flags, node); 609 610 return __kmalloc_node(size, flags, node); 611 } 612 #endif 613 614 /** 615 * kmalloc_array - allocate memory for an array. 616 * @n: number of elements. 617 * @size: element size. 618 * @flags: the type of memory to allocate (see kmalloc). 619 */ 620 static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_t flags) 621 { 622 size_t bytes; 623 624 if (unlikely(check_mul_overflow(n, size, &bytes))) 625 return NULL; 626 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 627 return kmalloc(bytes, flags); 628 return __kmalloc(bytes, flags); 629 } 630 631 /** 632 * krealloc_array - reallocate memory for an array. 633 * @p: pointer to the memory chunk to reallocate 634 * @new_n: new number of elements to alloc 635 * @new_size: new size of a single member of the array 636 * @flags: the type of memory to allocate (see kmalloc) 637 */ 638 static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, 639 size_t new_n, 640 size_t new_size, 641 gfp_t flags) 642 { 643 size_t bytes; 644 645 if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) 646 return NULL; 647 648 return krealloc(p, bytes, flags); 649 } 650 651 /** 652 * kcalloc - allocate memory for an array. The memory is set to zero. 653 * @n: number of elements. 654 * @size: element size. 655 * @flags: the type of memory to allocate (see kmalloc). 656 */ 657 static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flags) 658 { 659 return kmalloc_array(n, size, flags | __GFP_ZERO); 660 } 661 662 void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, 663 unsigned long caller) __alloc_size(1); 664 #define kmalloc_node_track_caller(size, flags, node) \ 665 __kmalloc_node_track_caller(size, flags, node, \ 666 _RET_IP_) 667 668 /* 669 * kmalloc_track_caller is a special version of kmalloc that records the 670 * calling function of the routine calling it for slab leak tracking instead 671 * of just the calling function (confusing, eh?). 672 * It's useful when the call to kmalloc comes from a widely-used standard 673 * allocator where we care about the real place the memory allocation 674 * request comes from. 675 */ 676 #define kmalloc_track_caller(size, flags) \ 677 __kmalloc_node_track_caller(size, flags, \ 678 NUMA_NO_NODE, _RET_IP_) 679 680 static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, 681 int node) 682 { 683 size_t bytes; 684 685 if (unlikely(check_mul_overflow(n, size, &bytes))) 686 return NULL; 687 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 688 return kmalloc_node(bytes, flags, node); 689 return __kmalloc_node(bytes, flags, node); 690 } 691 692 static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node) 693 { 694 return kmalloc_array_node(n, size, flags | __GFP_ZERO, node); 695 } 696 697 /* 698 * Shortcuts 699 */ 700 static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) 701 { 702 return kmem_cache_alloc(k, flags | __GFP_ZERO); 703 } 704 705 /** 706 * kzalloc - allocate memory. The memory is set to zero. 707 * @size: how many bytes of memory are required. 708 * @flags: the type of memory to allocate (see kmalloc). 709 */ 710 static inline __alloc_size(1) void *kzalloc(size_t size, gfp_t flags) 711 { 712 return kmalloc(size, flags | __GFP_ZERO); 713 } 714 715 /** 716 * kzalloc_node - allocate zeroed memory from a particular memory node. 717 * @size: how many bytes of memory are required. 718 * @flags: the type of memory to allocate (see kmalloc). 719 * @node: memory node from which to allocate 720 */ 721 static inline __alloc_size(1) void *kzalloc_node(size_t size, gfp_t flags, int node) 722 { 723 return kmalloc_node(size, flags | __GFP_ZERO, node); 724 } 725 726 extern void *kvmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1); 727 static inline __alloc_size(1) void *kvmalloc(size_t size, gfp_t flags) 728 { 729 return kvmalloc_node(size, flags, NUMA_NO_NODE); 730 } 731 static inline __alloc_size(1) void *kvzalloc_node(size_t size, gfp_t flags, int node) 732 { 733 return kvmalloc_node(size, flags | __GFP_ZERO, node); 734 } 735 static inline __alloc_size(1) void *kvzalloc(size_t size, gfp_t flags) 736 { 737 return kvmalloc(size, flags | __GFP_ZERO); 738 } 739 740 static inline __alloc_size(1, 2) void *kvmalloc_array(size_t n, size_t size, gfp_t flags) 741 { 742 size_t bytes; 743 744 if (unlikely(check_mul_overflow(n, size, &bytes))) 745 return NULL; 746 747 return kvmalloc(bytes, flags); 748 } 749 750 static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags) 751 { 752 return kvmalloc_array(n, size, flags | __GFP_ZERO); 753 } 754 755 extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) 756 __realloc_size(3); 757 extern void kvfree(const void *addr); 758 extern void kvfree_sensitive(const void *addr, size_t len); 759 760 unsigned int kmem_cache_size(struct kmem_cache *s); 761 762 /** 763 * kmalloc_size_roundup - Report allocation bucket size for the given size 764 * 765 * @size: Number of bytes to round up from. 766 * 767 * This returns the number of bytes that would be available in a kmalloc() 768 * allocation of @size bytes. For example, a 126 byte request would be 769 * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly 770 * for the general-purpose kmalloc()-based allocations, and is not for the 771 * pre-sized kmem_cache_alloc()-based allocations.) 772 * 773 * Use this to kmalloc() the full bucket size ahead of time instead of using 774 * ksize() to query the size after an allocation. 775 */ 776 size_t kmalloc_size_roundup(size_t size); 777 778 void __init kmem_cache_init_late(void); 779 780 #if defined(CONFIG_SMP) && defined(CONFIG_SLAB) 781 int slab_prepare_cpu(unsigned int cpu); 782 int slab_dead_cpu(unsigned int cpu); 783 #else 784 #define slab_prepare_cpu NULL 785 #define slab_dead_cpu NULL 786 #endif 787 788 #endif /* _LINUX_SLAB_H */ 789