18d59ecb2SHans Petter Selasky /*-
28d59ecb2SHans Petter Selasky * Copyright (c) 2010 Isilon Systems, Inc.
38d59ecb2SHans Petter Selasky * Copyright (c) 2010 iX Systems, Inc.
48d59ecb2SHans Petter Selasky * Copyright (c) 2010 Panasas, Inc.
5ebe5cf35SHans Petter Selasky * Copyright (c) 2013-2021 Mellanox Technologies, Ltd.
68d59ecb2SHans Petter Selasky * All rights reserved.
78d59ecb2SHans Petter Selasky *
88d59ecb2SHans Petter Selasky * Redistribution and use in source and binary forms, with or without
98d59ecb2SHans Petter Selasky * modification, are permitted provided that the following conditions
108d59ecb2SHans Petter Selasky * are met:
118d59ecb2SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright
128d59ecb2SHans Petter Selasky * notice unmodified, this list of conditions, and the following
138d59ecb2SHans Petter Selasky * disclaimer.
148d59ecb2SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright
158d59ecb2SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the
168d59ecb2SHans Petter Selasky * documentation and/or other materials provided with the distribution.
178d59ecb2SHans Petter Selasky *
188d59ecb2SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
198d59ecb2SHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
208d59ecb2SHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
218d59ecb2SHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
228d59ecb2SHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
238d59ecb2SHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
248d59ecb2SHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
258d59ecb2SHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
268d59ecb2SHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
278d59ecb2SHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
288d59ecb2SHans Petter Selasky */
29307f78f3SVladimir Kondratyev #ifndef _LINUXKPI_LINUX_SLAB_H_
30307f78f3SVladimir Kondratyev #define _LINUXKPI_LINUX_SLAB_H_
318d59ecb2SHans Petter Selasky
32a2b83b59SVladimir Kondratyev #include <sys/types.h>
338d59ecb2SHans Petter Selasky #include <sys/malloc.h>
340a475c59SHans Petter Selasky #include <sys/limits.h>
358d59ecb2SHans Petter Selasky
36ebe5cf35SHans Petter Selasky #include <linux/compat.h>
378d59ecb2SHans Petter Selasky #include <linux/types.h>
388d59ecb2SHans Petter Selasky #include <linux/gfp.h>
39ec25b6faSVladimir Kondratyev #include <linux/llist.h>
40f1a7639aSVladimir Kondratyev #include <linux/overflow.h>
418d59ecb2SHans Petter Selasky
428d59ecb2SHans Petter Selasky MALLOC_DECLARE(M_KMALLOC);
438d59ecb2SHans Petter Selasky
44c0b8047bSVladimir Kondratyev #define kmalloc(size, flags) lkpi_kmalloc(size, flags)
45925245aaSHans Petter Selasky #define kvmalloc(size, flags) kmalloc(size, flags)
46925245aaSHans Petter Selasky #define kvzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
47925245aaSHans Petter Selasky #define kvcalloc(n, size, flags) kvmalloc_array(n, size, (flags) | __GFP_ZERO)
4814c5024dSHans Petter Selasky #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
49ebe5cf35SHans Petter Selasky #define kzalloc_node(size, flags, node) kmalloc_node(size, (flags) | __GFP_ZERO, node)
50510ebed7SHans Petter Selasky #define kfree_const(ptr) kfree(ptr)
518a8e86b8SJean-Sébastien Pédron #define vzalloc(size) __vmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0)
528d59ecb2SHans Petter Selasky #define vfree(arg) kfree(arg)
538d59ecb2SHans Petter Selasky #define kvfree(arg) kfree(arg)
548a8e86b8SJean-Sébastien Pédron #define vmalloc_node(size, node) __vmalloc_node(size, GFP_KERNEL, node)
558a8e86b8SJean-Sébastien Pédron #define vmalloc_user(size) __vmalloc(size, GFP_KERNEL | __GFP_ZERO, 0)
568a8e86b8SJean-Sébastien Pédron #define vmalloc(size) __vmalloc(size, GFP_KERNEL, 0)
577fdce5c4SHans Petter Selasky
587fdce5c4SHans Petter Selasky /*
597fdce5c4SHans Petter Selasky * Prefix some functions with linux_ to avoid namespace conflict
607fdce5c4SHans Petter Selasky * with the OpenSolaris code in the kernel.
617fdce5c4SHans Petter Selasky */
627fdce5c4SHans Petter Selasky #define kmem_cache linux_kmem_cache
637fdce5c4SHans Petter Selasky #define kmem_cache_create(...) linux_kmem_cache_create(__VA_ARGS__)
64a2b83b59SVladimir Kondratyev #define kmem_cache_alloc(...) lkpi_kmem_cache_alloc(__VA_ARGS__)
65a2b83b59SVladimir Kondratyev #define kmem_cache_zalloc(...) lkpi_kmem_cache_zalloc(__VA_ARGS__)
66a2b83b59SVladimir Kondratyev #define kmem_cache_free(...) lkpi_kmem_cache_free(__VA_ARGS__)
677fdce5c4SHans Petter Selasky #define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__)
6835b7625eSEmmanuel Vadot #define kmem_cache_shrink(x) (0)
697fdce5c4SHans Petter Selasky
7078d74419SHans Petter Selasky #define KMEM_CACHE(__struct, flags) \
7178d74419SHans Petter Selasky linux_kmem_cache_create(#__struct, sizeof(struct __struct), \
7278d74419SHans Petter Selasky __alignof(struct __struct), (flags), NULL)
7378d74419SHans Petter Selasky
7414c5024dSHans Petter Selasky typedef void linux_kmem_ctor_t (void *);
7514c5024dSHans Petter Selasky
76a2b83b59SVladimir Kondratyev struct linux_kmem_cache;
778d59ecb2SHans Petter Selasky
7814c5024dSHans Petter Selasky #define SLAB_HWCACHE_ALIGN (1 << 0)
79782a90d1SHans Petter Selasky #define SLAB_TYPESAFE_BY_RCU (1 << 1)
8014c5024dSHans Petter Selasky #define SLAB_RECLAIM_ACCOUNT (1 << 2)
818d59ecb2SHans Petter Selasky
82ce930365SHans Petter Selasky #define SLAB_DESTROY_BY_RCU \
83ce930365SHans Petter Selasky SLAB_TYPESAFE_BY_RCU
84ce930365SHans Petter Selasky
8571ee95ddSHans Petter Selasky #define ARCH_KMALLOC_MINALIGN \
8671ee95ddSHans Petter Selasky __alignof(unsigned long long)
8771ee95ddSHans Petter Selasky
880b1244bdSVladimir Kondratyev /* drm-kmod 5.4 compat */
890b1244bdSVladimir Kondratyev #define kfree_async(ptr) kfree(ptr);
90ec25b6faSVladimir Kondratyev
911b4e08b4SJean-Sébastien Pédron #define ZERO_SIZE_PTR ((void *)16)
921b4e08b4SJean-Sébastien Pédron #define ZERO_OR_NULL_PTR(x) ((x) == NULL || (x) == ZERO_SIZE_PTR)
93208d02fdSJean-Sébastien Pédron
9414c5024dSHans Petter Selasky static inline gfp_t
linux_check_m_flags(gfp_t flags)9514c5024dSHans Petter Selasky linux_check_m_flags(gfp_t flags)
968d59ecb2SHans Petter Selasky {
9714c5024dSHans Petter Selasky const gfp_t m = M_NOWAIT | M_WAITOK;
988d59ecb2SHans Petter Selasky
9914c5024dSHans Petter Selasky /* make sure either M_NOWAIT or M_WAITOK is set */
10014c5024dSHans Petter Selasky if ((flags & m) == 0)
10114c5024dSHans Petter Selasky flags |= M_NOWAIT;
10214c5024dSHans Petter Selasky else if ((flags & m) == m)
10314c5024dSHans Petter Selasky flags &= ~M_WAITOK;
1048d59ecb2SHans Petter Selasky
10514c5024dSHans Petter Selasky /* mask away LinuxKPI specific flags */
10614c5024dSHans Petter Selasky return (flags & GFP_NATIVE_MASK);
1078d59ecb2SHans Petter Selasky }
1088d59ecb2SHans Petter Selasky
1098d59ecb2SHans Petter Selasky static inline void *
__kmalloc(size_t size,gfp_t flags)110c0b8047bSVladimir Kondratyev __kmalloc(size_t size, gfp_t flags)
1118d59ecb2SHans Petter Selasky {
1120b1244bdSVladimir Kondratyev return (malloc(MAX(size, sizeof(struct llist_node)), M_KMALLOC,
1130b1244bdSVladimir Kondratyev linux_check_m_flags(flags)));
11414c5024dSHans Petter Selasky }
11514c5024dSHans Petter Selasky
11614c5024dSHans Petter Selasky static inline void *
kmalloc_node(size_t size,gfp_t flags,int node)117ebe5cf35SHans Petter Selasky kmalloc_node(size_t size, gfp_t flags, int node)
118ebe5cf35SHans Petter Selasky {
119ebe5cf35SHans Petter Selasky return (malloc_domainset(size, M_KMALLOC,
120ebe5cf35SHans Petter Selasky linux_get_vm_domain_set(node), linux_check_m_flags(flags)));
121ebe5cf35SHans Petter Selasky }
122ebe5cf35SHans Petter Selasky
123ebe5cf35SHans Petter Selasky static inline void *
kcalloc(size_t n,size_t size,gfp_t flags)124e70c77caSKristof Provost kcalloc(size_t n, size_t size, gfp_t flags)
125e70c77caSKristof Provost {
126e70c77caSKristof Provost flags |= __GFP_ZERO;
127e70c77caSKristof Provost return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags)));
128e70c77caSKristof Provost }
129e70c77caSKristof Provost
130e70c77caSKristof Provost static inline void *
kcalloc_node(size_t n,size_t size,gfp_t flags,int node)131ebe5cf35SHans Petter Selasky kcalloc_node(size_t n, size_t size, gfp_t flags, int node)
132ebe5cf35SHans Petter Selasky {
133ebe5cf35SHans Petter Selasky flags |= __GFP_ZERO;
134ebe5cf35SHans Petter Selasky return (mallocarray_domainset(n, size, M_KMALLOC,
135ebe5cf35SHans Petter Selasky linux_get_vm_domain_set(node), linux_check_m_flags(flags)));
136ebe5cf35SHans Petter Selasky }
137ebe5cf35SHans Petter Selasky
138ebe5cf35SHans Petter Selasky static inline void *
__vmalloc(size_t size,gfp_t flags,int other)13914c5024dSHans Petter Selasky __vmalloc(size_t size, gfp_t flags, int other)
14014c5024dSHans Petter Selasky {
14114c5024dSHans Petter Selasky return (malloc(size, M_KMALLOC, linux_check_m_flags(flags)));
14214c5024dSHans Petter Selasky }
14314c5024dSHans Petter Selasky
14414c5024dSHans Petter Selasky static inline void *
__vmalloc_node(size_t size,gfp_t flags,int node)145ebe5cf35SHans Petter Selasky __vmalloc_node(size_t size, gfp_t flags, int node)
146ebe5cf35SHans Petter Selasky {
147ebe5cf35SHans Petter Selasky return (malloc_domainset(size, M_KMALLOC,
148ebe5cf35SHans Petter Selasky linux_get_vm_domain_set(node), linux_check_m_flags(flags)));
149ebe5cf35SHans Petter Selasky }
150ebe5cf35SHans Petter Selasky
151ebe5cf35SHans Petter Selasky static inline void *
vmalloc_32(size_t size)1520a475c59SHans Petter Selasky vmalloc_32(size_t size)
1530a475c59SHans Petter Selasky {
1540a475c59SHans Petter Selasky return (contigmalloc(size, M_KMALLOC, M_WAITOK, 0, UINT_MAX, 1, 1));
1550a475c59SHans Petter Selasky }
1560a475c59SHans Petter Selasky
1570a475c59SHans Petter Selasky static inline void *
kmalloc_array(size_t n,size_t size,gfp_t flags)15814c5024dSHans Petter Selasky kmalloc_array(size_t n, size_t size, gfp_t flags)
15914c5024dSHans Petter Selasky {
160ed595433SPedro F. Giffuni return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags)));
16114c5024dSHans Petter Selasky }
16214c5024dSHans Petter Selasky
16314c5024dSHans Petter Selasky static inline void *
kmalloc_array_node(size_t n,size_t size,gfp_t flags,int node)164ebe5cf35SHans Petter Selasky kmalloc_array_node(size_t n, size_t size, gfp_t flags, int node)
165ebe5cf35SHans Petter Selasky {
166ebe5cf35SHans Petter Selasky return (mallocarray_domainset(n, size, M_KMALLOC,
167ebe5cf35SHans Petter Selasky linux_get_vm_domain_set(node), linux_check_m_flags(flags)));
168ebe5cf35SHans Petter Selasky }
169ebe5cf35SHans Petter Selasky
170ebe5cf35SHans Petter Selasky static inline void *
kvmalloc_array(size_t n,size_t size,gfp_t flags)171f1aa567bSHans Petter Selasky kvmalloc_array(size_t n, size_t size, gfp_t flags)
172f1aa567bSHans Petter Selasky {
173f1aa567bSHans Petter Selasky return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags)));
174f1aa567bSHans Petter Selasky }
175f1aa567bSHans Petter Selasky
176f1aa567bSHans Petter Selasky static inline void *
krealloc(void * ptr,size_t size,gfp_t flags)17714c5024dSHans Petter Selasky krealloc(void *ptr, size_t size, gfp_t flags)
17814c5024dSHans Petter Selasky {
17914c5024dSHans Petter Selasky return (realloc(ptr, size, M_KMALLOC, linux_check_m_flags(flags)));
1808d59ecb2SHans Petter Selasky }
1818d59ecb2SHans Petter Selasky
1821ad6b2b1SJean-Sébastien Pédron static inline void *
krealloc_array(void * ptr,size_t n,size_t size,gfp_t flags)1831ad6b2b1SJean-Sébastien Pédron krealloc_array(void *ptr, size_t n, size_t size, gfp_t flags)
1841ad6b2b1SJean-Sébastien Pédron {
1851ad6b2b1SJean-Sébastien Pédron if (WOULD_OVERFLOW(n, size)) {
1861ad6b2b1SJean-Sébastien Pédron return NULL;
1871ad6b2b1SJean-Sébastien Pédron }
1881ad6b2b1SJean-Sébastien Pédron
1891ad6b2b1SJean-Sébastien Pédron return (realloc(ptr, n * size, M_KMALLOC, linux_check_m_flags(flags)));
1901ad6b2b1SJean-Sébastien Pédron }
1911ad6b2b1SJean-Sébastien Pédron
1920b1244bdSVladimir Kondratyev extern void linux_kfree_async(void *);
1930b1244bdSVladimir Kondratyev
1948d59ecb2SHans Petter Selasky static inline void
kfree(const void * ptr)19514c5024dSHans Petter Selasky kfree(const void *ptr)
1968d59ecb2SHans Petter Selasky {
1971b4e08b4SJean-Sébastien Pédron if (ZERO_OR_NULL_PTR(ptr))
1981b4e08b4SJean-Sébastien Pédron return;
1991b4e08b4SJean-Sébastien Pédron
2000b1244bdSVladimir Kondratyev if (curthread->td_critnest != 0)
2010b1244bdSVladimir Kondratyev linux_kfree_async(__DECONST(void *, ptr));
2020b1244bdSVladimir Kondratyev else
20314c5024dSHans Petter Selasky free(__DECONST(void *, ptr), M_KMALLOC);
20414c5024dSHans Petter Selasky }
20514c5024dSHans Petter Selasky
206abcac97fSBjoern A. Zeeb static __inline void
kfree_sensitive(const void * ptr)207abcac97fSBjoern A. Zeeb kfree_sensitive(const void *ptr)
208abcac97fSBjoern A. Zeeb {
2091b4e08b4SJean-Sébastien Pédron if (ZERO_OR_NULL_PTR(ptr))
2101b4e08b4SJean-Sébastien Pédron return;
2111b4e08b4SJean-Sébastien Pédron
212abcac97fSBjoern A. Zeeb zfree(__DECONST(void *, ptr), M_KMALLOC);
213abcac97fSBjoern A. Zeeb }
214abcac97fSBjoern A. Zeeb
2155d4bf057SVladimir Kondratyev static inline size_t
ksize(const void * ptr)2165d4bf057SVladimir Kondratyev ksize(const void *ptr)
2175d4bf057SVladimir Kondratyev {
2185d4bf057SVladimir Kondratyev return (malloc_usable_size(ptr));
2195d4bf057SVladimir Kondratyev }
2205d4bf057SVladimir Kondratyev
221c0b8047bSVladimir Kondratyev extern void *lkpi_kmalloc(size_t size, gfp_t flags);
22214c5024dSHans Petter Selasky extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name,
22314c5024dSHans Petter Selasky size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor);
224a2b83b59SVladimir Kondratyev extern void *lkpi_kmem_cache_alloc(struct linux_kmem_cache *, gfp_t);
225a2b83b59SVladimir Kondratyev extern void *lkpi_kmem_cache_zalloc(struct linux_kmem_cache *, gfp_t);
226a2b83b59SVladimir Kondratyev extern void lkpi_kmem_cache_free(struct linux_kmem_cache *, void *);
22714c5024dSHans Petter Selasky extern void linux_kmem_cache_destroy(struct linux_kmem_cache *);
2288d59ecb2SHans Petter Selasky
229307f78f3SVladimir Kondratyev #endif /* _LINUXKPI_LINUX_SLAB_H_ */
230