1 /* 2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 3 * Copyright (C) 2007 The Regents of the University of California. 4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 6 * UCRL-CODE-235197 7 * 8 * This file is part of the SPL, Solaris Porting Layer. 9 * 10 * The SPL is free software; you can redistribute it and/or modify it 11 * under the terms of the GNU General Public License as published by the 12 * Free Software Foundation; either version 2 of the License, or (at your 13 * option) any later version. 14 * 15 * The SPL is distributed in the hope that it will be useful, but WITHOUT 16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 18 * for more details. 19 * 20 * You should have received a copy of the GNU General Public License along 21 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 22 */ 23 24 #ifndef _SPL_KMEM_H 25 #define _SPL_KMEM_H 26 27 #include <sys/debug.h> 28 #include <linux/slab.h> 29 #include <linux/sched.h> 30 #include <linux/mm.h> 31 #include <linux/vmalloc.h> 32 33 extern int kmem_debugging(void); 34 __attribute__((format(printf, 1, 0))) 35 extern char *kmem_vasprintf(const char *fmt, va_list ap); 36 __attribute__((format(printf, 1, 2))) 37 extern char *kmem_asprintf(const char *fmt, ...); 38 extern char *kmem_strdup(const char *str); 39 extern void kmem_strfree(char *str); 40 41 #define kmem_scnprintf scnprintf 42 43 #define POINTER_IS_VALID(p) (!((uintptr_t)(p) & 0x3)) 44 #define POINTER_INVALIDATE(pp) (*(pp) = (void *)((uintptr_t)(*(pp)) | 0x1)) 45 46 /* 47 * Memory allocation interfaces 48 */ 49 #define KM_SLEEP 0x0000 /* can block for memory; success guaranteed */ 50 #define KM_NOSLEEP 0x0001 /* cannot block for memory; may fail */ 51 #define KM_PUSHPAGE 0x0004 /* can block for memory; may use reserve */ 52 #define KM_ZERO 0x1000 /* zero the allocation */ 53 #define KM_VMEM 0x2000 /* caller is vmem_* wrapper */ 54 55 #define KM_PUBLIC_MASK (KM_SLEEP | KM_NOSLEEP | KM_PUSHPAGE) 56 57 static int spl_fstrans_check(void); 58 void *spl_kvmalloc(size_t size, gfp_t flags); 59 60 /* 61 * Convert a KM_* flags mask to its Linux GFP_* counterpart. The conversion 62 * function is context aware which means that KM_SLEEP allocations can be 63 * safely used in syncing contexts which have set PF_FSTRANS. 64 */ 65 static inline gfp_t 66 kmem_flags_convert(int flags) 67 { 68 gfp_t lflags = __GFP_NOWARN | __GFP_COMP; 69 70 if (flags & KM_NOSLEEP) { 71 lflags |= GFP_ATOMIC | __GFP_NORETRY; 72 } else { 73 lflags |= GFP_KERNEL; 74 if (spl_fstrans_check()) 75 lflags &= ~(__GFP_IO|__GFP_FS); 76 } 77 78 if (flags & KM_PUSHPAGE) 79 lflags |= __GFP_HIGH; 80 81 if (flags & KM_ZERO) 82 lflags |= __GFP_ZERO; 83 84 return (lflags); 85 } 86 87 typedef struct { 88 struct task_struct *fstrans_thread; 89 unsigned int saved_flags; 90 } fstrans_cookie_t; 91 92 /* 93 * Introduced in Linux 3.9, however this cannot be solely relied on before 94 * Linux 3.18 as it doesn't turn off __GFP_FS as it should. 95 */ 96 #ifdef PF_MEMALLOC_NOIO 97 #define __SPL_PF_MEMALLOC_NOIO (PF_MEMALLOC_NOIO) 98 #else 99 #define __SPL_PF_MEMALLOC_NOIO (0) 100 #endif 101 102 /* 103 * PF_FSTRANS is removed from Linux 4.12 104 */ 105 #ifdef PF_FSTRANS 106 #define __SPL_PF_FSTRANS (PF_FSTRANS) 107 #else 108 #define __SPL_PF_FSTRANS (0) 109 #endif 110 111 #define SPL_FSTRANS (__SPL_PF_FSTRANS|__SPL_PF_MEMALLOC_NOIO) 112 113 static inline fstrans_cookie_t 114 spl_fstrans_mark(void) 115 { 116 fstrans_cookie_t cookie; 117 118 BUILD_BUG_ON(SPL_FSTRANS == 0); 119 120 cookie.fstrans_thread = current; 121 cookie.saved_flags = current->flags & SPL_FSTRANS; 122 current->flags |= SPL_FSTRANS; 123 124 return (cookie); 125 } 126 127 static inline void 128 spl_fstrans_unmark(fstrans_cookie_t cookie) 129 { 130 ASSERT3P(cookie.fstrans_thread, ==, current); 131 ASSERT((current->flags & SPL_FSTRANS) == SPL_FSTRANS); 132 133 current->flags &= ~SPL_FSTRANS; 134 current->flags |= cookie.saved_flags; 135 } 136 137 static inline int 138 spl_fstrans_check(void) 139 { 140 return (current->flags & SPL_FSTRANS); 141 } 142 143 /* 144 * specifically used to check PF_FSTRANS flag, cannot be relied on for 145 * checking spl_fstrans_mark(). 146 */ 147 static inline int 148 __spl_pf_fstrans_check(void) 149 { 150 return (current->flags & __SPL_PF_FSTRANS); 151 } 152 153 /* 154 * Kernel compatibility for GFP flags 155 */ 156 /* < 4.13 */ 157 #ifndef __GFP_RETRY_MAYFAIL 158 #define __GFP_RETRY_MAYFAIL __GFP_REPEAT 159 #endif 160 /* < 4.4 */ 161 #ifndef __GFP_RECLAIM 162 #define __GFP_RECLAIM __GFP_WAIT 163 #endif 164 165 #ifdef HAVE_ATOMIC64_T 166 #define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used) 167 #define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used) 168 #define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used) 169 #define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size) 170 extern atomic64_t kmem_alloc_used; 171 extern unsigned long long kmem_alloc_max; 172 #else /* HAVE_ATOMIC64_T */ 173 #define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used) 174 #define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used) 175 #define kmem_alloc_used_read() atomic_read(&kmem_alloc_used) 176 #define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size) 177 extern atomic_t kmem_alloc_used; 178 extern unsigned long long kmem_alloc_max; 179 #endif /* HAVE_ATOMIC64_T */ 180 181 extern unsigned int spl_kmem_alloc_warn; 182 extern unsigned int spl_kmem_alloc_max; 183 184 #define kmem_alloc(sz, fl) spl_kmem_alloc((sz), (fl), __func__, __LINE__) 185 #define kmem_zalloc(sz, fl) spl_kmem_zalloc((sz), (fl), __func__, __LINE__) 186 #define kmem_free(ptr, sz) spl_kmem_free((ptr), (sz)) 187 #define kmem_cache_reap_active spl_kmem_cache_reap_active 188 189 __attribute__((malloc, alloc_size(1))) 190 extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line); 191 __attribute__((malloc, alloc_size(1))) 192 extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line); 193 extern void spl_kmem_free(const void *ptr, size_t sz); 194 195 /* 196 * 5.8 API change, pgprot_t argument removed. 197 */ 198 #ifdef HAVE_VMALLOC_PAGE_KERNEL 199 #define spl_vmalloc(size, flags) __vmalloc(size, flags, PAGE_KERNEL) 200 #else 201 #define spl_vmalloc(size, flags) __vmalloc(size, flags) 202 #endif 203 204 /* 205 * The following functions are only available for internal use. 206 */ 207 extern void *spl_kmem_alloc_impl(size_t size, int flags, int node); 208 extern void *spl_kmem_alloc_debug(size_t size, int flags, int node); 209 extern void *spl_kmem_alloc_track(size_t size, int flags, 210 const char *func, int line, int node); 211 extern void spl_kmem_free_impl(const void *buf, size_t size); 212 extern void spl_kmem_free_debug(const void *buf, size_t size); 213 extern void spl_kmem_free_track(const void *buf, size_t size); 214 215 extern int spl_kmem_init(void); 216 extern void spl_kmem_fini(void); 217 extern int spl_kmem_cache_reap_active(void); 218 219 #endif /* _SPL_KMEM_H */ 220