1 /*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
6 * Copyright (c) 2015 François Tigeot
7 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice unmodified, this list of conditions, and the following
15 * disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31 #ifndef _LINUXKPI_LINUX_MM_H_
32 #define _LINUXKPI_LINUX_MM_H_
33
34 #include <linux/spinlock.h>
35 #include <linux/gfp.h>
36 #include <linux/kernel.h>
37 #include <linux/mm_types.h>
38 #include <linux/pfn.h>
39 #include <linux/list.h>
40 #include <linux/mmap_lock.h>
41 #include <linux/overflow.h>
42 #include <linux/shrinker.h>
43 #include <linux/page.h>
44
45 #include <asm/pgtable.h>
46
47 #define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
48
49 /*
50 * Make sure our LinuxKPI defined virtual memory flags don't conflict
51 * with the ones defined by FreeBSD:
52 */
53 CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0);
54
55 #define VM_READ VM_PROT_READ
56 #define VM_WRITE VM_PROT_WRITE
57 #define VM_EXEC VM_PROT_EXECUTE
58
59 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
60
61 #define VM_PFNINTERNAL (1 << 8) /* FreeBSD private flag to vm_insert_pfn() */
62 #define VM_MIXEDMAP (1 << 9)
63 #define VM_NORESERVE (1 << 10)
64 #define VM_PFNMAP (1 << 11)
65 #define VM_IO (1 << 12)
66 #define VM_MAYWRITE (1 << 13)
67 #define VM_DONTCOPY (1 << 14)
68 #define VM_DONTEXPAND (1 << 15)
69 #define VM_DONTDUMP (1 << 16)
70 #define VM_SHARED (1 << 17)
71
72 #define VMA_MAX_PREFAULT_RECORD 1
73
74 #define FOLL_WRITE (1 << 0)
75 #define FOLL_FORCE (1 << 1)
76
77 #define VM_FAULT_OOM (1 << 0)
78 #define VM_FAULT_SIGBUS (1 << 1)
79 #define VM_FAULT_MAJOR (1 << 2)
80 #define VM_FAULT_WRITE (1 << 3)
81 #define VM_FAULT_HWPOISON (1 << 4)
82 #define VM_FAULT_HWPOISON_LARGE (1 << 5)
83 #define VM_FAULT_SIGSEGV (1 << 6)
84 #define VM_FAULT_NOPAGE (1 << 7)
85 #define VM_FAULT_LOCKED (1 << 8)
86 #define VM_FAULT_RETRY (1 << 9)
87 #define VM_FAULT_FALLBACK (1 << 10)
88
89 #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
90 VM_FAULT_HWPOISON |VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)
91
92 #define FAULT_FLAG_WRITE (1 << 0)
93 #define FAULT_FLAG_MKWRITE (1 << 1)
94 #define FAULT_FLAG_ALLOW_RETRY (1 << 2)
95 #define FAULT_FLAG_RETRY_NOWAIT (1 << 3)
96 #define FAULT_FLAG_KILLABLE (1 << 4)
97 #define FAULT_FLAG_TRIED (1 << 5)
98 #define FAULT_FLAG_USER (1 << 6)
99 #define FAULT_FLAG_REMOTE (1 << 7)
100 #define FAULT_FLAG_INSTRUCTION (1 << 8)
101
102 #define fault_flag_allow_retry_first(flags) \
103 (((flags) & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) == FAULT_FLAG_ALLOW_RETRY)
104
105 typedef int (*pte_fn_t)(linux_pte_t *, unsigned long addr, void *data);
106
107 struct vm_area_struct {
108 vm_offset_t vm_start;
109 vm_offset_t vm_end;
110 vm_offset_t vm_pgoff;
111 pgprot_t vm_page_prot;
112 unsigned long vm_flags;
113 struct mm_struct *vm_mm;
114 void *vm_private_data;
115 const struct vm_operations_struct *vm_ops;
116 struct linux_file *vm_file;
117
118 /* internal operation */
119 vm_paddr_t vm_pfn; /* PFN for memory map */
120 vm_size_t vm_len; /* length for memory map */
121 vm_pindex_t vm_pfn_first;
122 int vm_pfn_count;
123 int *vm_pfn_pcount;
124 vm_object_t vm_obj;
125 vm_map_t vm_cached_map;
126 TAILQ_ENTRY(vm_area_struct) vm_entry;
127 };
128
129 struct vm_fault {
130 unsigned int flags;
131 pgoff_t pgoff;
132 union {
133 /* user-space address */
134 void *virtual_address; /* < 4.11 */
135 unsigned long address; /* >= 4.11 */
136 };
137 struct page *page;
138 struct vm_area_struct *vma;
139 };
140
141 struct vm_operations_struct {
142 void (*open) (struct vm_area_struct *);
143 void (*close) (struct vm_area_struct *);
144 int (*fault) (struct vm_fault *);
145 int (*access) (struct vm_area_struct *, unsigned long, void *, int, int);
146 };
147
148 struct sysinfo {
149 uint64_t totalram; /* Total usable main memory size */
150 uint64_t freeram; /* Available memory size */
151 uint64_t totalhigh; /* Total high memory size */
152 uint64_t freehigh; /* Available high memory size */
153 uint32_t mem_unit; /* Memory unit size in bytes */
154 };
155
156 static inline struct page *
virt_to_head_page(const void * p)157 virt_to_head_page(const void *p)
158 {
159
160 return (virt_to_page(p));
161 }
162
163 /*
164 * Compute log2 of the power of two rounded up count of pages
165 * needed for size bytes.
166 */
167 static inline int
get_order(unsigned long size)168 get_order(unsigned long size)
169 {
170 int order;
171
172 size = (size - 1) >> PAGE_SHIFT;
173 order = 0;
174 while (size) {
175 order++;
176 size >>= 1;
177 }
178 return (order);
179 }
180
181 /*
182 * Resolve a page into a virtual address:
183 *
184 * NOTE: This function only works for pages allocated by the kernel.
185 */
186 void *linux_page_address(struct page *);
187 #define page_address(page) linux_page_address(page)
188
189 static inline void *
lowmem_page_address(struct page * page)190 lowmem_page_address(struct page *page)
191 {
192 return (page_address(page));
193 }
194
195 /*
196 * This only works via memory map operations.
197 */
198 static inline int
io_remap_pfn_range(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,vm_memattr_t prot)199 io_remap_pfn_range(struct vm_area_struct *vma,
200 unsigned long addr, unsigned long pfn, unsigned long size,
201 vm_memattr_t prot)
202 {
203 vma->vm_page_prot = prot;
204 vma->vm_pfn = pfn;
205 vma->vm_len = size;
206
207 return (0);
208 }
209
210 vm_fault_t
211 lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
212 unsigned long pfn, pgprot_t prot);
213
214 static inline vm_fault_t
vmf_insert_pfn_prot(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,pgprot_t prot)215 vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
216 unsigned long pfn, pgprot_t prot)
217 {
218 vm_fault_t ret;
219
220 VM_OBJECT_WLOCK(vma->vm_obj);
221 ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot);
222 VM_OBJECT_WUNLOCK(vma->vm_obj);
223
224 return (ret);
225 }
226 #define vmf_insert_pfn_prot(...) \
227 _Static_assert(false, \
228 "This function is always called in a loop. Consider using the locked version")
229
230 static inline int
apply_to_page_range(struct mm_struct * mm,unsigned long address,unsigned long size,pte_fn_t fn,void * data)231 apply_to_page_range(struct mm_struct *mm, unsigned long address,
232 unsigned long size, pte_fn_t fn, void *data)
233 {
234 return (-ENOTSUP);
235 }
236
237 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
238 unsigned long size);
239
240 int lkpi_remap_pfn_range(struct vm_area_struct *vma,
241 unsigned long start_addr, unsigned long start_pfn, unsigned long size,
242 pgprot_t prot);
243
244 static inline int
remap_pfn_range(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)245 remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
246 unsigned long pfn, unsigned long size, pgprot_t prot)
247 {
248 return (lkpi_remap_pfn_range(vma, addr, pfn, size, prot));
249 }
250
251 static inline unsigned long
vma_pages(struct vm_area_struct * vma)252 vma_pages(struct vm_area_struct *vma)
253 {
254 return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
255 }
256
257 #define offset_in_page(off) ((unsigned long)(off) & (PAGE_SIZE - 1))
258
259 static inline void
set_page_dirty(struct page * page)260 set_page_dirty(struct page *page)
261 {
262 vm_page_dirty(page);
263 }
264
265 static inline void
mark_page_accessed(struct page * page)266 mark_page_accessed(struct page *page)
267 {
268 vm_page_reference(page);
269 }
270
271 static inline void
get_page(struct page * page)272 get_page(struct page *page)
273 {
274 vm_page_wire(page);
275 }
276
277 extern long
278 get_user_pages(unsigned long start, unsigned long nr_pages,
279 unsigned int gup_flags, struct page **,
280 struct vm_area_struct **);
281
282 static inline long
pin_user_pages(unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas)283 pin_user_pages(unsigned long start, unsigned long nr_pages,
284 unsigned int gup_flags, struct page **pages,
285 struct vm_area_struct **vmas)
286 {
287 return get_user_pages(start, nr_pages, gup_flags, pages, vmas);
288 }
289
290 extern int
291 __get_user_pages_fast(unsigned long start, int nr_pages, int write,
292 struct page **);
293
294 static inline int
pin_user_pages_fast(unsigned long start,int nr_pages,unsigned int gup_flags,struct page ** pages)295 pin_user_pages_fast(unsigned long start, int nr_pages,
296 unsigned int gup_flags, struct page **pages)
297 {
298 return __get_user_pages_fast(
299 start, nr_pages, !!(gup_flags & FOLL_WRITE), pages);
300 }
301
302 extern long
303 get_user_pages_remote(struct task_struct *, struct mm_struct *,
304 unsigned long start, unsigned long nr_pages,
305 unsigned int gup_flags, struct page **,
306 struct vm_area_struct **);
307
308 static inline long
pin_user_pages_remote(struct task_struct * task,struct mm_struct * mm,unsigned long start,unsigned long nr_pages,unsigned int gup_flags,struct page ** pages,struct vm_area_struct ** vmas)309 pin_user_pages_remote(struct task_struct *task, struct mm_struct *mm,
310 unsigned long start, unsigned long nr_pages,
311 unsigned int gup_flags, struct page **pages,
312 struct vm_area_struct **vmas)
313 {
314 return get_user_pages_remote(
315 task, mm, start, nr_pages, gup_flags, pages, vmas);
316 }
317
318 static inline void
put_page(struct page * page)319 put_page(struct page *page)
320 {
321 vm_page_unwire(page, PQ_ACTIVE);
322 }
323
324 #define unpin_user_page(page) put_page(page)
325 #define unpin_user_pages(pages, npages) release_pages(pages, npages)
326
327 #define copy_highpage(to, from) pmap_copy_page(from, to)
328
329 static inline pgprot_t
vm_get_page_prot(unsigned long vm_flags)330 vm_get_page_prot(unsigned long vm_flags)
331 {
332 return (vm_flags & VM_PROT_ALL);
333 }
334
335 static inline void
vm_flags_set(struct vm_area_struct * vma,unsigned long flags)336 vm_flags_set(struct vm_area_struct *vma, unsigned long flags)
337 {
338 vma->vm_flags |= flags;
339 }
340
341 static inline void
vm_flags_clear(struct vm_area_struct * vma,unsigned long flags)342 vm_flags_clear(struct vm_area_struct *vma, unsigned long flags)
343 {
344 vma->vm_flags &= ~flags;
345 }
346
347 static inline struct page *
vmalloc_to_page(const void * addr)348 vmalloc_to_page(const void *addr)
349 {
350 vm_paddr_t paddr;
351
352 paddr = pmap_kextract((vm_offset_t)addr);
353 return (PHYS_TO_VM_PAGE(paddr));
354 }
355
356 static inline int
trylock_page(struct page * page)357 trylock_page(struct page *page)
358 {
359 return (vm_page_trylock(page));
360 }
361
362 static inline void
unlock_page(struct page * page)363 unlock_page(struct page *page)
364 {
365
366 vm_page_unlock(page);
367 }
368
369 extern int is_vmalloc_addr(const void *addr);
370 void si_meminfo(struct sysinfo *si);
371
372 static inline unsigned long
totalram_pages(void)373 totalram_pages(void)
374 {
375 return ((unsigned long)physmem);
376 }
377
378 #define unmap_mapping_range(...) lkpi_unmap_mapping_range(__VA_ARGS__)
379 void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused,
380 loff_t const holelen, int even_cows __unused);
381
382 #define PAGE_ALIGNED(p) __is_aligned(p, PAGE_SIZE)
383
384 void vma_set_file(struct vm_area_struct *vma, struct linux_file *file);
385
386 static inline void
might_alloc(gfp_t gfp_mask __unused)387 might_alloc(gfp_t gfp_mask __unused)
388 {
389 }
390
391 #define is_cow_mapping(flags) (false)
392
393 static inline bool
want_init_on_free(void)394 want_init_on_free(void)
395 {
396 return (false);
397 }
398
399 #endif /* _LINUXKPI_LINUX_MM_H_ */
400