1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * Copyright 2013 Red Hat Inc.
4  *
5  * Authors: Jérôme Glisse <jglisse@redhat.com>
6  *
7  * See Documentation/vm/hmm.rst for reasons and overview of what HMM is.
8  */
9 #ifndef LINUX_HMM_H
10 #define LINUX_HMM_H
11 
12 #include <linux/kconfig.h>
13 #include <linux/pgtable.h>
14 
15 #include <linux/device.h>
16 #include <linux/migrate.h>
17 #include <linux/memremap.h>
18 #include <linux/completion.h>
19 #include <linux/mmu_notifier.h>
20 
21 /*
22  * On output:
23  * 0             - The page is faultable and a future call with
24  *                 HMM_PFN_REQ_FAULT could succeed.
25  * HMM_PFN_VALID - the pfn field points to a valid PFN. This PFN is at
26  *                 least readable. If dev_private_owner is !NULL then this could
27  *                 point at a DEVICE_PRIVATE page.
28  * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
29  * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
30  *                 fail. ie poisoned memory, special pages, no vma, etc
31  *
32  * On input:
33  * 0                 - Return the current state of the page, do not fault it.
34  * HMM_PFN_REQ_FAULT - The output must have HMM_PFN_VALID or hmm_range_fault()
35  *                     will fail
36  * HMM_PFN_REQ_WRITE - The output must have HMM_PFN_WRITE or hmm_range_fault()
37  *                     will fail. Must be combined with HMM_PFN_REQ_FAULT.
38  */
39 enum hmm_pfn_flags {
40 	/* Output fields and flags */
41 	HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
42 	HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
43 	HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
44 	HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
45 
46 	/* Input flags */
47 	HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
48 	HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
49 
50 	HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
51 };
52 
53 /*
54  * hmm_pfn_to_page() - return struct page pointed to by a device entry
55  *
56  * This must be called under the caller 'user_lock' after a successful
57  * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
58  * already.
59  */
hmm_pfn_to_page(unsigned long hmm_pfn)60 static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn)
61 {
62 	return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
63 }
64 
65 /*
66  * hmm_pfn_to_map_order() - return the CPU mapping size order
67  *
68  * This is optionally useful to optimize processing of the pfn result
69  * array. It indicates that the page starts at the order aligned VA and is
70  * 1<<order bytes long.  Every pfn within an high order page will have the
71  * same pfn flags, both access protections and the map_order.  The caller must
72  * be careful with edge cases as the start and end VA of the given page may
73  * extend past the range used with hmm_range_fault().
74  *
75  * This must be called under the caller 'user_lock' after a successful
76  * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
77  * already.
78  */
hmm_pfn_to_map_order(unsigned long hmm_pfn)79 static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
80 {
81 	return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
82 }
83 
84 /*
85  * struct hmm_range - track invalidation lock on virtual address range
86  *
87  * @notifier: a mmu_interval_notifier that includes the start/end
88  * @notifier_seq: result of mmu_interval_read_begin()
89  * @start: range virtual start address (inclusive)
90  * @end: range virtual end address (exclusive)
91  * @hmm_pfns: array of pfns (big enough for the range)
92  * @default_flags: default flags for the range (write, read, ... see hmm doc)
93  * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
94  * @dev_private_owner: owner of device private pages
95  */
96 struct hmm_range {
97 	struct mmu_interval_notifier *notifier;
98 	unsigned long		notifier_seq;
99 	unsigned long		start;
100 	unsigned long		end;
101 	unsigned long		*hmm_pfns;
102 	unsigned long		default_flags;
103 	unsigned long		pfn_flags_mask;
104 	void			*dev_private_owner;
105 };
106 
107 /*
108  * Please see Documentation/vm/hmm.rst for how to use the range API.
109  */
110 int hmm_range_fault(struct hmm_range *range);
111 
112 /*
113  * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
114  *
115  * When waiting for mmu notifiers we need some kind of time out otherwise we
116  * could potentialy wait for ever, 1000ms ie 1s sounds like a long time to
117  * wait already.
118  */
119 #define HMM_RANGE_DEFAULT_TIMEOUT 1000
120 
121 #endif /* LINUX_HMM_H */
122