1 /*******************************************************************************
2 Copyright (c) 2015-2019 NVIDIA Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to
6 deal in the Software without restriction, including without limitation the
7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8 sell copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 DEALINGS IN THE SOFTWARE.
21
22 *******************************************************************************/
23
24 #ifndef __UVM_API_H__
25 #define __UVM_API_H__
26
27 #include "uvm_types.h"
28 #include "uvm_common.h"
29 #include "uvm_ioctl.h"
30 #include "uvm_linux.h"
31 #include "uvm_lock.h"
32 #include "uvm_thread_context.h"
33 #include "uvm_kvmalloc.h"
34 #include "uvm_va_space.h"
35 #include "nv_uvm_types.h"
36
37 // This weird number comes from UVM_PREVENT_MIGRATION_RANGE_GROUPS_PARAMS. That
38 // ioctl is called frequently so we don't want to allocate a copy every time.
39 // It's a little over 256 bytes in size.
40 #define UVM_MAX_IOCTL_PARAM_STACK_SIZE 288
41
42 // The UVM_ROUTE_CMD_* macros are only intended for use in the ioctl routines
43
44 // If the BUILD_BUG_ON fires, use __UVM_ROUTE_CMD_ALLOC instead.
45 #define __UVM_ROUTE_CMD_STACK(cmd, params_type, function_name, do_init_check) \
46 case cmd: \
47 { \
48 params_type params; \
49 BUILD_BUG_ON(sizeof(params) > UVM_MAX_IOCTL_PARAM_STACK_SIZE); \
50 if (nv_copy_from_user(¶ms, (void __user*)arg, sizeof(params))) \
51 return -EFAULT; \
52 \
53 params.rmStatus = uvm_global_get_status(); \
54 if (params.rmStatus == NV_OK) { \
55 if (do_init_check) { \
56 if (!uvm_fd_va_space(filp)) \
57 params.rmStatus = NV_ERR_ILLEGAL_ACTION; \
58 } \
59 if (likely(params.rmStatus == NV_OK)) \
60 params.rmStatus = function_name(¶ms, filp); \
61 } \
62 \
63 if (nv_copy_to_user((void __user*)arg, ¶ms, sizeof(params))) \
64 return -EFAULT; \
65 \
66 return 0; \
67 }
68
69 // We need to concatenate cmd##_PARAMS here to avoid the preprocessor's argument
70 // prescan. Attempting concatenation in the lower-level macro will fail because
71 // it will have been expanded to a literal by then.
72 #define UVM_ROUTE_CMD_STACK_NO_INIT_CHECK(cmd, function_name) \
73 __UVM_ROUTE_CMD_STACK(cmd, cmd##_PARAMS, function_name, false)
74
75 #define UVM_ROUTE_CMD_STACK_INIT_CHECK(cmd, function_name) \
76 __UVM_ROUTE_CMD_STACK(cmd, cmd##_PARAMS, function_name, true)
77
78 // If the BUILD_BUG_ON fires, use __UVM_ROUTE_CMD_STACK instead
79 #define __UVM_ROUTE_CMD_ALLOC(cmd, params_type, function_name, do_init_check) \
80 case cmd: \
81 { \
82 int ret = 0; \
83 params_type *params = uvm_kvmalloc(sizeof(*params)); \
84 if (!params) \
85 return -ENOMEM; \
86 BUILD_BUG_ON(sizeof(*params) <= UVM_MAX_IOCTL_PARAM_STACK_SIZE); \
87 if (nv_copy_from_user(params, (void __user*)arg, sizeof(*params))) { \
88 uvm_kvfree(params); \
89 return -EFAULT; \
90 } \
91 \
92 params->rmStatus = uvm_global_get_status(); \
93 if (params->rmStatus == NV_OK) { \
94 if (do_init_check) { \
95 if (!uvm_fd_va_space(filp)) \
96 params->rmStatus = NV_ERR_ILLEGAL_ACTION; \
97 } \
98 if (likely(params->rmStatus == NV_OK)) \
99 params->rmStatus = function_name(params, filp); \
100 } \
101 \
102 if (nv_copy_to_user((void __user*)arg, params, sizeof(*params))) \
103 ret = -EFAULT; \
104 \
105 uvm_kvfree(params); \
106 return ret; \
107 }
108
109 #define UVM_ROUTE_CMD_ALLOC_NO_INIT_CHECK(cmd, function_name) \
110 __UVM_ROUTE_CMD_ALLOC(cmd, cmd##_PARAMS, function_name, false)
111
112 #define UVM_ROUTE_CMD_ALLOC_INIT_CHECK(cmd, function_name) \
113 __UVM_ROUTE_CMD_ALLOC(cmd, cmd##_PARAMS, function_name, true)
114
115 // Wrap an entry point into the UVM module.
116 //
117 // An entry function with signature
118 //
119 // return_type foo(...);
120 //
121 // is required to have a counterpart of the form
122 //
123 // return_type foo_entry(...) {
124 // UVM_ENTRY_RET(foo(...));
125 // }
126 //
127 // An entry function with signature
128 //
129 // void foo(...);
130 //
131 // is required to have a counterpart of the form
132 //
133 // void foo_entry(...) {
134 // UVM_ENTRY_VOID(foo(...));
135 // }
136 //
137 // Invocations of foo must be replaced by invocations of foo_entry at the entry
138 // points.
139 #define UVM_ENTRY_WRAP(line) \
140 do { \
141 bool added; \
142 \
143 if (in_interrupt()) { \
144 line; \
145 } \
146 else if (uvm_thread_context_wrapper_is_used()) { \
147 uvm_thread_context_wrapper_t thread_context_wrapper; \
148 \
149 added = uvm_thread_context_add(&thread_context_wrapper.context); \
150 line; \
151 if (added) \
152 uvm_thread_context_remove(&thread_context_wrapper.context); \
153 } \
154 else { \
155 uvm_thread_context_t thread_context; \
156 \
157 added = uvm_thread_context_add(&thread_context); \
158 line; \
159 if (added) \
160 uvm_thread_context_remove(&thread_context); \
161 } \
162 } while (0) \
163
164 // Wrapper for non-void functions
165 #define UVM_ENTRY_RET(func_call) \
166 do { \
167 typeof(func_call) ret; \
168 UVM_ENTRY_WRAP((ret = (func_call))); \
169 return ret; \
170 } while (0) \
171
172 // Wrapper for void functions
173 #define UVM_ENTRY_VOID UVM_ENTRY_WRAP
174
175 // Validate input ranges from the user with specific alignment requirement
uvm_api_range_invalid_aligned(NvU64 base,NvU64 length,NvU64 alignment)176 static bool uvm_api_range_invalid_aligned(NvU64 base, NvU64 length, NvU64 alignment)
177 {
178 return !IS_ALIGNED(base, alignment) ||
179 !IS_ALIGNED(length, alignment) ||
180 base == 0 ||
181 length == 0 ||
182 base + length < base; // Overflow
183 }
184
185 // Most APIs require PAGE_SIZE alignment
uvm_api_range_invalid(NvU64 base,NvU64 length)186 static bool uvm_api_range_invalid(NvU64 base, NvU64 length)
187 {
188 return uvm_api_range_invalid_aligned(base, length, PAGE_SIZE);
189 }
190
191 // Some APIs can only enforce 4K alignment as it's the smallest GPU page size
192 // even when the smallest host page is larger (e.g. 64K on ppc64le).
uvm_api_range_invalid_4k(NvU64 base,NvU64 length)193 static bool uvm_api_range_invalid_4k(NvU64 base, NvU64 length)
194 {
195 return uvm_api_range_invalid_aligned(base, length, UVM_PAGE_SIZE_4K);
196 }
197
198 // Verify alignment on a 64K boundary.
uvm_api_range_invalid_64k(NvU64 base,NvU64 length)199 static bool uvm_api_range_invalid_64k(NvU64 base, NvU64 length)
200 {
201 return uvm_api_range_invalid_aligned(base, length, UVM_PAGE_SIZE_64K);
202 }
203
204 typedef enum
205 {
206 UVM_API_RANGE_TYPE_MANAGED,
207 UVM_API_RANGE_TYPE_HMM,
208 UVM_API_RANGE_TYPE_ATS,
209 UVM_API_RANGE_TYPE_INVALID
210 } uvm_api_range_type_t;
211
212 // If the interval [base, base + length) is fully covered by VMAs which all have
213 // the same uvm_api_range_type_t, that range type is returned.
214 //
215 // LOCKING: va_space->lock must be held in at least read mode. If mm != NULL,
216 // mm->mmap_lock must also be held in at least read mode.
217 uvm_api_range_type_t uvm_api_range_type_check(uvm_va_space_t *va_space, struct mm_struct *mm, NvU64 base, NvU64 length);
218
219 NV_STATUS uvm_api_pageable_mem_access_on_gpu(UVM_PAGEABLE_MEM_ACCESS_ON_GPU_PARAMS *params, struct file *filp);
220 NV_STATUS uvm_api_register_gpu(UVM_REGISTER_GPU_PARAMS *params, struct file *filp);
221 NV_STATUS uvm_api_unregister_gpu(UVM_UNREGISTER_GPU_PARAMS *params, struct file *filp);
222 NV_STATUS uvm_api_create_range_group(UVM_CREATE_RANGE_GROUP_PARAMS *params, struct file *filp);
223 NV_STATUS uvm_api_destroy_range_group(UVM_DESTROY_RANGE_GROUP_PARAMS *params, struct file *filp);
224 NV_STATUS uvm_api_enable_peer_access(UVM_ENABLE_PEER_ACCESS_PARAMS *params, struct file *filp);
225 NV_STATUS uvm_api_disable_peer_access(UVM_DISABLE_PEER_ACCESS_PARAMS *params, struct file *filp);
226 NV_STATUS uvm_api_set_range_group(UVM_SET_RANGE_GROUP_PARAMS *params, struct file *filp);
227 NV_STATUS uvm_api_create_external_range(UVM_CREATE_EXTERNAL_RANGE_PARAMS *params, struct file *filp);
228 NV_STATUS uvm_api_map_external_allocation(UVM_MAP_EXTERNAL_ALLOCATION_PARAMS *params, struct file *filp);
229 NV_STATUS uvm_api_map_external_sparse(UVM_MAP_EXTERNAL_SPARSE_PARAMS *params, struct file *filp);
230 NV_STATUS uvm_api_free(UVM_FREE_PARAMS *params, struct file *filp);
231 NV_STATUS uvm_api_prevent_migration_range_groups(UVM_PREVENT_MIGRATION_RANGE_GROUPS_PARAMS *params, struct file *filp);
232 NV_STATUS uvm_api_allow_migration_range_groups(UVM_ALLOW_MIGRATION_RANGE_GROUPS_PARAMS *params, struct file *filp);
233 NV_STATUS uvm_api_set_preferred_location(const UVM_SET_PREFERRED_LOCATION_PARAMS *params, struct file *filp);
234 NV_STATUS uvm_api_unset_preferred_location(const UVM_UNSET_PREFERRED_LOCATION_PARAMS *params, struct file *filp);
235 NV_STATUS uvm_api_set_accessed_by(const UVM_SET_ACCESSED_BY_PARAMS *params, struct file *filp);
236 NV_STATUS uvm_api_unset_accessed_by(const UVM_UNSET_ACCESSED_BY_PARAMS *params, struct file *filp);
237 NV_STATUS uvm_api_register_gpu_va_space(UVM_REGISTER_GPU_VASPACE_PARAMS *params, struct file *filp);
238 NV_STATUS uvm_api_unregister_gpu_va_space(UVM_UNREGISTER_GPU_VASPACE_PARAMS *params, struct file *filp);
239 NV_STATUS uvm_api_register_channel(UVM_REGISTER_CHANNEL_PARAMS *params, struct file *filp);
240 NV_STATUS uvm_api_unregister_channel(UVM_UNREGISTER_CHANNEL_PARAMS *params, struct file *filp);
241 NV_STATUS uvm_api_enable_read_duplication(const UVM_ENABLE_READ_DUPLICATION_PARAMS *params, struct file *filp);
242 NV_STATUS uvm_api_disable_read_duplication(const UVM_DISABLE_READ_DUPLICATION_PARAMS *params, struct file *filp);
243 NV_STATUS uvm_api_migrate(UVM_MIGRATE_PARAMS *params, struct file *filp);
244 NV_STATUS uvm_api_enable_system_wide_atomics(UVM_ENABLE_SYSTEM_WIDE_ATOMICS_PARAMS *params, struct file *filp);
245 NV_STATUS uvm_api_disable_system_wide_atomics(UVM_DISABLE_SYSTEM_WIDE_ATOMICS_PARAMS *params, struct file *filp);
246 NV_STATUS uvm_api_tools_init_event_tracker(UVM_TOOLS_INIT_EVENT_TRACKER_PARAMS *params, struct file *filp);
247 NV_STATUS uvm_api_tools_set_notification_threshold(UVM_TOOLS_SET_NOTIFICATION_THRESHOLD_PARAMS *params, struct file *filp);
248 NV_STATUS uvm_api_tools_event_queue_enable_events(UVM_TOOLS_EVENT_QUEUE_ENABLE_EVENTS_PARAMS *params, struct file *filp);
249 NV_STATUS uvm_api_tools_event_queue_disable_events(UVM_TOOLS_EVENT_QUEUE_DISABLE_EVENTS_PARAMS *params, struct file *filp);
250 NV_STATUS uvm_api_tools_enable_counters(UVM_TOOLS_ENABLE_COUNTERS_PARAMS *params, struct file *filp);
251 NV_STATUS uvm_api_tools_disable_counters(UVM_TOOLS_DISABLE_COUNTERS_PARAMS *params, struct file *filp);
252 NV_STATUS uvm_api_tools_read_process_memory(UVM_TOOLS_READ_PROCESS_MEMORY_PARAMS *params, struct file *filp);
253 NV_STATUS uvm_api_tools_write_process_memory(UVM_TOOLS_WRITE_PROCESS_MEMORY_PARAMS *params, struct file *filp);
254 NV_STATUS uvm_api_map_dynamic_parallelism_region(UVM_MAP_DYNAMIC_PARALLELISM_REGION_PARAMS *params, struct file *filp);
255 NV_STATUS uvm_api_unmap_external(UVM_UNMAP_EXTERNAL_PARAMS *params, struct file *filp);
256 NV_STATUS uvm_api_migrate_range_group(UVM_MIGRATE_RANGE_GROUP_PARAMS *params, struct file *filp);
257 NV_STATUS uvm_api_alloc_semaphore_pool(UVM_ALLOC_SEMAPHORE_POOL_PARAMS *params, struct file *filp);
258 NV_STATUS uvm_api_populate_pageable(const UVM_POPULATE_PAGEABLE_PARAMS *params, struct file *filp);
259
260 #endif // __UVM_API_H__
261