1 /*******************************************************************************
2 Copyright (c) 2013-2023 NVIDIA Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to
6 deal in the Software without restriction, including without limitation the
7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8 sell copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 DEALINGS IN THE SOFTWARE.
21
22 *******************************************************************************/
23
24 #ifndef __UVM_COMMON_H__
25 #define __UVM_COMMON_H__
26
27 #ifdef DEBUG
28 #define UVM_IS_DEBUG() 1
29 #else
30 #define UVM_IS_DEBUG() 0
31 #endif
32
33 // NVIDIA_UVM_DEVELOP implies DEBUG, but not vice-versa
34 // TODO Bug 1773100: Figure out the right distinction between develop and debug
35 // builds.
36 #ifdef NVIDIA_UVM_DEVELOP
37 #define UVM_IS_DEVELOP() 1
38 #else
39 #define UVM_IS_DEVELOP() 0
40 #endif
41
42 #include "uvm_types.h"
43 #include "uvm_linux.h"
44
45 enum {
46 NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0,
47 NVIDIA_UVM_TOOLS_MINOR_NUMBER = 1,
48 // to ensure backward-compatiblity and correct counting, please insert any
49 // new minor devices just above the following field:
50 NVIDIA_UVM_NUM_MINOR_DEVICES
51 };
52
53 #define UVM_GPU_UUID_TEXT_BUFFER_LENGTH (8+16*2+4+1)
54
55 int format_uuid_to_buffer(char *buffer, unsigned bufferLength, const NvProcessorUuid *pGpuUuid);
56
57 #define UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ...) \
58 func(prefix "%s:%u %s[pid:%d]" fmt, \
59 kbasename(__FILE__), \
60 __LINE__, \
61 __FUNCTION__, \
62 current->pid, \
63 ##__VA_ARGS__)
64
65 #define UVM_PRINT_FUNC(func, fmt, ...) \
66 UVM_PRINT_FUNC_PREFIX(func, "", fmt, ##__VA_ARGS__)
67
68 // Check whether UVM_{ERR,DBG,INFO)_PRINT* should be enabled
69 bool uvm_debug_prints_enabled(void);
70
71 // A printing helper like UVM_PRINT_FUNC_PREFIX that only prints if
72 // uvm_debug_prints_enabled() returns true.
73 #define UVM_PRINT_FUNC_PREFIX_CHECK(func, prefix, fmt, ...) \
74 do { \
75 if (uvm_debug_prints_enabled()) { \
76 UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ##__VA_ARGS__); \
77 } \
78 } while (0)
79
80 #define UVM_ASSERT_PRINT(fmt, ...) \
81 UVM_PRINT_FUNC_PREFIX(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
82
83 #define UVM_ASSERT_PRINT_RL(fmt, ...) \
84 UVM_PRINT_FUNC_PREFIX(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
85
86 #define UVM_ERR_PRINT(fmt, ...) \
87 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
88
89 #define UVM_ERR_PRINT_RL(fmt, ...) \
90 UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
91
92 #define UVM_DBG_PRINT(fmt, ...) \
93 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
94
95 #define UVM_DBG_PRINT_RL(fmt, ...) \
96 UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
97
98 #define UVM_INFO_PRINT(fmt, ...) \
99 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_INFO NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__)
100
101 //
102 // Please see the documentation of format_uuid_to_buffer, for details on what
103 // this routine prints for you.
104 //
105 #define UVM_DBG_PRINT_UUID(msg, uuidPtr) \
106 do { \
107 char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH]; \
108 format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \
109 UVM_DBG_PRINT("%s: %s\n", msg, uuidBuffer); \
110 } while (0)
111
112 #define UVM_ERR_PRINT_NV_STATUS(msg, rmStatus, ...) \
113 UVM_ERR_PRINT("ERROR: %s : " msg "\n", nvstatusToString(rmStatus), ##__VA_ARGS__)
114
115 #define UVM_ERR_PRINT_UUID(msg, uuidPtr, ...) \
116 do { \
117 char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH]; \
118 format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \
119 UVM_ERR_PRINT("ERROR: %s : " msg "\n", uuidBuffer, ##__VA_ARGS__); \
120 } while (0)
121
122 #define UVM_PANIC() UVM_PRINT_FUNC(panic, "\n")
123 #define UVM_PANIC_MSG(fmt, ...) UVM_PRINT_FUNC(panic, ": " fmt, ##__VA_ARGS__)
124
125 #define UVM_PANIC_ON_MSG(cond, fmt, ...) \
126 do { \
127 if (unlikely(cond)) \
128 UVM_PANIC_MSG(fmt, ##__VA_ARGS__); \
129 } while (0)
130
131 #define UVM_PANIC_ON(cond) UVM_PANIC_ON_MSG(cond, "failed cond %s\n", #cond)
132
133 // expr may include function calls. Use sizeof to prevent it from being
134 // evaluated while also preventing unused variable warnings. sizeof() can't be
135 // used on a bitfield however, so use ! to force the expression to evaluate as
136 // an int.
137 #define UVM_IGNORE_EXPR(expr) ((void)sizeof(!(expr)))
138
139 #define UVM_IGNORE_EXPR2(expr1, expr2) \
140 do { \
141 UVM_IGNORE_EXPR(expr1); \
142 UVM_IGNORE_EXPR(expr2); \
143 } while (0)
144
145 // NO-OP function to break on_uvm_test_fail - that is just to set a breakpoint
146 void on_uvm_test_fail(void);
147
148 // NO-OP function to break on_uvm_assert - that is just to set a breakpoint
149 // Unlike on_uvm_test_fail it provides 'panic' coverity semantics
150 void on_uvm_assert(void);
151
152 #define _UVM_ASSERT_MSG(expr, cond, fmt, ...) \
153 do { \
154 if (unlikely(!(expr))) { \
155 UVM_ASSERT_PRINT("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \
156 dump_stack(); \
157 on_uvm_assert(); \
158 } \
159 } while (0)
160
161 // Prevent function calls in expr and the print argument list from being
162 // evaluated.
163 #define UVM_ASSERT_MSG_IGNORE(expr, fmt, ...) \
164 do { \
165 UVM_IGNORE_EXPR(expr); \
166 UVM_NO_PRINT(fmt, ##__VA_ARGS__); \
167 } while (0)
168
169 // UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity builds
170 #if UVM_IS_DEBUG() || defined __COVERITY__
171 #define UVM_ASSERT_MSG(expr, fmt, ...) _UVM_ASSERT_MSG(expr, #expr, ": " fmt, ##__VA_ARGS__)
172 #define UVM_ASSERT(expr) _UVM_ASSERT_MSG(expr, #expr, "\n")
173 #else
174 #define UVM_ASSERT_MSG(expr, fmt, ...) UVM_ASSERT_MSG_IGNORE(expr, fmt, ##__VA_ARGS__)
175 #define UVM_ASSERT(expr) UVM_ASSERT_MSG_IGNORE(expr, "\n")
176 #endif
177
178 // UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always included in the
179 // build, even on release builds. They are skipped at runtime if
180 // uvm_release_asserts is 0.
181
182 // Whether release asserts are enabled and whether they should dump the stack
183 // and set the global error.
184 extern int uvm_release_asserts;
185 extern int uvm_release_asserts_dump_stack;
186 extern int uvm_release_asserts_set_global_error;
187 extern bool uvm_release_asserts_set_global_error_for_tests;
188
189 // Given these are enabled for release builds, we need to be more cautious than
190 // in UVM_ASSERT(). Use a ratelimited print and only dump the stack if a module
191 // param is enabled.
192 #define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \
193 do { \
194 if (uvm_release_asserts && unlikely(!(expr))) { \
195 UVM_ASSERT_PRINT_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \
196 if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests) \
197 uvm_global_set_fatal_error(NV_ERR_INVALID_STATE); \
198 if (uvm_release_asserts_dump_stack) \
199 dump_stack(); \
200 on_uvm_assert(); \
201 } \
202 } while (0)
203
204 #define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__)
205 #define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n")
206
207 #define UVM_SIZE_1KB (1024ULL)
208 #define UVM_SIZE_1MB (1024 * UVM_SIZE_1KB)
209 #define UVM_SIZE_1GB (1024 * UVM_SIZE_1MB)
210 #define UVM_SIZE_1TB (1024 * UVM_SIZE_1GB)
211 #define UVM_SIZE_1PB (1024 * UVM_SIZE_1TB)
212
213 #define UVM_ALIGN_DOWN(x, a) ({ \
214 typeof(x) _a = a; \
215 UVM_ASSERT(is_power_of_2(_a)); \
216 (x) & ~(_a - 1); \
217 })
218
219 #define UVM_ALIGN_UP(x, a) ({ \
220 typeof(x) _a = a; \
221 UVM_ASSERT(is_power_of_2(_a)); \
222 ((x) + _a - 1) & ~(_a - 1); \
223 })
224
225 #define UVM_PAGE_ALIGN_UP(value) UVM_ALIGN_UP(value, PAGE_SIZE)
226 #define UVM_PAGE_ALIGN_DOWN(value) UVM_ALIGN_DOWN(value, PAGE_SIZE)
227
228 // These macros provide a convenient way to string-ify enum values.
229 #define UVM_ENUM_STRING_CASE(value) case value: return #value
230 #define UVM_ENUM_STRING_DEFAULT() default: return "UNKNOWN"
231
232 // Divide by a dynamic value known at runtime to be a power of 2. ilog2 is
233 // optimized as a single instruction in many processors, whereas integer
234 // division is always slow.
uvm_div_pow2_32(NvU32 numerator,NvU32 denominator_pow2)235 static inline NvU32 uvm_div_pow2_32(NvU32 numerator, NvU32 denominator_pow2)
236 {
237 UVM_ASSERT(is_power_of_2(denominator_pow2));
238 UVM_ASSERT(denominator_pow2);
239 return numerator >> ilog2(denominator_pow2);
240 }
241
uvm_div_pow2_64(NvU64 numerator,NvU64 denominator_pow2)242 static inline NvU64 uvm_div_pow2_64(NvU64 numerator, NvU64 denominator_pow2)
243 {
244 UVM_ASSERT(is_power_of_2(denominator_pow2));
245 UVM_ASSERT(denominator_pow2);
246 return numerator >> ilog2(denominator_pow2);
247 }
248
249 #define SUM_FROM_0_TO_N(n) (((n) * ((n) + 1)) / 2)
250
251 // Start and end are inclusive
uvm_ranges_overlap(NvU64 a_start,NvU64 a_end,NvU64 b_start,NvU64 b_end)252 static inline NvBool uvm_ranges_overlap(NvU64 a_start, NvU64 a_end, NvU64 b_start, NvU64 b_end)
253 {
254 // De Morgan's of: !(a_end < b_start || b_end < a_start)
255 return a_end >= b_start && b_end >= a_start;
256 }
257
debug_mode(void)258 static int debug_mode(void)
259 {
260 #ifdef DEBUG
261 return 1;
262 #else
263 return 0;
264 #endif
265 }
266
kmem_cache_destroy_safe(struct kmem_cache ** ppCache)267 static inline void kmem_cache_destroy_safe(struct kmem_cache **ppCache)
268 {
269 if (ppCache)
270 {
271 if (*ppCache)
272 kmem_cache_destroy(*ppCache);
273
274 *ppCache = NULL;
275 }
276 }
277
278 typedef struct
279 {
280 NvU64 start_time_ns;
281 NvU64 print_time_ns;
282 } uvm_spin_loop_t;
283
uvm_spin_loop_init(uvm_spin_loop_t * spin)284 static inline void uvm_spin_loop_init(uvm_spin_loop_t *spin)
285 {
286 NvU64 curr = NV_GETTIME();
287 spin->start_time_ns = curr;
288 spin->print_time_ns = curr;
289 }
290
291 // Periodically yields the CPU when not called from interrupt context. Returns
292 // NV_ERR_TIMEOUT_RETRY if the caller should print a warning that we've been
293 // waiting too long, and NV_OK otherwise.
294 NV_STATUS uvm_spin_loop(uvm_spin_loop_t *spin);
295
uvm_spin_loop_elapsed(const uvm_spin_loop_t * spin)296 static NvU64 uvm_spin_loop_elapsed(const uvm_spin_loop_t *spin)
297 {
298 NvU64 curr = NV_GETTIME();
299 return curr - spin->start_time_ns;
300 }
301
302 #define UVM_SPIN_LOOP(__spin) ({ \
303 NV_STATUS __status = uvm_spin_loop(__spin); \
304 if (__status == NV_ERR_TIMEOUT_RETRY) { \
305 UVM_DBG_PRINT("Warning: stuck waiting for %llus\n", \
306 uvm_spin_loop_elapsed(__spin) / (1000*1000*1000)); \
307 \
308 if (uvm_debug_prints_enabled()) \
309 dump_stack(); \
310 } \
311 __status; \
312 })
313
314 // Execute the loop code while cond is true. Invokes uvm_spin_loop_iter at the
315 // end of each iteration.
316 #define UVM_SPIN_WHILE(cond, spin) \
317 if (cond) \
318 for (uvm_spin_loop_init(spin); (cond); UVM_SPIN_LOOP(spin))
319
320 //
321 // Documentation for the internal routines listed below may be found in the
322 // implementation file(s).
323 //
324 NV_STATUS errno_to_nv_status(int errnoCode);
325 int nv_status_to_errno(NV_STATUS status);
326 unsigned uvm_get_stale_process_id(void);
327 unsigned uvm_get_stale_thread_id(void);
328
329 extern int uvm_enable_builtin_tests;
330
uvm_init_character_device(struct cdev * cdev,const struct file_operations * fops)331 static inline void uvm_init_character_device(struct cdev *cdev, const struct file_operations *fops)
332 {
333 cdev_init(cdev, fops);
334 cdev->owner = THIS_MODULE;
335 }
336
337 typedef struct
338 {
339 int rm_control_fd;
340 NvHandle user_client;
341 NvHandle user_object;
342 } uvm_rm_user_object_t;
343
344 typedef enum
345 {
346 UVM_FD_UNINITIALIZED,
347 UVM_FD_INITIALIZING,
348 UVM_FD_VA_SPACE,
349 UVM_FD_MM,
350 UVM_FD_COUNT
351 } uvm_fd_type_t;
352
353 // This should be large enough to fit the valid values from uvm_fd_type_t above.
354 // Note we can't use order_base_2(UVM_FD_COUNT) to define this because our code
355 // coverage tool fails due when the preprocessor expands that to a huge mess of
356 // ternary operators.
357 #define UVM_FD_TYPE_BITS 2
358 #define UVM_FD_TYPE_MASK ((1UL << UVM_FD_TYPE_BITS) - 1)
359
360 // Macro used to compare two values for types that support less than operator.
361 // It returns -1 if a < b, 1 if a > b and 0 if a == 0
362 #define UVM_CMP_DEFAULT(a,b) \
363 ({ \
364 typeof(a) _a = a; \
365 typeof(b) _b = b; \
366 int __ret; \
367 BUILD_BUG_ON(sizeof(a) != sizeof(b)); \
368 if (_a < _b) \
369 __ret = -1; \
370 else if (_b < _a) \
371 __ret = 1; \
372 else \
373 __ret = 0; \
374 \
375 __ret; \
376 })
377
378 // Returns whether the input file was opened against the UVM character device
379 // file. A NULL input returns false.
380 bool uvm_file_is_nvidia_uvm(struct file *filp);
381
382 // Returns the type of data filp->private_data contains to and if ptr_val !=
383 // NULL returns the value of the pointer.
384 uvm_fd_type_t uvm_fd_type(struct file *filp, void **ptr_val);
385
386 // Returns the pointer stored in filp->private_data if the type
387 // matches, otherwise returns NULL.
388 void *uvm_fd_get_type(struct file *filp, uvm_fd_type_t type);
389
390 // Reads the first word in the supplied struct page.
uvm_touch_page(struct page * page)391 static inline void uvm_touch_page(struct page *page)
392 {
393 char *mapping;
394
395 UVM_ASSERT(page);
396
397 mapping = (char *) kmap(page);
398 (void)UVM_READ_ONCE(*mapping);
399 kunmap(page);
400 }
401
402 // Return true if the VMA is one used by UVM managed allocations.
403 bool uvm_vma_is_managed(struct vm_area_struct *vma);
404
uvm_platform_uses_canonical_form_address(void)405 static bool uvm_platform_uses_canonical_form_address(void)
406 {
407 if (NVCPU_IS_PPC64LE)
408 return false;
409
410 return true;
411 }
412
413 // Similar to the GPU MMU HAL num_va_bits(), it returns the CPU's num_va_bits().
uvm_cpu_num_va_bits(void)414 static NvU32 uvm_cpu_num_va_bits(void)
415 {
416 return fls64(TASK_SIZE - 1) + 1;
417 }
418
419 // Return the unaddressable range in a num_va_bits-wide VA space, [first, outer)
uvm_get_unaddressable_range(NvU32 num_va_bits,NvU64 * first,NvU64 * outer)420 static void uvm_get_unaddressable_range(NvU32 num_va_bits, NvU64 *first, NvU64 *outer)
421 {
422 UVM_ASSERT(num_va_bits < 64);
423 UVM_ASSERT(first);
424 UVM_ASSERT(outer);
425
426 if (uvm_platform_uses_canonical_form_address()) {
427 *first = 1ULL << (num_va_bits - 1);
428 *outer = (NvU64)((NvS64)(1ULL << 63) >> (64 - num_va_bits));
429 }
430 else {
431 *first = 1ULL << num_va_bits;
432 *outer = ~0Ull;
433 }
434 }
435
uvm_cpu_get_unaddressable_range(NvU64 * first,NvU64 * outer)436 static void uvm_cpu_get_unaddressable_range(NvU64 *first, NvU64 *outer)
437 {
438 return uvm_get_unaddressable_range(uvm_cpu_num_va_bits(), first, outer);
439 }
440
441 #endif /* __UVM_COMMON_H__ */
442