1 /******************************************************************************* 2 Copyright (c) 2013-2021 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #ifndef _UVM_COMMON_H 25 #define _UVM_COMMON_H 26 27 #ifdef DEBUG 28 #define UVM_IS_DEBUG() 1 29 #else 30 #define UVM_IS_DEBUG() 0 31 #endif 32 33 // NVIDIA_UVM_DEVELOP implies DEBUG, but not vice-versa 34 // TODO Bug 1773100: Figure out the right distinction between develop and debug 35 // builds. 36 #ifdef NVIDIA_UVM_DEVELOP 37 #define UVM_IS_DEVELOP() 1 38 #else 39 #define UVM_IS_DEVELOP() 0 40 #endif 41 42 #include "uvm_types.h" 43 #include "uvm_linux.h" 44 45 enum { 46 NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0, 47 NVIDIA_UVM_TOOLS_MINOR_NUMBER = 1, 48 // to ensure backward-compatiblity and correct counting, please insert any 49 // new minor devices just above the following field: 50 NVIDIA_UVM_NUM_MINOR_DEVICES 51 }; 52 53 #define UVM_GPU_UUID_TEXT_BUFFER_LENGTH (8+16*2+4+1) 54 55 int format_uuid_to_buffer(char *buffer, unsigned bufferLength, const NvProcessorUuid *pGpuUuid); 56 57 #define UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ...) \ 58 func(prefix "%s:%u %s[pid:%d]" fmt, \ 59 kbasename(__FILE__), \ 60 __LINE__, \ 61 __FUNCTION__, \ 62 current->pid, \ 63 ##__VA_ARGS__) 64 65 #define UVM_PRINT_FUNC(func, fmt, ...) \ 66 UVM_PRINT_FUNC_PREFIX(func, "", fmt, ##__VA_ARGS__) 67 68 // Check whether UVM_{ERR,DBG,INFO)_PRINT* should be enabled 69 bool uvm_debug_prints_enabled(void); 70 71 // A printing helper like UVM_PRINT_FUNC_PREFIX that only prints if 72 // uvm_debug_prints_enabled() returns true. 73 #define UVM_PRINT_FUNC_PREFIX_CHECK(func, prefix, fmt, ...) \ 74 do { \ 75 if (uvm_debug_prints_enabled()) { \ 76 UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ##__VA_ARGS__); \ 77 } \ 78 } while (0) 79 80 #define UVM_ASSERT_PRINT(fmt, ...) \ 81 UVM_PRINT_FUNC_PREFIX(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 82 83 #define UVM_ASSERT_PRINT_RL(fmt, ...) \ 84 UVM_PRINT_FUNC_PREFIX(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 85 86 #define UVM_ERR_PRINT(fmt, ...) \ 87 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 88 89 #define UVM_ERR_PRINT_RL(fmt, ...) \ 90 UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 91 92 #define UVM_DBG_PRINT(fmt, ...) \ 93 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 94 95 #define UVM_DBG_PRINT_RL(fmt, ...) \ 96 UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 97 98 #define UVM_INFO_PRINT(fmt, ...) \ 99 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_INFO NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 100 101 // 102 // Please see the documentation of format_uuid_to_buffer, for details on what 103 // this routine prints for you. 104 // 105 #define UVM_DBG_PRINT_UUID(msg, uuidPtr) \ 106 do { \ 107 char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH]; \ 108 format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \ 109 UVM_DBG_PRINT("%s: %s\n", msg, uuidBuffer); \ 110 } while (0) 111 112 #define UVM_ERR_PRINT_NV_STATUS(msg, rmStatus, ...) \ 113 UVM_ERR_PRINT("ERROR: %s : " msg "\n", nvstatusToString(rmStatus), ##__VA_ARGS__) 114 115 #define UVM_ERR_PRINT_UUID(msg, uuidPtr, ...) \ 116 do { \ 117 char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH]; \ 118 format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \ 119 UVM_ERR_PRINT("ERROR: %s : " msg "\n", uuidBuffer, ##__VA_ARGS__); \ 120 } while (0) 121 122 #define UVM_PANIC() UVM_PRINT_FUNC(panic, "\n") 123 #define UVM_PANIC_MSG(fmt, ...) UVM_PRINT_FUNC(panic, ": " fmt, ##__VA_ARGS__) 124 125 #define UVM_PANIC_ON_MSG(cond, fmt, ...) \ 126 do { \ 127 if (unlikely(cond)) \ 128 UVM_PANIC_MSG(fmt, ##__VA_ARGS__); \ 129 } while (0) 130 131 #define UVM_PANIC_ON(cond) UVM_PANIC_ON_MSG(cond, "failed cond %s\n", #cond) 132 133 // expr may include function calls. Use sizeof to prevent it from being 134 // evaluated while also preventing unused variable warnings. sizeof() can't be 135 // used on a bitfield however, so use ! to force the expression to evaluate as 136 // an int. 137 #define UVM_IGNORE_EXPR(expr) ((void)sizeof(!(expr))) 138 139 #define UVM_IGNORE_EXPR2(expr1, expr2) \ 140 do { \ 141 UVM_IGNORE_EXPR(expr1); \ 142 UVM_IGNORE_EXPR(expr2); \ 143 } while (0) 144 145 // NO-OP function to break on_uvm_test_fail - that is just to set a breakpoint 146 void on_uvm_test_fail(void); 147 148 // NO-OP function to break on_uvm_assert - that is just to set a breakpoint 149 // Unlike on_uvm_test_fail it provides 'panic' coverity semantics 150 void on_uvm_assert(void); 151 152 #define _UVM_ASSERT_MSG(expr, cond, fmt, ...) \ 153 do { \ 154 if (unlikely(!(expr))) { \ 155 UVM_ASSERT_PRINT("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \ 156 dump_stack(); \ 157 on_uvm_assert(); \ 158 } \ 159 } while (0) 160 161 // Prevent function calls in expr and the print argument list from being 162 // evaluated. 163 #define UVM_ASSERT_MSG_IGNORE(expr, fmt, ...) \ 164 do { \ 165 UVM_IGNORE_EXPR(expr); \ 166 UVM_NO_PRINT(fmt, ##__VA_ARGS__); \ 167 } while (0) 168 169 // UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity builds 170 #if UVM_IS_DEBUG() || defined __COVERITY__ 171 #define UVM_ASSERT_MSG(expr, fmt, ...) _UVM_ASSERT_MSG(expr, #expr, ": " fmt, ##__VA_ARGS__) 172 #define UVM_ASSERT(expr) _UVM_ASSERT_MSG(expr, #expr, "\n") 173 #else 174 #define UVM_ASSERT_MSG(expr, fmt, ...) UVM_ASSERT_MSG_IGNORE(expr, fmt, ##__VA_ARGS__) 175 #define UVM_ASSERT(expr) UVM_ASSERT_MSG_IGNORE(expr, "\n") 176 #endif 177 178 // UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always included in the 179 // build, even on release builds. They are skipped at runtime if 180 // uvm_release_asserts is 0. 181 182 // Whether release asserts are enabled and whether they should dump the stack 183 // and set the global error. 184 extern int uvm_release_asserts; 185 extern int uvm_release_asserts_dump_stack; 186 extern int uvm_release_asserts_set_global_error; 187 extern bool uvm_release_asserts_set_global_error_for_tests; 188 189 // Given these are enabled for release builds, we need to be more cautious than 190 // in UVM_ASSERT(). Use a ratelimited print and only dump the stack if a module 191 // param is enabled. 192 #define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \ 193 do { \ 194 if (uvm_release_asserts && unlikely(!(expr))) { \ 195 UVM_ASSERT_PRINT_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \ 196 if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests) \ 197 uvm_global_set_fatal_error(NV_ERR_INVALID_STATE); \ 198 if (uvm_release_asserts_dump_stack) \ 199 dump_stack(); \ 200 on_uvm_assert(); \ 201 } \ 202 } while (0) 203 204 #define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__) 205 #define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n") 206 207 // Provide a short form of UUID's, typically for use in debug printing: 208 #define ABBREV_UUID(uuid) (unsigned)(uuid) 209 210 static inline NvBool uvm_uuid_is_cpu(const NvProcessorUuid *uuid) 211 { 212 return memcmp(uuid, &NV_PROCESSOR_UUID_CPU_DEFAULT, sizeof(*uuid)) == 0; 213 } 214 #define UVM_SIZE_1KB (1024ULL) 215 #define UVM_SIZE_1MB (1024 * UVM_SIZE_1KB) 216 #define UVM_SIZE_1GB (1024 * UVM_SIZE_1MB) 217 #define UVM_SIZE_1TB (1024 * UVM_SIZE_1GB) 218 #define UVM_SIZE_1PB (1024 * UVM_SIZE_1TB) 219 220 #define UVM_ALIGN_DOWN(x, a) ({ \ 221 typeof(x) _a = a; \ 222 UVM_ASSERT(is_power_of_2(_a)); \ 223 (x) & ~(_a - 1); \ 224 }) 225 226 #define UVM_ALIGN_UP(x, a) ({ \ 227 typeof(x) _a = a; \ 228 UVM_ASSERT(is_power_of_2(_a)); \ 229 ((x) + _a - 1) & ~(_a - 1); \ 230 }) 231 232 #define UVM_PAGE_ALIGN_UP(value) UVM_ALIGN_UP(value, PAGE_SIZE) 233 #define UVM_PAGE_ALIGN_DOWN(value) UVM_ALIGN_DOWN(value, PAGE_SIZE) 234 235 // These macros provide a convenient way to string-ify enum values. 236 #define UVM_ENUM_STRING_CASE(value) case value: return #value 237 #define UVM_ENUM_STRING_DEFAULT() default: return "UNKNOWN" 238 239 // Divide by a dynamic value known at runtime to be a power of 2. ilog2 is 240 // optimized as a single instruction in many processors, whereas integer 241 // division is always slow. 242 static inline NvU32 uvm_div_pow2_32(NvU32 numerator, NvU32 denominator_pow2) 243 { 244 UVM_ASSERT(is_power_of_2(denominator_pow2)); 245 UVM_ASSERT(denominator_pow2); 246 return numerator >> ilog2(denominator_pow2); 247 } 248 249 static inline NvU64 uvm_div_pow2_64(NvU64 numerator, NvU64 denominator_pow2) 250 { 251 UVM_ASSERT(is_power_of_2(denominator_pow2)); 252 UVM_ASSERT(denominator_pow2); 253 return numerator >> ilog2(denominator_pow2); 254 } 255 256 #define SUM_FROM_0_TO_N(n) (((n) * ((n) + 1)) / 2) 257 258 // Start and end are inclusive 259 static inline NvBool uvm_ranges_overlap(NvU64 a_start, NvU64 a_end, NvU64 b_start, NvU64 b_end) 260 { 261 // De Morgan's of: !(a_end < b_start || b_end < a_start) 262 return a_end >= b_start && b_end >= a_start; 263 } 264 265 static int debug_mode(void) 266 { 267 #ifdef DEBUG 268 return 1; 269 #else 270 return 0; 271 #endif 272 } 273 274 static inline void kmem_cache_destroy_safe(struct kmem_cache **ppCache) 275 { 276 if (ppCache) 277 { 278 if (*ppCache) 279 kmem_cache_destroy(*ppCache); 280 281 *ppCache = NULL; 282 } 283 } 284 285 static const uid_t UVM_ROOT_UID = 0; 286 287 288 typedef struct 289 { 290 NvU64 start_time_ns; 291 NvU64 print_time_ns; 292 } uvm_spin_loop_t; 293 294 static inline void uvm_spin_loop_init(uvm_spin_loop_t *spin) 295 { 296 NvU64 curr = NV_GETTIME(); 297 spin->start_time_ns = curr; 298 spin->print_time_ns = curr; 299 } 300 301 // Periodically yields the CPU when not called from interrupt context. Returns 302 // NV_ERR_TIMEOUT_RETRY if the caller should print a warning that we've been 303 // waiting too long, and NV_OK otherwise. 304 NV_STATUS uvm_spin_loop(uvm_spin_loop_t *spin); 305 306 static NvU64 uvm_spin_loop_elapsed(const uvm_spin_loop_t *spin) 307 { 308 NvU64 curr = NV_GETTIME(); 309 return curr - spin->start_time_ns; 310 } 311 312 #define UVM_SPIN_LOOP(__spin) ({ \ 313 NV_STATUS __status = uvm_spin_loop(__spin); \ 314 if (__status == NV_ERR_TIMEOUT_RETRY) { \ 315 UVM_DBG_PRINT("Warning: stuck waiting for %llus\n", \ 316 uvm_spin_loop_elapsed(__spin) / (1000*1000*1000)); \ 317 \ 318 if (uvm_debug_prints_enabled()) \ 319 dump_stack(); \ 320 } \ 321 __status; \ 322 }) 323 324 // Execute the loop code while cond is true. Invokes uvm_spin_loop_iter at the 325 // end of each iteration. 326 #define UVM_SPIN_WHILE(cond, spin) \ 327 if (cond) \ 328 for (uvm_spin_loop_init(spin); (cond); UVM_SPIN_LOOP(spin)) 329 330 // 331 // Documentation for the internal routines listed below may be found in the 332 // implementation file(s). 333 // 334 NV_STATUS errno_to_nv_status(int errnoCode); 335 int nv_status_to_errno(NV_STATUS status); 336 unsigned uvm_get_stale_process_id(void); 337 unsigned uvm_get_stale_thread_id(void); 338 NvBool uvm_user_id_security_check(uid_t euidTarget); 339 340 extern int uvm_enable_builtin_tests; 341 342 static inline void uvm_init_character_device(struct cdev *cdev, const struct file_operations *fops) 343 { 344 cdev_init(cdev, fops); 345 cdev->owner = THIS_MODULE; 346 } 347 348 typedef struct 349 { 350 int rm_control_fd; 351 NvHandle user_client; 352 NvHandle user_object; 353 } uvm_rm_user_object_t; 354 355 typedef enum 356 { 357 UVM_FD_UNINITIALIZED, 358 UVM_FD_INITIALIZING, 359 UVM_FD_VA_SPACE, 360 UVM_FD_MM, 361 UVM_FD_COUNT 362 } uvm_fd_type_t; 363 364 // This should be large enough to fit the valid values from uvm_fd_type_t above. 365 // Note we can't use order_base_2(UVM_FD_COUNT) to define this because our code 366 // coverage tool fails due when the preprocessor expands that to a huge mess of 367 // ternary operators. 368 #define UVM_FD_TYPE_BITS 2 369 #define UVM_FD_TYPE_MASK ((1UL << UVM_FD_TYPE_BITS) - 1) 370 371 // Macro used to compare two values for types that support less than operator. 372 // It returns -1 if a < b, 1 if a > b and 0 if a == 0 373 #define UVM_CMP_DEFAULT(a,b) \ 374 ({ \ 375 typeof(a) _a = a; \ 376 typeof(b) _b = b; \ 377 int __ret; \ 378 BUILD_BUG_ON(sizeof(a) != sizeof(b)); \ 379 if (_a < _b) \ 380 __ret = -1; \ 381 else if (_b < _a) \ 382 __ret = 1; \ 383 else \ 384 __ret = 0; \ 385 \ 386 __ret; \ 387 }) 388 389 // Returns whether the input file was opened against the UVM character device 390 // file. A NULL input returns false. 391 bool uvm_file_is_nvidia_uvm(struct file *filp); 392 393 // Returns the type of data filp->private_data contains to and if ptr_val != 394 // NULL returns the value of the pointer. 395 uvm_fd_type_t uvm_fd_type(struct file *filp, void **ptr_val); 396 397 // Returns the pointer stored in filp->private_data if the type 398 // matches, otherwise returns NULL. 399 void *uvm_fd_get_type(struct file *filp, uvm_fd_type_t type); 400 401 // Reads the first word in the supplied struct page. 402 static inline void uvm_touch_page(struct page *page) 403 { 404 char *mapping; 405 406 UVM_ASSERT(page); 407 408 mapping = (char *) kmap(page); 409 (void)UVM_READ_ONCE(*mapping); 410 kunmap(page); 411 } 412 413 // Return true if the VMA is one used by UVM managed allocations. 414 bool uvm_vma_is_managed(struct vm_area_struct *vma); 415 416 #endif /* _UVM_COMMON_H */ 417