1 /******************************************************************************* 2 Copyright (c) 2013-2023 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #ifndef _UVM_COMMON_H 25 #define _UVM_COMMON_H 26 27 #ifdef DEBUG 28 #define UVM_IS_DEBUG() 1 29 #else 30 #define UVM_IS_DEBUG() 0 31 #endif 32 33 // NVIDIA_UVM_DEVELOP implies DEBUG, but not vice-versa 34 // TODO Bug 1773100: Figure out the right distinction between develop and debug 35 // builds. 36 #ifdef NVIDIA_UVM_DEVELOP 37 #define UVM_IS_DEVELOP() 1 38 #else 39 #define UVM_IS_DEVELOP() 0 40 #endif 41 42 #include "uvm_types.h" 43 #include "uvm_linux.h" 44 45 enum { 46 NVIDIA_UVM_PRIMARY_MINOR_NUMBER = 0, 47 NVIDIA_UVM_TOOLS_MINOR_NUMBER = 1, 48 // to ensure backward-compatiblity and correct counting, please insert any 49 // new minor devices just above the following field: 50 NVIDIA_UVM_NUM_MINOR_DEVICES 51 }; 52 53 #define UVM_GPU_UUID_TEXT_BUFFER_LENGTH (8+16*2+4+1) 54 55 int format_uuid_to_buffer(char *buffer, unsigned bufferLength, const NvProcessorUuid *pGpuUuid); 56 57 #define UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ...) \ 58 func(prefix "%s:%u %s[pid:%d]" fmt, \ 59 kbasename(__FILE__), \ 60 __LINE__, \ 61 __FUNCTION__, \ 62 current->pid, \ 63 ##__VA_ARGS__) 64 65 #define UVM_PRINT_FUNC(func, fmt, ...) \ 66 UVM_PRINT_FUNC_PREFIX(func, "", fmt, ##__VA_ARGS__) 67 68 // Check whether UVM_{ERR,DBG,INFO)_PRINT* should be enabled 69 bool uvm_debug_prints_enabled(void); 70 71 // A printing helper like UVM_PRINT_FUNC_PREFIX that only prints if 72 // uvm_debug_prints_enabled() returns true. 73 #define UVM_PRINT_FUNC_PREFIX_CHECK(func, prefix, fmt, ...) \ 74 do { \ 75 if (uvm_debug_prints_enabled()) { \ 76 UVM_PRINT_FUNC_PREFIX(func, prefix, fmt, ##__VA_ARGS__); \ 77 } \ 78 } while (0) 79 80 #define UVM_ASSERT_PRINT(fmt, ...) \ 81 UVM_PRINT_FUNC_PREFIX(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 82 83 #define UVM_ASSERT_PRINT_RL(fmt, ...) \ 84 UVM_PRINT_FUNC_PREFIX(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 85 86 #define UVM_ERR_PRINT(fmt, ...) \ 87 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 88 89 #define UVM_ERR_PRINT_RL(fmt, ...) \ 90 UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_ERR NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 91 92 #define UVM_DBG_PRINT(fmt, ...) \ 93 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 94 95 #define UVM_DBG_PRINT_RL(fmt, ...) \ 96 UVM_PRINT_FUNC_PREFIX_CHECK(printk_ratelimited, KERN_DEBUG NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 97 98 #define UVM_INFO_PRINT(fmt, ...) \ 99 UVM_PRINT_FUNC_PREFIX_CHECK(printk, KERN_INFO NVIDIA_UVM_PRETTY_PRINTING_PREFIX, " " fmt, ##__VA_ARGS__) 100 101 // 102 // Please see the documentation of format_uuid_to_buffer, for details on what 103 // this routine prints for you. 104 // 105 #define UVM_DBG_PRINT_UUID(msg, uuidPtr) \ 106 do { \ 107 char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH]; \ 108 format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \ 109 UVM_DBG_PRINT("%s: %s\n", msg, uuidBuffer); \ 110 } while (0) 111 112 #define UVM_ERR_PRINT_NV_STATUS(msg, rmStatus, ...) \ 113 UVM_ERR_PRINT("ERROR: %s : " msg "\n", nvstatusToString(rmStatus), ##__VA_ARGS__) 114 115 #define UVM_ERR_PRINT_UUID(msg, uuidPtr, ...) \ 116 do { \ 117 char uuidBuffer[UVM_GPU_UUID_TEXT_BUFFER_LENGTH]; \ 118 format_uuid_to_buffer(uuidBuffer, sizeof(uuidBuffer), uuidPtr); \ 119 UVM_ERR_PRINT("ERROR: %s : " msg "\n", uuidBuffer, ##__VA_ARGS__); \ 120 } while (0) 121 122 #define UVM_PANIC() UVM_PRINT_FUNC(panic, "\n") 123 #define UVM_PANIC_MSG(fmt, ...) UVM_PRINT_FUNC(panic, ": " fmt, ##__VA_ARGS__) 124 125 #define UVM_PANIC_ON_MSG(cond, fmt, ...) \ 126 do { \ 127 if (unlikely(cond)) \ 128 UVM_PANIC_MSG(fmt, ##__VA_ARGS__); \ 129 } while (0) 130 131 #define UVM_PANIC_ON(cond) UVM_PANIC_ON_MSG(cond, "failed cond %s\n", #cond) 132 133 // expr may include function calls. Use sizeof to prevent it from being 134 // evaluated while also preventing unused variable warnings. sizeof() can't be 135 // used on a bitfield however, so use ! to force the expression to evaluate as 136 // an int. 137 #define UVM_IGNORE_EXPR(expr) ((void)sizeof(!(expr))) 138 139 #define UVM_IGNORE_EXPR2(expr1, expr2) \ 140 do { \ 141 UVM_IGNORE_EXPR(expr1); \ 142 UVM_IGNORE_EXPR(expr2); \ 143 } while (0) 144 145 // NO-OP function to break on_uvm_test_fail - that is just to set a breakpoint 146 void on_uvm_test_fail(void); 147 148 // NO-OP function to break on_uvm_assert - that is just to set a breakpoint 149 // Unlike on_uvm_test_fail it provides 'panic' coverity semantics 150 void on_uvm_assert(void); 151 152 #define _UVM_ASSERT_MSG(expr, cond, fmt, ...) \ 153 do { \ 154 if (unlikely(!(expr))) { \ 155 UVM_ASSERT_PRINT("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \ 156 dump_stack(); \ 157 on_uvm_assert(); \ 158 } \ 159 } while (0) 160 161 // Prevent function calls in expr and the print argument list from being 162 // evaluated. 163 #define UVM_ASSERT_MSG_IGNORE(expr, fmt, ...) \ 164 do { \ 165 UVM_IGNORE_EXPR(expr); \ 166 UVM_NO_PRINT(fmt, ##__VA_ARGS__); \ 167 } while (0) 168 169 // UVM_ASSERT and UVM_ASSERT_MSG are only enabled on non-release and Coverity builds 170 #if UVM_IS_DEBUG() || defined __COVERITY__ 171 #define UVM_ASSERT_MSG(expr, fmt, ...) _UVM_ASSERT_MSG(expr, #expr, ": " fmt, ##__VA_ARGS__) 172 #define UVM_ASSERT(expr) _UVM_ASSERT_MSG(expr, #expr, "\n") 173 #else 174 #define UVM_ASSERT_MSG(expr, fmt, ...) UVM_ASSERT_MSG_IGNORE(expr, fmt, ##__VA_ARGS__) 175 #define UVM_ASSERT(expr) UVM_ASSERT_MSG_IGNORE(expr, "\n") 176 #endif 177 178 // UVM_ASSERT_RELEASE and UVM_ASSERT_MSG_RELEASE are always included in the 179 // build, even on release builds. They are skipped at runtime if 180 // uvm_release_asserts is 0. 181 182 // Whether release asserts are enabled and whether they should dump the stack 183 // and set the global error. 184 extern int uvm_release_asserts; 185 extern int uvm_release_asserts_dump_stack; 186 extern int uvm_release_asserts_set_global_error; 187 extern bool uvm_release_asserts_set_global_error_for_tests; 188 189 // Given these are enabled for release builds, we need to be more cautious than 190 // in UVM_ASSERT(). Use a ratelimited print and only dump the stack if a module 191 // param is enabled. 192 #define _UVM_ASSERT_MSG_RELEASE(expr, cond, fmt, ...) \ 193 do { \ 194 if (uvm_release_asserts && unlikely(!(expr))) { \ 195 UVM_ASSERT_PRINT_RL("Assert failed, condition %s not true" fmt, cond, ##__VA_ARGS__); \ 196 if (uvm_release_asserts_set_global_error || uvm_release_asserts_set_global_error_for_tests) \ 197 uvm_global_set_fatal_error(NV_ERR_INVALID_STATE); \ 198 if (uvm_release_asserts_dump_stack) \ 199 dump_stack(); \ 200 on_uvm_assert(); \ 201 } \ 202 } while (0) 203 204 #define UVM_ASSERT_MSG_RELEASE(expr, fmt, ...) _UVM_ASSERT_MSG_RELEASE(expr, #expr, ": " fmt, ##__VA_ARGS__) 205 #define UVM_ASSERT_RELEASE(expr) _UVM_ASSERT_MSG_RELEASE(expr, #expr, "\n") 206 207 // Provide a short form of UUID's, typically for use in debug printing: 208 #define ABBREV_UUID(uuid) (unsigned)(uuid) 209 210 static inline NvBool uvm_uuid_is_cpu(const NvProcessorUuid *uuid) 211 { 212 return memcmp(uuid, &NV_PROCESSOR_UUID_CPU_DEFAULT, sizeof(*uuid)) == 0; 213 } 214 #define UVM_SIZE_1KB (1024ULL) 215 #define UVM_SIZE_1MB (1024 * UVM_SIZE_1KB) 216 #define UVM_SIZE_1GB (1024 * UVM_SIZE_1MB) 217 #define UVM_SIZE_1TB (1024 * UVM_SIZE_1GB) 218 #define UVM_SIZE_1PB (1024 * UVM_SIZE_1TB) 219 220 #define UVM_ALIGN_DOWN(x, a) ({ \ 221 typeof(x) _a = a; \ 222 UVM_ASSERT(is_power_of_2(_a)); \ 223 (x) & ~(_a - 1); \ 224 }) 225 226 #define UVM_ALIGN_UP(x, a) ({ \ 227 typeof(x) _a = a; \ 228 UVM_ASSERT(is_power_of_2(_a)); \ 229 ((x) + _a - 1) & ~(_a - 1); \ 230 }) 231 232 #define UVM_PAGE_ALIGN_UP(value) UVM_ALIGN_UP(value, PAGE_SIZE) 233 #define UVM_PAGE_ALIGN_DOWN(value) UVM_ALIGN_DOWN(value, PAGE_SIZE) 234 235 // These macros provide a convenient way to string-ify enum values. 236 #define UVM_ENUM_STRING_CASE(value) case value: return #value 237 #define UVM_ENUM_STRING_DEFAULT() default: return "UNKNOWN" 238 239 // Divide by a dynamic value known at runtime to be a power of 2. ilog2 is 240 // optimized as a single instruction in many processors, whereas integer 241 // division is always slow. 242 static inline NvU32 uvm_div_pow2_32(NvU32 numerator, NvU32 denominator_pow2) 243 { 244 UVM_ASSERT(is_power_of_2(denominator_pow2)); 245 UVM_ASSERT(denominator_pow2); 246 return numerator >> ilog2(denominator_pow2); 247 } 248 249 static inline NvU64 uvm_div_pow2_64(NvU64 numerator, NvU64 denominator_pow2) 250 { 251 UVM_ASSERT(is_power_of_2(denominator_pow2)); 252 UVM_ASSERT(denominator_pow2); 253 return numerator >> ilog2(denominator_pow2); 254 } 255 256 #define SUM_FROM_0_TO_N(n) (((n) * ((n) + 1)) / 2) 257 258 // Start and end are inclusive 259 static inline NvBool uvm_ranges_overlap(NvU64 a_start, NvU64 a_end, NvU64 b_start, NvU64 b_end) 260 { 261 // De Morgan's of: !(a_end < b_start || b_end < a_start) 262 return a_end >= b_start && b_end >= a_start; 263 } 264 265 static int debug_mode(void) 266 { 267 #ifdef DEBUG 268 return 1; 269 #else 270 return 0; 271 #endif 272 } 273 274 static inline void kmem_cache_destroy_safe(struct kmem_cache **ppCache) 275 { 276 if (ppCache) 277 { 278 if (*ppCache) 279 kmem_cache_destroy(*ppCache); 280 281 *ppCache = NULL; 282 } 283 } 284 285 typedef struct 286 { 287 NvU64 start_time_ns; 288 NvU64 print_time_ns; 289 } uvm_spin_loop_t; 290 291 static inline void uvm_spin_loop_init(uvm_spin_loop_t *spin) 292 { 293 NvU64 curr = NV_GETTIME(); 294 spin->start_time_ns = curr; 295 spin->print_time_ns = curr; 296 } 297 298 // Periodically yields the CPU when not called from interrupt context. Returns 299 // NV_ERR_TIMEOUT_RETRY if the caller should print a warning that we've been 300 // waiting too long, and NV_OK otherwise. 301 NV_STATUS uvm_spin_loop(uvm_spin_loop_t *spin); 302 303 static NvU64 uvm_spin_loop_elapsed(const uvm_spin_loop_t *spin) 304 { 305 NvU64 curr = NV_GETTIME(); 306 return curr - spin->start_time_ns; 307 } 308 309 #define UVM_SPIN_LOOP(__spin) ({ \ 310 NV_STATUS __status = uvm_spin_loop(__spin); \ 311 if (__status == NV_ERR_TIMEOUT_RETRY) { \ 312 UVM_DBG_PRINT("Warning: stuck waiting for %llus\n", \ 313 uvm_spin_loop_elapsed(__spin) / (1000*1000*1000)); \ 314 \ 315 if (uvm_debug_prints_enabled()) \ 316 dump_stack(); \ 317 } \ 318 __status; \ 319 }) 320 321 // Execute the loop code while cond is true. Invokes uvm_spin_loop_iter at the 322 // end of each iteration. 323 #define UVM_SPIN_WHILE(cond, spin) \ 324 if (cond) \ 325 for (uvm_spin_loop_init(spin); (cond); UVM_SPIN_LOOP(spin)) 326 327 // 328 // Documentation for the internal routines listed below may be found in the 329 // implementation file(s). 330 // 331 NV_STATUS errno_to_nv_status(int errnoCode); 332 int nv_status_to_errno(NV_STATUS status); 333 unsigned uvm_get_stale_process_id(void); 334 unsigned uvm_get_stale_thread_id(void); 335 336 extern int uvm_enable_builtin_tests; 337 338 static inline void uvm_init_character_device(struct cdev *cdev, const struct file_operations *fops) 339 { 340 cdev_init(cdev, fops); 341 cdev->owner = THIS_MODULE; 342 } 343 344 typedef struct 345 { 346 int rm_control_fd; 347 NvHandle user_client; 348 NvHandle user_object; 349 } uvm_rm_user_object_t; 350 351 typedef enum 352 { 353 UVM_FD_UNINITIALIZED, 354 UVM_FD_INITIALIZING, 355 UVM_FD_VA_SPACE, 356 UVM_FD_MM, 357 UVM_FD_COUNT 358 } uvm_fd_type_t; 359 360 // This should be large enough to fit the valid values from uvm_fd_type_t above. 361 // Note we can't use order_base_2(UVM_FD_COUNT) to define this because our code 362 // coverage tool fails due when the preprocessor expands that to a huge mess of 363 // ternary operators. 364 #define UVM_FD_TYPE_BITS 2 365 #define UVM_FD_TYPE_MASK ((1UL << UVM_FD_TYPE_BITS) - 1) 366 367 // Macro used to compare two values for types that support less than operator. 368 // It returns -1 if a < b, 1 if a > b and 0 if a == 0 369 #define UVM_CMP_DEFAULT(a,b) \ 370 ({ \ 371 typeof(a) _a = a; \ 372 typeof(b) _b = b; \ 373 int __ret; \ 374 BUILD_BUG_ON(sizeof(a) != sizeof(b)); \ 375 if (_a < _b) \ 376 __ret = -1; \ 377 else if (_b < _a) \ 378 __ret = 1; \ 379 else \ 380 __ret = 0; \ 381 \ 382 __ret; \ 383 }) 384 385 // Returns whether the input file was opened against the UVM character device 386 // file. A NULL input returns false. 387 bool uvm_file_is_nvidia_uvm(struct file *filp); 388 389 // Returns the type of data filp->private_data contains to and if ptr_val != 390 // NULL returns the value of the pointer. 391 uvm_fd_type_t uvm_fd_type(struct file *filp, void **ptr_val); 392 393 // Returns the pointer stored in filp->private_data if the type 394 // matches, otherwise returns NULL. 395 void *uvm_fd_get_type(struct file *filp, uvm_fd_type_t type); 396 397 // Reads the first word in the supplied struct page. 398 static inline void uvm_touch_page(struct page *page) 399 { 400 char *mapping; 401 402 UVM_ASSERT(page); 403 404 mapping = (char *) kmap(page); 405 (void)UVM_READ_ONCE(*mapping); 406 kunmap(page); 407 } 408 409 // Return true if the VMA is one used by UVM managed allocations. 410 bool uvm_vma_is_managed(struct vm_area_struct *vma); 411 412 #endif /* _UVM_COMMON_H */ 413