1 /*******************************************************************************
2 Copyright (c) 2015-2023 NVIDIA Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to
6 deal in the Software without restriction, including without limitation the
7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8 sell copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 DEALINGS IN THE SOFTWARE.
21
22 *******************************************************************************/
23
24 #include "uvm_forward_decl.h"
25 #include "uvm_thread_context.h"
26
27 #include "uvm_linux.h"
28 #include "uvm_common.h"
29
30 // Thread local storage implementation.
31 //
32 // The global data structure that contains the set of active thread contexts
33 // is a table of UVM_THREAD_CONTEXT_TABLE_SIZE entries of type
34 // uvm_thread_context_table_entry_t.
35 // Each entry contains a small array of UVM_THREAD_CONTEXT_ARRAY_SIZE entries,
36 // a red-black tree, and a lock protecting the tree.
37 //
38 // The thread_context_non_interrupt_table_entry() function maps the current task
39 // (i.e. the current thread context) to a table entry. That function also
40 // recommends a position within the entry's array, but that index can be safely
41 // ignored: the thread context can be located in any array slot, or in the
42 // red-black tree.
43 //
44 // The described global data structures try to minimize contention among
45 // threads at two levels. First, thread_context_non_interrupt_table_entry()
46 // relies on a hash function to evenly spread threads among table entries.
47 // Second, when several threads are mapped to the same table entry, the same
48 // hash function spreads them evenly among the array entries, which can
49 // be independently and atomically updated. If the array is full, the thread
50 // context of the current task is stored in the red-black tree of the table
51 // entry, which is protected by a single lock.
52 //
53 // Both the table and array entries are cache aligned to avoid false sharing
54 // overheads due to cache thrashing between concurrent operations on separate
55 // thread contexts.
56
57 #define UVM_THREAD_CONTEXT_ARRAY_SIZE 8
58
59 typedef struct {
60 void *acquired[UVM_LOCK_ORDER_COUNT];
61 } uvm_thread_context_lock_acquired_t;
62
63 typedef struct {
64 // If zero, the entry is empty. Otherwise, task is equal to the value of
65 // get_current() for the thread associated with thread_context;
66 atomic64_t task;
67
68 uvm_thread_context_t *thread_context;
69 } ____cacheline_aligned_in_smp uvm_thread_context_array_entry_t;
70
71 // The thread's context information is stored in the array or the red-black
72 // tree.
73 typedef struct {
74 // Small array where thread contexts are stored first. Each array entry
75 // can be atomically claimed or released.
76 uvm_thread_context_array_entry_t array[UVM_THREAD_CONTEXT_ARRAY_SIZE];
77
78 // Red-black tree, used when the array is full. A red-black tree is chosen
79 // because additions and removals are frequent operations: every time the
80 // UVM module is entered, there is one addition, one removal, and one
81 // lookup. The same UVM call may result on additional lookups.
82 struct rb_root tree;
83
84 // Spinlock protecting the tree. A raw lock is chosen because UVM locks
85 // rely on thread context information to be available for lock tracking.
86 spinlock_t tree_lock;
87 } ____cacheline_aligned_in_smp uvm_thread_context_table_entry_t;
88
89 // Global data structure containing all the active thread contexts
90 static uvm_thread_context_table_entry_t g_thread_context_table[UVM_THREAD_CONTEXT_TABLE_SIZE];
91
92 static bool g_thread_context_table_initialized __read_mostly = false;
93
94 // Per CPU context wrapper, used for interrupt context. Zero initialized.
95 static DEFINE_PER_CPU(uvm_thread_context_wrapper_t, interrupt_thread_context_wrapper);
96
97 // Array of acquired locks, used in the interrupt path. The non-interrupt path
98 // allocates the array when locking for the first time.
99 static DEFINE_PER_CPU(uvm_thread_context_lock_acquired_t, interrupt_thread_context_lock_acquired);
100
101 static void thread_context_non_interrupt_remove(uvm_thread_context_t *thread_context,
102 uvm_thread_context_table_entry_t *thread_context_entry);
103
uvm_thread_context_wrapper_is_used(void)104 bool uvm_thread_context_wrapper_is_used(void)
105 {
106 // The wrapper contains lock information. While uvm_record_lock_X
107 // routines are a no-op outside of debug mode, unit tests do invoke their
108 // internal counterparts __uvm_record_lock_X. To add coverage, lock
109 // information is made available in develop and release modes if the
110 // builtin tests are enabled.
111 return UVM_IS_DEBUG() || uvm_enable_builtin_tests;
112 }
113
uvm_thread_context_global_initialized(void)114 bool uvm_thread_context_global_initialized(void)
115 {
116 return g_thread_context_table_initialized;
117 }
118
uvm_thread_context_global_init(void)119 void uvm_thread_context_global_init(void)
120 {
121 size_t table_index;
122
123 UVM_ASSERT(!uvm_thread_context_global_initialized());
124
125 for (table_index = 0; table_index < UVM_THREAD_CONTEXT_TABLE_SIZE; table_index++) {
126 uvm_thread_context_table_entry_t *table_entry = g_thread_context_table + table_index;
127
128 spin_lock_init(&table_entry->tree_lock);
129 table_entry->tree = RB_ROOT;
130 }
131
132 g_thread_context_table_initialized = true;
133 }
134
uvm_thread_context_global_exit(void)135 void uvm_thread_context_global_exit(void)
136 {
137 size_t table_index;
138 uvm_thread_context_t *curr_thread_context = uvm_thread_context();
139
140 UVM_ASSERT(uvm_thread_context_global_initialized());
141
142 // Search for thread contexts that were added but never removed.
143 // There should be one thread context left: the one added by the UVM module
144 // exit routine that invoked this function. To prevent the exit routine from
145 // deleting its thread context after deinitialization of the global table,
146 // it is deleted here. uvm_thread_context_remove will detect that the global
147 // shutdown already happened and skip.
148 for (table_index = 0; table_index < UVM_THREAD_CONTEXT_TABLE_SIZE; table_index++) {
149 size_t array_index;
150 struct rb_node *node;
151 uvm_thread_context_table_entry_t *table_entry = g_thread_context_table + table_index;
152
153 for (array_index = 0; array_index < UVM_THREAD_CONTEXT_ARRAY_SIZE; array_index++) {
154 uvm_thread_context_t *thread_context;
155 uvm_thread_context_array_entry_t *array_entry = table_entry->array + array_index;
156
157 NvU64 task = atomic64_read(&array_entry->task);
158
159 if (task == 0)
160 continue;
161
162 thread_context = array_entry->thread_context;
163
164 UVM_ASSERT_MSG(thread_context == curr_thread_context,
165 "Left-over thread_context 0x%llx task 0x%llx\n",
166 (NvU64) thread_context,
167 (NvU64) thread_context->task);
168
169 thread_context_non_interrupt_remove(thread_context, table_entry);
170 }
171
172 node = rb_first(&table_entry->tree);
173
174 while (node) {
175 uvm_thread_context_t *thread_context = rb_entry(node, uvm_thread_context_t, node);
176
177 UVM_ASSERT_MSG(thread_context == curr_thread_context,
178 "Left-over thread_context 0x%llx task 0x%llx\n",
179 (NvU64) thread_context,
180 (NvU64) thread_context->task);
181
182 thread_context_non_interrupt_remove(thread_context, table_entry);
183 node = rb_first(&table_entry->tree);
184 }
185 }
186
187 g_thread_context_table_initialized = false;
188 }
189
thread_context_non_interrupt_tree_search(struct rb_root * root,struct task_struct * task)190 static uvm_thread_context_t *thread_context_non_interrupt_tree_search(struct rb_root *root, struct task_struct *task)
191 {
192 struct rb_node *node = root->rb_node;
193 uintptr_t task_uintptr = (uintptr_t) task;
194
195 while (node) {
196 uvm_thread_context_t *thread_context = rb_entry(node, uvm_thread_context_t, node);
197 uintptr_t thread_context_task_uintptr = (uintptr_t) thread_context->task;
198
199 if (thread_context_task_uintptr == task_uintptr)
200 return thread_context;
201
202 node = (thread_context_task_uintptr > task_uintptr)? node->rb_left : node->rb_right;
203 }
204
205 return NULL;
206 }
207
thread_context_non_interrupt_tree_insert(struct rb_root * root,uvm_thread_context_t * new_thread_context)208 static bool thread_context_non_interrupt_tree_insert(struct rb_root *root, uvm_thread_context_t *new_thread_context)
209 {
210 struct rb_node **node_ptr = &root->rb_node;
211 struct rb_node *node = root->rb_node;
212 struct rb_node *parent = NULL;
213 const struct task_struct *task = new_thread_context->task;
214 uintptr_t task_uintptr = (uintptr_t) task;
215
216 while (node) {
217 uvm_thread_context_t *thread_context = rb_entry(node, uvm_thread_context_t, node);
218 uintptr_t thread_context_task_uintptr = (uintptr_t) thread_context->task;
219
220 if (thread_context_task_uintptr == task_uintptr)
221 return false;
222
223 parent = node;
224 node_ptr = (thread_context_task_uintptr > task_uintptr) ? &node->rb_left : &node->rb_right;
225 node = *node_ptr;
226 }
227
228 rb_link_node(&new_thread_context->node, parent, node_ptr);
229 rb_insert_color(&new_thread_context->node, root);
230
231 return true;
232 }
233
thread_context_lock_interrupt_patch_acquired(uvm_thread_context_lock_t * context_lock)234 static void thread_context_lock_interrupt_patch_acquired(uvm_thread_context_lock_t *context_lock)
235 {
236 uvm_thread_context_lock_acquired_t *thread_context_lock_acquired;
237
238 UVM_ASSERT(in_interrupt());
239 UVM_ASSERT(context_lock->acquired == NULL);
240
241 // Stich the preallocated, per-CPU array to the thread context lock.
242 thread_context_lock_acquired = &get_cpu_var(interrupt_thread_context_lock_acquired);
243 put_cpu_var(interrupt_thread_context_lock_acquired);
244 context_lock->acquired = (void**) thread_context_lock_acquired;
245 }
246
thread_context_lock_of(uvm_thread_context_t * thread_context)247 static uvm_thread_context_lock_t *thread_context_lock_of(uvm_thread_context_t *thread_context)
248 {
249 uvm_thread_context_wrapper_t *thread_context_wrapper;
250 uvm_thread_context_lock_t *context_lock;
251
252 if (!uvm_thread_context_wrapper_is_used())
253 return NULL;
254
255 thread_context_wrapper = container_of(thread_context, uvm_thread_context_wrapper_t, context);
256 context_lock = &thread_context_wrapper->context_lock;
257
258 // When the wrapper is used, the thread context lock is always present but
259 // its acquired locks array may not, due to a failed allocation. Instead of
260 // working around the missing array, pretend that the entire lock context
261 // does not exist. This situation can only happen in non-interrupt paths.
262 if (context_lock->acquired == NULL) {
263 if (in_interrupt())
264 thread_context_lock_interrupt_patch_acquired(context_lock);
265 else
266 return NULL;
267 }
268
269 return context_lock;
270 }
271
thread_context_non_interrupt_init(uvm_thread_context_t * thread_context)272 static void thread_context_non_interrupt_init(uvm_thread_context_t *thread_context)
273 {
274 UVM_ASSERT(!in_interrupt());
275
276 thread_context->array_index = UVM_THREAD_CONTEXT_ARRAY_SIZE;
277
278 if (uvm_thread_context_wrapper_is_used()) {
279 uvm_thread_context_wrapper_t *thread_context_wrapper;
280 uvm_thread_context_lock_t *context_lock;
281
282 thread_context_wrapper = container_of(thread_context, uvm_thread_context_wrapper_t, context);
283 context_lock = &thread_context_wrapper->context_lock;
284
285 memset(context_lock, 0, sizeof(*context_lock));
286
287 // If this allocation fails, the lock context will appear as not
288 // present, but the rest of the thread context is usable.
289 context_lock->acquired = kmalloc(sizeof(context_lock->acquired[0]) * UVM_LOCK_ORDER_COUNT, NV_UVM_GFP_FLAGS);
290 }
291 }
292
thread_context_non_interrupt_deinit(uvm_thread_context_t * thread_context)293 static void thread_context_non_interrupt_deinit(uvm_thread_context_t *thread_context)
294 {
295 uvm_thread_context_lock_t *context_lock;
296
297 UVM_ASSERT(!in_interrupt());
298
299 context_lock = thread_context_lock_of(thread_context);
300 if (context_lock != NULL) {
301 UVM_ASSERT(__uvm_check_all_unlocked(context_lock));
302
303 kfree(context_lock->acquired);
304 context_lock->acquired = NULL;
305 }
306 }
307
308 // Return the table entry and array index within that entry where the thread
309 // context of the current task is located.
310 //
311 // The array index should be interpreted as a hint: the thread context of the
312 // current taks may be stored at a different array index, or in the tree.
thread_context_non_interrupt_table_entry(size_t * array_index_hint)313 static uvm_thread_context_table_entry_t *thread_context_non_interrupt_table_entry(size_t *array_index_hint)
314 {
315 size_t table_index;
316 NvU64 current_ptr = (NvU64) current;
317 NvU32 hash = jhash_2words((NvU32) current_ptr, (NvU32) (current_ptr >> 32), 0);
318
319 BUILD_BUG_ON(UVM_THREAD_CONTEXT_TABLE_SIZE > (1 << 16));
320 BUILD_BUG_ON(UVM_THREAD_CONTEXT_ARRAY_SIZE > (1 << 16));
321 UVM_ASSERT(!in_interrupt());
322
323 // The upper 16 bits of the hash value index the table; the lower 16
324 // index the array
325 table_index = (hash >> 16) % UVM_THREAD_CONTEXT_TABLE_SIZE;
326
327 if (array_index_hint != NULL)
328 *array_index_hint = hash % UVM_THREAD_CONTEXT_ARRAY_SIZE;
329
330 return g_thread_context_table + table_index;
331 }
332
thread_context_non_interrupt(void)333 static uvm_thread_context_t *thread_context_non_interrupt(void)
334 {
335 unsigned long flags;
336 size_t i, array_index;
337 uvm_thread_context_t *thread_context;
338 uvm_thread_context_table_entry_t *table_entry = thread_context_non_interrupt_table_entry(&array_index);
339
340 for (i = array_index; i < (UVM_THREAD_CONTEXT_ARRAY_SIZE + array_index); i++) {
341 size_t curr_array_index = i % UVM_THREAD_CONTEXT_ARRAY_SIZE;
342 uvm_thread_context_array_entry_t *array_entry = table_entry->array + curr_array_index;
343
344 if (atomic64_read(&array_entry->task) == (NvU64) current) {
345 thread_context = array_entry->thread_context;
346
347 UVM_ASSERT(thread_context != NULL);
348 UVM_ASSERT(thread_context->array_index == curr_array_index);
349
350 return thread_context;
351 }
352 }
353
354 spin_lock_irqsave(&table_entry->tree_lock, flags);
355 thread_context = thread_context_non_interrupt_tree_search(&table_entry->tree, current);
356 spin_unlock_irqrestore(&table_entry->tree_lock, flags);
357
358 return thread_context;
359 }
360
thread_context_interrupt(void)361 static uvm_thread_context_t *thread_context_interrupt(void)
362 {
363 uvm_thread_context_wrapper_t *thread_context_wrapper;
364
365 // As we are in interrupt anyway it would be best to just use this_cpu_ptr()
366 // but it was added in 2.6.33 and the interface is non-trivial to implement
367 // prior to that.
368 thread_context_wrapper = &get_cpu_var(interrupt_thread_context_wrapper);
369 put_cpu_var(interrupt_thread_context_wrapper);
370
371 return &thread_context_wrapper->context;
372 }
373
thread_context_current(void)374 static uvm_thread_context_t *thread_context_current(void)
375 {
376 return in_interrupt() ? thread_context_interrupt() : thread_context_non_interrupt();
377 }
378
uvm_thread_context_present(void)379 bool uvm_thread_context_present(void)
380 {
381 return thread_context_current() != NULL;
382 }
383
uvm_thread_context(void)384 uvm_thread_context_t *uvm_thread_context(void)
385 {
386 uvm_thread_context_t *thread_context = thread_context_current();
387
388 // If this assertion fires is probably because an entry point into the
389 // UVM module has not been wrapped with a UVM_ENTRY_X macro. The entry point
390 // to wrap is the first nvidia-uvm function in the error call stack printed
391 // by the assertion.
392 UVM_ASSERT(thread_context != NULL);
393
394 return thread_context;
395 }
396
397 // The addition logic takes into account that there may be a different thread
398 // context already associated with the given task. This happens in the uncommon
399 // case of re-entering the UVM module. Therefore, it is worth approaching the
400 // addition in a optimistic (speculative) fashion: if a slot is empty in the
401 // array, it is immediately taken. Should we discover later on that the task
402 // already has a thread context associated with it in the rest of the array or
403 // the tree, the previously claimed array slot is released.
thread_context_non_interrupt_add(uvm_thread_context_t * thread_context,uvm_thread_context_table_entry_t * table_entry,size_t array_index_hint)404 static bool thread_context_non_interrupt_add(uvm_thread_context_t *thread_context,
405 uvm_thread_context_table_entry_t *table_entry,
406 size_t array_index_hint)
407 {
408 size_t i;
409 NvU64 task;
410 unsigned long flags;
411 bool added;
412
413 UVM_ASSERT(!in_interrupt());
414 UVM_ASSERT(thread_context != NULL);
415 UVM_ASSERT(table_entry != NULL);
416 UVM_ASSERT(table_entry - g_thread_context_table >= 0);
417 UVM_ASSERT(table_entry - g_thread_context_table < UVM_THREAD_CONTEXT_TABLE_SIZE);
418 UVM_ASSERT(array_index_hint < UVM_THREAD_CONTEXT_ARRAY_SIZE);
419
420 thread_context_non_interrupt_init(thread_context);
421 UVM_ASSERT(thread_context->array_index == UVM_THREAD_CONTEXT_ARRAY_SIZE);
422
423 task = (NvU64) thread_context->task;
424 UVM_ASSERT(task > 0);
425
426 for (i = array_index_hint; i < (array_index_hint + UVM_THREAD_CONTEXT_ARRAY_SIZE); i++) {
427 const size_t curr_array_index = i % UVM_THREAD_CONTEXT_ARRAY_SIZE;
428 uvm_thread_context_array_entry_t *array_entry = table_entry->array + curr_array_index;
429
430 if (thread_context->array_index == UVM_THREAD_CONTEXT_ARRAY_SIZE) {
431 NvU64 old = atomic64_cmpxchg(&array_entry->task, 0, task);
432
433 // Task already added a different thread context. The current thread
434 // context has not been inserted but needs to be freed.
435 if (old == task) {
436 thread_context_non_interrupt_deinit(thread_context);
437 return false;
438 }
439
440 // Speculatively add the current thread context.
441 if (old == 0)
442 thread_context->array_index = curr_array_index;
443 }
444 else if (atomic64_read(&array_entry->task) == task) {
445
446 // Task already added a different thread context to the array, so
447 // undo the speculative insertion
448 atomic64_set(&table_entry->array[thread_context->array_index].task, 0);
449 thread_context_non_interrupt_deinit(thread_context);
450
451 return false;
452 }
453 }
454
455 spin_lock_irqsave(&table_entry->tree_lock, flags);
456
457 if (thread_context->array_index == UVM_THREAD_CONTEXT_ARRAY_SIZE) {
458
459 // If the task already added a different thread context to the tree,
460 // there is nothing to undo because the current thread context has not
461 // been inserted.
462 added = thread_context_non_interrupt_tree_insert(&table_entry->tree, thread_context);
463 }
464 else if (thread_context_non_interrupt_tree_search(&table_entry->tree, thread_context->task) != NULL) {
465
466 // Task already added a different thread context to the tree, so undo
467 // the speculative insertion
468 atomic64_set(&table_entry->array[thread_context->array_index].task, 0);
469
470 added = false;
471 }
472 else {
473
474 // Speculative insertion succeeded: a thread context associated with the
475 // same task has not been found in the array or the tree.
476 table_entry->array[thread_context->array_index].thread_context = thread_context;
477 added = true;
478 }
479
480 if (!added)
481 thread_context_non_interrupt_deinit(thread_context);
482
483 spin_unlock_irqrestore(&table_entry->tree_lock, flags);
484 return added;
485 }
486
uvm_thread_context_add(uvm_thread_context_t * thread_context)487 bool uvm_thread_context_add(uvm_thread_context_t *thread_context)
488 {
489 uvm_thread_context_table_entry_t *table_entry;
490 size_t array_index;
491
492 UVM_ASSERT(thread_context != NULL);
493 UVM_ASSERT(!in_interrupt());
494
495 // Initialize the thread context table. This can only happen when loading
496 // the UVM module
497 if (!uvm_thread_context_global_initialized())
498 uvm_thread_context_global_init();
499
500 thread_context->task = current;
501 thread_context->ignore_hmm_invalidate_va_block = NULL;
502 table_entry = thread_context_non_interrupt_table_entry(&array_index);
503 return thread_context_non_interrupt_add(thread_context, table_entry, array_index);
504 }
505
uvm_thread_context_add_at(uvm_thread_context_t * thread_context,size_t table_index)506 bool uvm_thread_context_add_at(uvm_thread_context_t *thread_context, size_t table_index)
507 {
508 uvm_thread_context_table_entry_t *table_entry;
509
510 UVM_ASSERT(uvm_enable_builtin_tests != 0);
511 UVM_ASSERT(uvm_thread_context_global_initialized());
512
513 table_entry = g_thread_context_table + table_index;
514 return thread_context_non_interrupt_add(thread_context, table_entry, 0);
515 }
516
thread_context_non_interrupt_remove(uvm_thread_context_t * thread_context,uvm_thread_context_table_entry_t * table_entry)517 static void thread_context_non_interrupt_remove(uvm_thread_context_t *thread_context,
518 uvm_thread_context_table_entry_t *table_entry)
519 {
520 NvU32 array_index;
521
522 UVM_ASSERT(!in_interrupt());
523 UVM_ASSERT(thread_context != NULL);
524 UVM_ASSERT(table_entry != NULL);
525 UVM_ASSERT(table_entry - g_thread_context_table >= 0);
526 UVM_ASSERT(table_entry - g_thread_context_table < UVM_THREAD_CONTEXT_TABLE_SIZE);
527
528 array_index = thread_context->array_index;
529 UVM_ASSERT(array_index <= UVM_THREAD_CONTEXT_ARRAY_SIZE);
530
531 // We cannot use RB_EMPTY_NODE to determine if the thread context is in the
532 // tree, because the tree lock is not held and we haven't called RB_CLEAR_NODE.
533 // If the thread context is indeed in the tree, concurrent operations on
534 // the parent pointer/color of the thread context's node could result in
535 // RB_EMPTY_NODE(thread_context->node) being true.
536 if (array_index != UVM_THREAD_CONTEXT_ARRAY_SIZE) {
537
538 uvm_thread_context_array_entry_t *array_entry = table_entry->array + array_index;
539
540 UVM_ASSERT(array_index < UVM_THREAD_CONTEXT_ARRAY_SIZE);
541 UVM_ASSERT(atomic64_read(&array_entry->task) == (NvU64) thread_context->task);
542
543 // Clear the task. The memory barrier prevents the write from being
544 // moved before a previous (in program order) write to the entry's
545 // thread_context field in thread_context_non_interrupt_add.
546 //
547 // A more detailed explanation about why the memory barrier is needed
548 // before an atomic write, and why we are not using a different flavor
549 // of atomic write such as atomic64_set_release, can be found in
550 // uvm_gpu_semaphore.c:update_completed_value_locked().
551 smp_mb__before_atomic();
552 atomic64_set(&array_entry->task, 0);
553 }
554 else {
555 unsigned long flags;
556
557 spin_lock_irqsave(&table_entry->tree_lock, flags);
558 rb_erase(&thread_context->node, &table_entry->tree);
559 spin_unlock_irqrestore(&table_entry->tree_lock, flags);
560 }
561
562 thread_context_non_interrupt_deinit(thread_context);
563 }
564
uvm_thread_context_remove(uvm_thread_context_t * thread_context)565 void uvm_thread_context_remove(uvm_thread_context_t *thread_context)
566 {
567 uvm_thread_context_table_entry_t *table_entry;
568
569 UVM_ASSERT(thread_context != NULL);
570 UVM_ASSERT(!in_interrupt());
571
572 // If the thread context table has been deinitialized, then we must be in
573 // the UVM module unload path, and the thread context added during the call
574 // of uvm_exit has already been removed in the global deinitialization.
575 if (!uvm_thread_context_global_initialized())
576 return;
577
578 UVM_ASSERT(thread_context->task == current);
579 UVM_ASSERT(uvm_thread_context() == thread_context);
580
581 table_entry = thread_context_non_interrupt_table_entry(NULL);
582 thread_context_non_interrupt_remove(thread_context, table_entry);
583 }
584
uvm_thread_context_remove_at(uvm_thread_context_t * thread_context,size_t table_index)585 void uvm_thread_context_remove_at(uvm_thread_context_t *thread_context, size_t table_index)
586 {
587 uvm_thread_context_table_entry_t *table_entry = g_thread_context_table + table_index;
588
589 UVM_ASSERT(uvm_enable_builtin_tests != 0);
590
591 thread_context_non_interrupt_remove(thread_context, table_entry);
592 }
593
594 // Move operation
595 // -Lock information is copied to the destination, and cleared in the source.
596 // -Locations in the global array or tree are not copied nor cleared, since
597 // they may be needed for a later removal of the source, and are no longer
598 // valid after it.
599 // -When adding new members to the thread context, consider if they need to be
600 // moved
thread_context_move(uvm_thread_context_t * dst,uvm_thread_context_t * src)601 static void thread_context_move(uvm_thread_context_t *dst, uvm_thread_context_t *src)
602 {
603 uvm_thread_context_lock_t *src_context_lock, *dst_context_lock;
604
605 UVM_ASSERT(uvm_enable_builtin_tests != 0);
606
607 src_context_lock = thread_context_lock_of(src);
608 dst_context_lock = thread_context_lock_of(dst);
609
610 if ((dst_context_lock != NULL) && (src_context_lock != NULL)) {
611 size_t acquired_size = sizeof(src_context_lock->acquired[0]) * UVM_LOCK_ORDER_COUNT;
612
613 dst_context_lock->skip_lock_tracking = src_context_lock->skip_lock_tracking;
614 src_context_lock->skip_lock_tracking = false;
615
616 // Note that the locks are not released, even when they appear as such
617 // if we query the source thread context. They are still acquired in the
618 // destination context.
619 bitmap_copy(dst_context_lock->acquired_lock_orders,
620 src_context_lock->acquired_lock_orders,
621 UVM_LOCK_ORDER_COUNT);
622 bitmap_zero(src_context_lock->acquired_lock_orders, UVM_LOCK_ORDER_COUNT);
623
624 bitmap_copy(dst_context_lock->exclusive_acquired_lock_orders,
625 src_context_lock->exclusive_acquired_lock_orders,
626 UVM_LOCK_ORDER_COUNT);
627 bitmap_zero(src_context_lock->exclusive_acquired_lock_orders, UVM_LOCK_ORDER_COUNT);
628
629 bitmap_copy(dst_context_lock->out_of_order_acquired_lock_orders,
630 src_context_lock->out_of_order_acquired_lock_orders,
631 UVM_LOCK_ORDER_COUNT);
632 bitmap_zero(src_context_lock->out_of_order_acquired_lock_orders, UVM_LOCK_ORDER_COUNT);
633
634 memcpy(dst_context_lock->acquired, src_context_lock->acquired, acquired_size);
635 }
636 }
637
uvm_thread_context_save(uvm_thread_context_t * dst)638 void uvm_thread_context_save(uvm_thread_context_t *dst)
639 {
640 thread_context_non_interrupt_init(dst);
641 thread_context_move(dst, uvm_thread_context());
642 }
643
uvm_thread_context_restore(uvm_thread_context_t * src)644 void uvm_thread_context_restore(uvm_thread_context_t *src)
645 {
646 thread_context_move(uvm_thread_context(), src);
647 thread_context_non_interrupt_deinit(src);
648 }
649
uvm_thread_context_lock_get(void)650 uvm_thread_context_lock_t *uvm_thread_context_lock_get(void)
651 {
652 return thread_context_lock_of(uvm_thread_context());
653 }
654
uvm_thread_context_lock_disable_tracking(void)655 void uvm_thread_context_lock_disable_tracking(void)
656 {
657 uvm_thread_context_lock_t *context_lock = thread_context_lock_of(uvm_thread_context());
658
659 if (context_lock == NULL)
660 return;
661
662 ++context_lock->skip_lock_tracking;
663
664 UVM_ASSERT(context_lock->skip_lock_tracking != 0);
665 }
666
uvm_thread_context_lock_enable_tracking(void)667 void uvm_thread_context_lock_enable_tracking(void)
668 {
669 uvm_thread_context_lock_t *context_lock = thread_context_lock_of(uvm_thread_context());
670
671 if (context_lock == NULL)
672 return;
673
674 UVM_ASSERT(context_lock->skip_lock_tracking > 0);
675
676 --context_lock->skip_lock_tracking;
677 }
678