1 /*******************************************************************************
2 Copyright (c) 2015-2019 NVIDIA Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to
6 deal in the Software without restriction, including without limitation the
7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8 sell copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 DEALINGS IN THE SOFTWARE.
21
22 *******************************************************************************/
23
24 #include "uvm_tracker.h"
25 #include "uvm_push.h"
26 #include "uvm_channel.h"
27 #include "uvm_kvmalloc.h"
28 #include "uvm_gpu.h"
29 #include "uvm_global.h"
30 #include "uvm_common.h"
31 #include "uvm_linux.h"
32
tracker_is_using_static_entries(uvm_tracker_t * tracker)33 static bool tracker_is_using_static_entries(uvm_tracker_t *tracker)
34 {
35 return tracker->max_size == ARRAY_SIZE(tracker->static_entries);
36 }
37
free_entries(uvm_tracker_t * tracker)38 static void free_entries(uvm_tracker_t *tracker)
39 {
40 if (tracker_is_using_static_entries(tracker))
41 return;
42 uvm_kvfree(tracker->dynamic_entries);
43 }
44
uvm_tracker_get_entries(uvm_tracker_t * tracker)45 uvm_tracker_entry_t *uvm_tracker_get_entries(uvm_tracker_t *tracker)
46 {
47 if (tracker_is_using_static_entries(tracker)) {
48 return tracker->static_entries;
49 }
50 else {
51 UVM_ASSERT(tracker->dynamic_entries != NULL);
52 return tracker->dynamic_entries;
53 }
54 }
55
get_new_entry(uvm_tracker_t * tracker)56 static uvm_tracker_entry_t *get_new_entry(uvm_tracker_t *tracker)
57 {
58 NV_STATUS status = uvm_tracker_reserve(tracker, 1);
59 if (status != NV_OK)
60 return NULL;
61 UVM_ASSERT(tracker->size < tracker->max_size);
62
63 return &uvm_tracker_get_entries(tracker)[tracker->size++];
64 }
65
uvm_tracker_init_from(uvm_tracker_t * dst,uvm_tracker_t * src)66 NV_STATUS uvm_tracker_init_from(uvm_tracker_t *dst, uvm_tracker_t *src)
67 {
68 NV_STATUS status;
69 uvm_tracker_init(dst);
70 status = uvm_tracker_overwrite(dst, src);
71 if (status != NV_OK) {
72 uvm_tracker_deinit(dst);
73 uvm_tracker_init(dst);
74 }
75 return status;
76 }
77
uvm_tracker_deinit(uvm_tracker_t * tracker)78 void uvm_tracker_deinit(uvm_tracker_t *tracker)
79 {
80 free_entries(tracker);
81 memset(tracker, 0, sizeof(*tracker));
82 }
83
uvm_tracker_overwrite(uvm_tracker_t * dst,uvm_tracker_t * src)84 NV_STATUS uvm_tracker_overwrite(uvm_tracker_t *dst, uvm_tracker_t *src)
85 {
86 NV_STATUS status;
87
88 uvm_tracker_clear(dst);
89
90 status = uvm_tracker_reserve(dst, src->size);
91 if (status != NV_OK)
92 return status;
93
94 dst->size = src->size;
95 memcpy(uvm_tracker_get_entries(dst),
96 uvm_tracker_get_entries(src),
97 src->size * sizeof(*uvm_tracker_get_entries(dst)));
98
99 return NV_OK;
100 }
101
uvm_tracker_reserve(uvm_tracker_t * tracker,NvU32 min_free_entries)102 NV_STATUS uvm_tracker_reserve(uvm_tracker_t *tracker, NvU32 min_free_entries)
103 {
104 if (tracker->size + min_free_entries > tracker->max_size) {
105 // Special case the first resize to jump from 1 all the way to 8.
106 // This is based on a guess that if a tracker needs more than 1
107 // entry it likely needs much more.
108 // TODO: Bug 1764961: Verify that guess.
109 NvU32 new_max_size = max((NvU32)8, (NvU32)roundup_pow_of_two(tracker->size + min_free_entries));
110 uvm_tracker_entry_t *new_entries;
111
112 if (tracker_is_using_static_entries(tracker)) {
113 new_entries = uvm_kvmalloc(sizeof(*new_entries) * new_max_size);
114 if (new_entries)
115 memcpy(new_entries, tracker->static_entries, sizeof(*new_entries) * tracker->size);
116 } else {
117 new_entries = uvm_kvrealloc(tracker->dynamic_entries, sizeof(*new_entries) * new_max_size);
118 }
119 if (!new_entries)
120 return NV_ERR_NO_MEMORY;
121 tracker->dynamic_entries = new_entries;
122 tracker->max_size = new_max_size;
123 }
124 UVM_ASSERT(tracker->size + min_free_entries <= tracker->max_size);
125 return NV_OK;
126 }
127
uvm_tracker_add_push(uvm_tracker_t * tracker,uvm_push_t * push)128 NV_STATUS uvm_tracker_add_push(uvm_tracker_t *tracker, uvm_push_t *push)
129 {
130 uvm_tracker_entry_t entry;
131
132 uvm_push_get_tracker_entry(push, &entry);
133
134 return uvm_tracker_add_entry(tracker, &entry);
135 }
136
uvm_tracker_add_entry(uvm_tracker_t * tracker,uvm_tracker_entry_t * new_entry)137 NV_STATUS uvm_tracker_add_entry(uvm_tracker_t *tracker, uvm_tracker_entry_t *new_entry)
138 {
139 uvm_tracker_entry_t *tracker_entry;
140
141 for_each_tracker_entry(tracker_entry, tracker) {
142 if (tracker_entry->channel == new_entry->channel) {
143 tracker_entry->value = max(tracker_entry->value, new_entry->value);
144 return NV_OK;
145 }
146 }
147
148 tracker_entry = get_new_entry(tracker);
149 if (tracker_entry == NULL)
150 return NV_ERR_NO_MEMORY;
151
152 *tracker_entry = *new_entry;
153
154 return NV_OK;
155 }
156
uvm_tracker_overwrite_with_entry(uvm_tracker_t * tracker,uvm_tracker_entry_t * entry)157 void uvm_tracker_overwrite_with_entry(uvm_tracker_t *tracker, uvm_tracker_entry_t *entry)
158 {
159 NV_STATUS status;
160
161 uvm_tracker_clear(tracker);
162
163 // An empty tracker always has space for at least one entry so this cannot
164 // fail.
165 status = uvm_tracker_add_entry(tracker, entry);
166 UVM_ASSERT(status == NV_OK);
167 }
168
uvm_tracker_overwrite_with_push(uvm_tracker_t * tracker,uvm_push_t * push)169 void uvm_tracker_overwrite_with_push(uvm_tracker_t *tracker, uvm_push_t *push)
170 {
171 uvm_tracker_entry_t entry;
172
173 uvm_push_get_tracker_entry(push, &entry);
174
175 uvm_tracker_overwrite_with_entry(tracker, &entry);
176 }
177
reserve_for_entries_from_tracker(uvm_tracker_t * dst,uvm_tracker_t * src)178 static NV_STATUS reserve_for_entries_from_tracker(uvm_tracker_t *dst, uvm_tracker_t *src)
179 {
180 NvU32 needed_free_entries = 0;
181 uvm_tracker_entry_t *src_entry, *dst_entry;
182
183 for_each_tracker_entry(src_entry, src) {
184 bool found = false;
185 for_each_tracker_entry(dst_entry, dst) {
186 if (dst_entry->channel == src_entry->channel) {
187 found = true;
188 break;
189 }
190 }
191 if (!found)
192 needed_free_entries++;
193 }
194
195 return uvm_tracker_reserve(dst, needed_free_entries);
196 }
197
uvm_tracker_add_tracker(uvm_tracker_t * dst,uvm_tracker_t * src)198 NV_STATUS uvm_tracker_add_tracker(uvm_tracker_t *dst, uvm_tracker_t *src)
199 {
200 NV_STATUS status;
201 uvm_tracker_entry_t *src_entry;
202
203 UVM_ASSERT(dst != NULL);
204
205 if (src == NULL)
206 return NV_OK;
207
208 if (src == dst)
209 return NV_OK;
210
211 if (uvm_tracker_is_empty(src))
212 return NV_OK;
213
214 status = uvm_tracker_reserve(dst, src->size);
215 if (status == NV_ERR_NO_MEMORY) {
216 uvm_tracker_remove_completed(dst);
217 uvm_tracker_remove_completed(src);
218 status = reserve_for_entries_from_tracker(dst, src);
219 }
220
221 if (status != NV_OK)
222 return status;
223
224 for_each_tracker_entry(src_entry, src) {
225 status = uvm_tracker_add_entry(dst, src_entry);
226 UVM_ASSERT_MSG(status == NV_OK, "Expected success with reserved memory but got error %d\n", status);
227 }
228
229 return NV_OK;
230 }
231
uvm_tracker_overwrite_safe(uvm_tracker_t * dst,uvm_tracker_t * src)232 NV_STATUS uvm_tracker_overwrite_safe(uvm_tracker_t *dst, uvm_tracker_t *src)
233 {
234 NV_STATUS status = uvm_tracker_overwrite(dst, src);
235 if (status == NV_ERR_NO_MEMORY) {
236 UVM_DBG_PRINT_RL("Failed to overwrite tracker, waiting\n");
237 status = uvm_tracker_wait(src);
238 }
239 return status;
240 }
241
uvm_tracker_add_push_safe(uvm_tracker_t * tracker,uvm_push_t * push)242 NV_STATUS uvm_tracker_add_push_safe(uvm_tracker_t *tracker, uvm_push_t *push)
243 {
244 NV_STATUS status = uvm_tracker_add_push(tracker, push);
245 if (status == NV_ERR_NO_MEMORY) {
246 UVM_DBG_PRINT_RL("Failed to add push to tracker, waiting\n");
247 status = uvm_push_wait(push);
248 }
249 return status;
250 }
251
uvm_tracker_add_entry_safe(uvm_tracker_t * tracker,uvm_tracker_entry_t * new_entry)252 NV_STATUS uvm_tracker_add_entry_safe(uvm_tracker_t *tracker, uvm_tracker_entry_t *new_entry)
253 {
254 NV_STATUS status = uvm_tracker_add_entry(tracker, new_entry);
255 if (status == NV_ERR_NO_MEMORY) {
256 UVM_DBG_PRINT_RL("Failed to add entry to tracker, waiting\n");
257 status = uvm_tracker_wait_for_entry(new_entry);
258 }
259 return status;
260 }
261
uvm_tracker_add_tracker_safe(uvm_tracker_t * dst,uvm_tracker_t * src)262 NV_STATUS uvm_tracker_add_tracker_safe(uvm_tracker_t *dst, uvm_tracker_t *src)
263 {
264 NV_STATUS status = uvm_tracker_add_tracker(dst, src);
265 if (status == NV_ERR_NO_MEMORY) {
266 UVM_DBG_PRINT_RL("Failed to add tracker to tracker, waiting\n");
267 status = uvm_tracker_wait(src);
268 }
269 return status;
270 }
271
uvm_tracker_is_entry_completed(uvm_tracker_entry_t * tracker_entry)272 bool uvm_tracker_is_entry_completed(uvm_tracker_entry_t *tracker_entry)
273 {
274 if (!tracker_entry->channel)
275 return true;
276
277 return uvm_channel_is_value_completed(tracker_entry->channel, tracker_entry->value);
278 }
279
uvm_tracker_entry_print_pending_pushes(uvm_tracker_entry_t * entry)280 static void uvm_tracker_entry_print_pending_pushes(uvm_tracker_entry_t *entry)
281 {
282 uvm_channel_t *channel = entry->channel;
283 uvm_gpu_t *gpu = uvm_channel_get_gpu(channel);
284
285 UVM_DBG_PRINT("Tracker entry for value %llu (sema VA 0x%llx) channel %s GPU %s\n",
286 entry->value,
287 uvm_channel_tracking_semaphore_get_gpu_va(channel),
288 channel->name,
289 uvm_gpu_name(gpu));
290
291 uvm_channel_print_pending_pushes(channel);
292 }
293
uvm_tracker_print_pending_pushes(uvm_tracker_t * tracker)294 static void uvm_tracker_print_pending_pushes(uvm_tracker_t *tracker)
295 {
296 uvm_tracker_entry_t *entry;
297 for_each_tracker_entry(entry, tracker)
298 uvm_tracker_entry_print_pending_pushes(entry);
299 }
300
wait_for_entry_with_spin(uvm_tracker_entry_t * tracker_entry,uvm_spin_loop_t * spin)301 static NV_STATUS wait_for_entry_with_spin(uvm_tracker_entry_t *tracker_entry, uvm_spin_loop_t *spin)
302 {
303 NV_STATUS status = NV_OK;
304
305 while (!uvm_tracker_is_entry_completed(tracker_entry) && status == NV_OK) {
306 if (UVM_SPIN_LOOP(spin) == NV_ERR_TIMEOUT_RETRY)
307 uvm_tracker_entry_print_pending_pushes(tracker_entry);
308
309 status = uvm_channel_check_errors(tracker_entry->channel);
310 if (status == NV_OK)
311 status = uvm_global_get_status();
312 }
313
314 if (status != NV_OK) {
315 UVM_ASSERT(status == uvm_global_get_status());
316 tracker_entry->channel = NULL;
317 tracker_entry->value = 0;
318 }
319
320 return status;
321 }
322
uvm_tracker_wait_for_entry(uvm_tracker_entry_t * tracker_entry)323 NV_STATUS uvm_tracker_wait_for_entry(uvm_tracker_entry_t *tracker_entry)
324 {
325 uvm_spin_loop_t spin;
326 uvm_spin_loop_init(&spin);
327 return wait_for_entry_with_spin(tracker_entry, &spin);
328 }
329
uvm_tracker_wait(uvm_tracker_t * tracker)330 NV_STATUS uvm_tracker_wait(uvm_tracker_t *tracker)
331 {
332 NV_STATUS status = NV_OK;
333 uvm_spin_loop_t spin;
334
335 uvm_spin_loop_init(&spin);
336 while (!uvm_tracker_is_completed(tracker) && status == NV_OK) {
337 if (UVM_SPIN_LOOP(&spin) == NV_ERR_TIMEOUT_RETRY)
338 uvm_tracker_print_pending_pushes(tracker);
339
340 status = uvm_tracker_check_errors(tracker);
341 }
342
343 if (status != NV_OK) {
344 UVM_ASSERT(status == uvm_global_get_status());
345
346 // Just clear the tracker without printing anything extra. If one of the
347 // entries from this tracker caused a channel error,
348 // uvm_tracker_check_errors() would have already printed it. And if we
349 // hit a global error for some other reason, we don't want to spam the
350 // log with all other pending entries.
351 //
352 // See the comment for uvm_tracker_wait() on why the entries are cleared.
353 uvm_tracker_clear(tracker);
354 }
355
356 return status;
357 }
358
uvm_tracker_wait_for_other_gpus(uvm_tracker_t * tracker,uvm_gpu_t * gpu)359 NV_STATUS uvm_tracker_wait_for_other_gpus(uvm_tracker_t *tracker, uvm_gpu_t *gpu)
360 {
361 NV_STATUS status = NV_OK;
362 uvm_tracker_entry_t *entry;
363 uvm_spin_loop_t spin;
364
365 uvm_spin_loop_init(&spin);
366
367 for_each_tracker_entry(entry, tracker) {
368 if (uvm_tracker_entry_gpu(entry) == gpu)
369 continue;
370
371 status = wait_for_entry_with_spin(entry, &spin);
372 if (status != NV_OK)
373 break;
374 }
375
376 if (status == NV_OK) {
377 uvm_tracker_remove_completed(tracker);
378 }
379 else {
380 UVM_ASSERT(status == uvm_global_get_status());
381 uvm_tracker_clear(tracker);
382 }
383
384 return status;
385 }
386
uvm_tracker_check_errors(uvm_tracker_t * tracker)387 NV_STATUS uvm_tracker_check_errors(uvm_tracker_t *tracker)
388 {
389 uvm_tracker_entry_t *tracker_entry;
390 NV_STATUS status = uvm_global_get_status();
391
392 if (status != NV_OK)
393 return status;
394
395 for_each_tracker_entry(tracker_entry, tracker) {
396 status = uvm_channel_check_errors(tracker_entry->channel);
397 if (status != NV_OK)
398 return status;
399 }
400
401 return NV_OK;
402 }
403
uvm_tracker_query(uvm_tracker_t * tracker)404 NV_STATUS uvm_tracker_query(uvm_tracker_t *tracker)
405 {
406 NV_STATUS status;
407 bool completed = uvm_tracker_is_completed(tracker);
408
409 status = uvm_tracker_check_errors(tracker);
410 if (status != NV_OK)
411 return status;
412
413 return completed ? NV_OK : NV_WARN_MORE_PROCESSING_REQUIRED;
414 }
415
uvm_tracker_remove_completed(uvm_tracker_t * tracker)416 void uvm_tracker_remove_completed(uvm_tracker_t *tracker)
417 {
418 NvU32 i = 0;
419
420 uvm_tracker_entry_t *entries = uvm_tracker_get_entries(tracker);
421
422 // Keep removing completed entries until we run out of entries
423 while (i < tracker->size) {
424 if (uvm_tracker_is_entry_completed(&entries[i])) {
425 --tracker->size;
426 if (i != tracker->size)
427 entries[i] = entries[tracker->size];
428 }
429 else {
430 ++i;
431 }
432 }
433 }
434
uvm_tracker_is_completed(uvm_tracker_t * tracker)435 bool uvm_tracker_is_completed(uvm_tracker_t *tracker)
436 {
437 uvm_tracker_remove_completed(tracker);
438
439 return tracker->size == 0;
440 }
441
uvm_tracker_entry_gpu(uvm_tracker_entry_t * entry)442 uvm_gpu_t *uvm_tracker_entry_gpu(uvm_tracker_entry_t *entry)
443 {
444 return uvm_channel_get_gpu(entry->channel);
445 }
446
447