1 /*******************************************************************************
2     Copyright (c) 2021 NVIDIA Corporation
3 
4     Permission is hereby granted, free of charge, to any person obtaining a copy
5     of this software and associated documentation files (the "Software"), to
6     deal in the Software without restriction, including without limitation the
7     rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8     sell copies of the Software, and to permit persons to whom the Software is
9     furnished to do so, subject to the following conditions:
10 
11         The above copyright notice and this permission notice shall be
12         included in all copies or substantial portions of the Software.
13 
14     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17     THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20     DEALINGS IN THE SOFTWARE.
21 
22 *******************************************************************************/
23 
24 #include "uvm_common.h"
25 #include "uvm_global.h"
26 #include "uvm_conf_computing.h"
27 #include "uvm_kvmalloc.h"
28 #include "uvm_gpu.h"
29 #include "uvm_mem.h"
30 #include "uvm_processors.h"
31 #include "uvm_tracker.h"
32 #include "nv_uvm_interface.h"
33 #include "uvm_va_block.h"
34 
35 
36 static UvmGpuConfComputeMode uvm_conf_computing_get_mode(const uvm_parent_gpu_t *parent)
37 {
38     return parent->rm_info.gpuConfComputeCaps.mode;
39 }
40 
41 bool uvm_conf_computing_mode_enabled_parent(const uvm_parent_gpu_t *parent)
42 {
43     return uvm_conf_computing_get_mode(parent) != UVM_GPU_CONF_COMPUTE_MODE_NONE;
44 }
45 
46 bool uvm_conf_computing_mode_enabled(const uvm_gpu_t *gpu)
47 {
48     return uvm_conf_computing_mode_enabled_parent(gpu->parent);
49 }
50 
51 bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu)
52 {
53     return uvm_conf_computing_get_mode(gpu->parent) == UVM_GPU_CONF_COMPUTE_MODE_HCC;
54 }
55 
56 NV_STATUS uvm_conf_computing_init_parent_gpu(const uvm_parent_gpu_t *parent)
57 {
58     UvmGpuConfComputeMode cc, sys_cc;
59     uvm_gpu_t *first;
60 
61     uvm_assert_mutex_locked(&g_uvm_global.global_lock);
62 
63     // TODO: Bug 2844714.
64     // Since we have no routine to traverse parent gpus,
65     // find first child GPU and get its parent.
66     first = uvm_global_processor_mask_find_first_gpu(&g_uvm_global.retained_gpus);
67     if (!first)
68         return NV_OK;
69 
70     sys_cc = uvm_conf_computing_get_mode(first->parent);
71     cc = uvm_conf_computing_get_mode(parent);
72 
73     return cc == sys_cc ? NV_OK : NV_ERR_NOT_SUPPORTED;
74 }
75 
76 static void dma_buffer_destroy_locked(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
77                                       uvm_conf_computing_dma_buffer_t *dma_buffer)
78 {
79     uvm_assert_mutex_locked(&dma_buffer_pool->lock);
80 
81     list_del(&dma_buffer->node);
82     uvm_tracker_wait_deinit(&dma_buffer->tracker);
83 
84     uvm_mem_free(dma_buffer->alloc);
85     uvm_mem_free(dma_buffer->auth_tag);
86     uvm_kvfree(dma_buffer);
87 }
88 
89 static uvm_gpu_t *dma_buffer_pool_to_gpu(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool)
90 {
91     return container_of(dma_buffer_pool, uvm_gpu_t, conf_computing.dma_buffer_pool);
92 }
93 
94 // Allocate and map a new DMA stage buffer to CPU and GPU (VA)
95 static NV_STATUS dma_buffer_create(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
96                                    uvm_conf_computing_dma_buffer_t **dma_buffer_out)
97 {
98     uvm_gpu_t *dma_owner;
99     uvm_conf_computing_dma_buffer_t *dma_buffer;
100     uvm_mem_t *alloc = NULL;
101     NV_STATUS status = NV_OK;
102     size_t auth_tags_size = (UVM_CONF_COMPUTING_DMA_BUFFER_SIZE / PAGE_SIZE) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE;
103 
104     dma_buffer = uvm_kvmalloc_zero(sizeof(*dma_buffer));
105     if (!dma_buffer)
106         return NV_ERR_NO_MEMORY;
107 
108     dma_owner = dma_buffer_pool_to_gpu(dma_buffer_pool);
109     uvm_tracker_init(&dma_buffer->tracker);
110     INIT_LIST_HEAD(&dma_buffer->node);
111 
112     status = uvm_mem_alloc_sysmem_dma_and_map_cpu_kernel(UVM_CONF_COMPUTING_DMA_BUFFER_SIZE, dma_owner, NULL, &alloc);
113     if (status != NV_OK)
114         goto err;
115 
116     dma_buffer->alloc = alloc;
117 
118     status = uvm_mem_map_gpu_kernel(alloc, dma_owner);
119     if (status != NV_OK)
120         goto err;
121 
122     status = uvm_mem_alloc_sysmem_dma_and_map_cpu_kernel(auth_tags_size, dma_owner, NULL, &alloc);
123     if (status != NV_OK)
124         goto err;
125 
126     dma_buffer->auth_tag = alloc;
127 
128     status = uvm_mem_map_gpu_kernel(alloc, dma_owner);
129     if (status != NV_OK)
130         goto err;
131 
132     *dma_buffer_out = dma_buffer;
133 
134     return status;
135 
136 err:
137     dma_buffer_destroy_locked(dma_buffer_pool, dma_buffer);
138     return status;
139 }
140 
141 void uvm_conf_computing_dma_buffer_pool_sync(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool)
142 {
143     uvm_conf_computing_dma_buffer_t *dma_buffer;
144 
145     if (dma_buffer_pool->num_dma_buffers == 0)
146         return;
147 
148     uvm_mutex_lock(&dma_buffer_pool->lock);
149     list_for_each_entry(dma_buffer, &dma_buffer_pool->free_dma_buffers, node)
150         uvm_tracker_wait(&dma_buffer->tracker);
151     uvm_mutex_unlock(&dma_buffer_pool->lock);
152 }
153 
154 static void conf_computing_dma_buffer_pool_deinit(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool)
155 {
156     uvm_conf_computing_dma_buffer_t *dma_buffer;
157     uvm_conf_computing_dma_buffer_t *next_buff;
158 
159     if (dma_buffer_pool->num_dma_buffers == 0)
160         return;
161 
162     // Because the pool is teared down at the same time the GPU is unregistered
163     // the lock is required only to quiet assertions not for functional reasons
164     // see dma_buffer_destroy_locked()).
165     uvm_mutex_lock(&dma_buffer_pool->lock);
166 
167     list_for_each_entry_safe(dma_buffer, next_buff, &dma_buffer_pool->free_dma_buffers, node) {
168         dma_buffer_destroy_locked(dma_buffer_pool, dma_buffer);
169         dma_buffer_pool->num_dma_buffers--;
170     }
171 
172     UVM_ASSERT(dma_buffer_pool->num_dma_buffers == 0);
173     UVM_ASSERT(list_empty(&dma_buffer_pool->free_dma_buffers));
174     uvm_mutex_unlock(&dma_buffer_pool->lock);
175 }
176 
177 static void dma_buffer_pool_add(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
178                                uvm_conf_computing_dma_buffer_t *dma_buffer)
179 {
180     uvm_assert_mutex_locked(&dma_buffer_pool->lock);
181     list_add_tail(&dma_buffer->node, &dma_buffer_pool->free_dma_buffers);
182 }
183 
184 static NV_STATUS conf_computing_dma_buffer_pool_init(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool)
185 {
186     size_t i;
187     uvm_gpu_t *gpu;
188     size_t num_dma_buffers = 32;
189     NV_STATUS status = NV_OK;
190 
191     UVM_ASSERT(dma_buffer_pool->num_dma_buffers == 0);
192 
193     gpu = dma_buffer_pool_to_gpu(dma_buffer_pool);
194 
195     UVM_ASSERT(uvm_conf_computing_mode_enabled(gpu));
196 
197     INIT_LIST_HEAD(&dma_buffer_pool->free_dma_buffers);
198     uvm_mutex_init(&dma_buffer_pool->lock, UVM_LOCK_ORDER_CONF_COMPUTING_DMA_BUFFER_POOL);
199     dma_buffer_pool->num_dma_buffers = num_dma_buffers;
200 
201     uvm_mutex_lock(&dma_buffer_pool->lock);
202     for (i = 0; i < num_dma_buffers; i++) {
203         uvm_conf_computing_dma_buffer_t *dma_buffer;
204 
205         status = dma_buffer_create(dma_buffer_pool, &dma_buffer);
206         if (status != NV_OK)
207             break;
208 
209         dma_buffer_pool_add(dma_buffer_pool, dma_buffer);
210     }
211     uvm_mutex_unlock(&dma_buffer_pool->lock);
212 
213     if (i < num_dma_buffers)
214         conf_computing_dma_buffer_pool_deinit(dma_buffer_pool);
215 
216     return status;
217 }
218 
219 static NV_STATUS dma_buffer_pool_expand_locked(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool)
220 {
221     size_t i;
222     uvm_gpu_t *gpu;
223     size_t nb_to_alloc;
224     NV_STATUS status = NV_OK;
225     UVM_ASSERT(dma_buffer_pool->num_dma_buffers > 0);
226 
227     gpu = dma_buffer_pool_to_gpu(dma_buffer_pool);
228     nb_to_alloc = dma_buffer_pool->num_dma_buffers;
229     for (i = 0; i < nb_to_alloc; ++i) {
230         uvm_conf_computing_dma_buffer_t *dma_buffer;
231 
232         status = dma_buffer_create(dma_buffer_pool, &dma_buffer);
233         if (status != NV_OK)
234             break;
235 
236         dma_buffer_pool_add(dma_buffer_pool, dma_buffer);
237     }
238 
239     dma_buffer_pool->num_dma_buffers += i;
240 
241     if (i == 0)
242         return status;
243 
244     return NV_OK;
245 }
246 
247 NV_STATUS uvm_conf_computing_dma_buffer_alloc(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
248                                               uvm_conf_computing_dma_buffer_t **dma_buffer_out,
249                                               uvm_tracker_t *out_tracker)
250 {
251     uvm_conf_computing_dma_buffer_t *dma_buffer = NULL;
252     NV_STATUS status;
253 
254     UVM_ASSERT(dma_buffer_pool->num_dma_buffers > 0);
255 
256     // TODO: Bug 3385623: Heuristically expand DMA memory pool
257     uvm_mutex_lock(&dma_buffer_pool->lock);
258     if (list_empty(&dma_buffer_pool->free_dma_buffers)) {
259         status = dma_buffer_pool_expand_locked(dma_buffer_pool);
260 
261         if (status != NV_OK) {
262             uvm_mutex_unlock(&dma_buffer_pool->lock);
263             return status;
264         }
265     }
266 
267     // We're guaranteed that at least one DMA stage buffer is available at this
268     // point.
269     dma_buffer = list_first_entry(&dma_buffer_pool->free_dma_buffers, uvm_conf_computing_dma_buffer_t, node);
270     list_del_init(&dma_buffer->node);
271     uvm_mutex_unlock(&dma_buffer_pool->lock);
272 
273     status = uvm_tracker_wait_for_other_gpus(&dma_buffer->tracker, dma_buffer->alloc->dma_owner);
274     if (status != NV_OK)
275         goto error;
276 
277     if (out_tracker)
278         status = uvm_tracker_add_tracker_safe(out_tracker, &dma_buffer->tracker);
279     else
280         status = uvm_tracker_wait(&dma_buffer->tracker);
281 
282     if (status != NV_OK)
283         goto error;
284 
285     uvm_page_mask_zero(&dma_buffer->encrypted_page_mask);
286     *dma_buffer_out = dma_buffer;
287 
288     return status;
289 
290 error:
291     uvm_tracker_deinit(&dma_buffer->tracker);
292     uvm_conf_computing_dma_buffer_free(dma_buffer_pool, dma_buffer, NULL);
293     return status;
294 }
295 
296 void uvm_conf_computing_dma_buffer_free(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool,
297                                         uvm_conf_computing_dma_buffer_t *dma_buffer,
298                                         uvm_tracker_t *tracker)
299 {
300 
301     NV_STATUS status;
302 
303     if (!dma_buffer)
304         return;
305 
306     UVM_ASSERT(dma_buffer_pool->num_dma_buffers > 0);
307 
308     uvm_tracker_remove_completed(&dma_buffer->tracker);
309     if (tracker) {
310         uvm_tracker_remove_completed(tracker);
311         status = uvm_tracker_add_tracker_safe(&dma_buffer->tracker, tracker);
312         if (status != NV_OK)
313             UVM_ASSERT(status == uvm_global_get_status());
314     }
315 
316     uvm_mutex_lock(&dma_buffer_pool->lock);
317     dma_buffer_pool_add(dma_buffer_pool, dma_buffer);
318     uvm_mutex_unlock(&dma_buffer_pool->lock);
319 }
320 
321 static void dummy_iv_mem_deinit(uvm_gpu_t *gpu)
322 {
323     uvm_mem_free(gpu->conf_computing.iv_mem);
324 }
325 
326 static NV_STATUS dummy_iv_mem_init(uvm_gpu_t *gpu)
327 {
328     NV_STATUS status;
329 
330     if (!uvm_conf_computing_mode_is_hcc(gpu))
331         return NV_OK;
332 
333     status = uvm_mem_alloc_sysmem_dma(sizeof(UvmCslIv), gpu, NULL, &gpu->conf_computing.iv_mem);
334     if (status != NV_OK)
335         return status;
336 
337     status = uvm_mem_map_gpu_kernel(gpu->conf_computing.iv_mem, gpu);
338     if (status != NV_OK)
339         goto error;
340 
341     return NV_OK;
342 
343 error:
344     dummy_iv_mem_deinit(gpu);
345     return status;
346 }
347 
348 NV_STATUS uvm_conf_computing_gpu_init(uvm_gpu_t *gpu)
349 {
350     NV_STATUS status;
351 
352     if (!uvm_conf_computing_mode_enabled(gpu))
353         return NV_OK;
354 
355     status = conf_computing_dma_buffer_pool_init(&gpu->conf_computing.dma_buffer_pool);
356     if (status != NV_OK)
357         return status;
358 
359     status = dummy_iv_mem_init(gpu);
360     if (status != NV_OK)
361         goto error;
362 
363     return NV_OK;
364 
365 error:
366     uvm_conf_computing_gpu_deinit(gpu);
367     return status;
368 }
369 
370 void uvm_conf_computing_gpu_deinit(uvm_gpu_t *gpu)
371 {
372     dummy_iv_mem_deinit(gpu);
373     conf_computing_dma_buffer_pool_deinit(&gpu->conf_computing.dma_buffer_pool);
374 }
375 
376 void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv)
377 {
378     NV_STATUS status;
379 
380     uvm_mutex_lock(&channel->csl.ctx_lock);
381     status = nvUvmInterfaceCslLogDeviceEncryption(&channel->csl.ctx, iv);
382     uvm_mutex_unlock(&channel->csl.ctx_lock);
383 
384     // nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter
385     // overflows. This is not supposed to happen on CC.
386     UVM_ASSERT(status == NV_OK);
387 }
388 
389 void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *iv)
390 {
391     NV_STATUS status;
392 
393     uvm_mutex_lock(&channel->csl.ctx_lock);
394     status = nvUvmInterfaceCslAcquireEncryptionIv(&channel->csl.ctx, iv);
395     uvm_mutex_unlock(&channel->csl.ctx_lock);
396 
397     // nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter
398     // overflows. This is not supposed to happen on CC.
399     UVM_ASSERT(status == NV_OK);
400 }
401 
402 void uvm_conf_computing_cpu_encrypt(uvm_channel_t *channel,
403                                     void *dst_cipher,
404                                     const void *src_plain,
405                                     UvmCslIv *encrypt_iv,
406                                     size_t size,
407                                     void *auth_tag_buffer)
408 {
409     NV_STATUS status;
410 
411     UVM_ASSERT(size);
412 
413     uvm_mutex_lock(&channel->csl.ctx_lock);
414     status = nvUvmInterfaceCslEncrypt(&channel->csl.ctx,
415                                       size,
416                                       (NvU8 const *) src_plain,
417                                       encrypt_iv,
418                                       (NvU8 *) dst_cipher,
419                                       (NvU8 *) auth_tag_buffer);
420     uvm_mutex_unlock(&channel->csl.ctx_lock);
421 
422     // nvUvmInterfaceCslEncrypt fails when a 64-bit encryption counter
423     // overflows. This is not supposed to happen on CC.
424     UVM_ASSERT(status == NV_OK);
425 }
426 
427 NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel,
428                                          void *dst_plain,
429                                          const void *src_cipher,
430                                          const UvmCslIv *src_iv,
431                                          size_t size,
432                                          const void *auth_tag_buffer)
433 {
434     NV_STATUS status;
435 
436     uvm_mutex_lock(&channel->csl.ctx_lock);
437     status = nvUvmInterfaceCslDecrypt(&channel->csl.ctx,
438                                       size,
439                                       (const NvU8 *) src_cipher,
440                                       src_iv,
441                                       (NvU8 *) dst_plain,
442                                       (const NvU8 *) auth_tag_buffer);
443     uvm_mutex_unlock(&channel->csl.ctx_lock);
444 
445     return status;
446 }
447