1 /******************************************************************************* 2 Copyright (c) 2021-2023 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #ifndef __UVM_CONF_COMPUTING_H__ 25 #define __UVM_CONF_COMPUTING_H__ 26 27 #include "nv_uvm_types.h" 28 #include "uvm_forward_decl.h" 29 #include "uvm_lock.h" 30 #include "uvm_tracker.h" 31 #include "uvm_va_block_types.h" 32 33 #include "linux/list.h" 34 35 #define UVM_CONF_COMPUTING_AUTH_TAG_SIZE (UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES) 36 37 // An authentication tag pointer is required by HW to be 16-bytes aligned. 38 #define UVM_CONF_COMPUTING_AUTH_TAG_ALIGNMENT 16 39 40 // An IV pointer is required by HW to be 16-bytes aligned. 41 // 42 // Use sizeof(UvmCslIv) to refer to the IV size. 43 #define UVM_CONF_COMPUTING_IV_ALIGNMENT 16 44 45 // SEC2 decrypt operation buffers are required to be 16-bytes aligned. 46 #define UVM_CONF_COMPUTING_SEC2_BUF_ALIGNMENT 16 47 48 // CE encrypt/decrypt can be unaligned if the entire buffer lies in a single 49 // 32B segment. Otherwise, it needs to be 32B aligned. 50 #define UVM_CONF_COMPUTING_BUF_ALIGNMENT 32 51 52 #define UVM_CONF_COMPUTING_DMA_BUFFER_SIZE UVM_VA_BLOCK_SIZE 53 54 // SEC2 supports at most a stream of 64 entries in the method stream for 55 // signing. Each entry is made of the method address and method data, therefore 56 // the maximum buffer size is: UVM_METHOD_SIZE * 2 * 64 = 512. 57 // UVM, however, won't use this amount of entries, in the worst case scenario, 58 // we push a semaphore_releases or a decrypt. A SEC2 semaphore_release uses 6 1U 59 // entries, whereas a SEC2 decrypt uses 10 1U entries. For 10 entries, 60 // UVM_METHOD_SIZE * 2 * 10 = 80. 61 #define UVM_CONF_COMPUTING_SIGN_BUF_MAX_SIZE 80 62 63 void uvm_conf_computing_check_parent_gpu(const uvm_parent_gpu_t *parent); 64 65 bool uvm_conf_computing_mode_enabled_parent(const uvm_parent_gpu_t *parent); 66 bool uvm_conf_computing_mode_enabled(const uvm_gpu_t *gpu); 67 bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu); 68 69 typedef struct 70 { 71 // List of free DMA buffers (uvm_conf_computing_dma_buffer_t). 72 // A free DMA buffer can be grabbed anytime, though the tracker 73 // inside it may still have pending work. 74 struct list_head free_dma_buffers; 75 76 // Used to grow the pool when full. 77 size_t num_dma_buffers; 78 79 // Lock protecting the dma_buffer_pool 80 uvm_mutex_t lock; 81 } uvm_conf_computing_dma_buffer_pool_t; 82 83 typedef struct 84 { 85 // Backing DMA allocation 86 uvm_mem_t *alloc; 87 88 // Used internally by the pool management code to track the state of 89 // a free buffer. 90 uvm_tracker_t tracker; 91 92 // When the DMA buffer is used as the destination of a GPU encryption, SEC2 93 // writes the authentication tag here. Later when the buffer is decrypted 94 // on the CPU the authentication tag is used again (read) for CSL to verify 95 // the authenticity. The allocation is big enough for one authentication 96 // tag per PAGE_SIZE page in the alloc buffer. 97 uvm_mem_t *auth_tag; 98 99 // CSL supports out-of-order decryption, the decrypt IV is used similarly 100 // to the authentication tag. The allocation is big enough for one IV per 101 // PAGE_SIZE page in the alloc buffer. The granularity between the decrypt 102 // IV and authentication tag must match. 103 UvmCslIv decrypt_iv[(UVM_CONF_COMPUTING_DMA_BUFFER_SIZE / PAGE_SIZE)]; 104 105 // Bitmap of the encrypted pages in the backing allocation 106 uvm_page_mask_t encrypted_page_mask; 107 108 // See uvm_conf_computing_dma_pool lists 109 struct list_head node; 110 } uvm_conf_computing_dma_buffer_t; 111 112 // Retrieve a DMA buffer from the given DMA allocation pool. 113 // NV_OK Stage buffer successfully retrieved 114 // NV_ERR_NO_MEMORY No free DMA buffers are available for grab, and 115 // expanding the memory pool to get new ones failed. 116 // 117 // out_dma_buffer is only valid if NV_OK is returned. The caller is responsible 118 // for calling uvm_conf_computing_dma_buffer_free once the operations on this 119 // buffer are done. 120 // When out_tracker is passed to the function, the buffer's dependencies are 121 // added to the tracker. The caller is guaranteed that all pending tracker 122 // entries come from the same GPU as the pool's owner. Before being able to use 123 // the DMA buffer, the caller is responsible for either acquiring or waiting 124 // on out_tracker. If out_tracker is NULL, the wait happens in the allocation 125 // itself. 126 // Upon success the encrypted_page_mask is cleared as part of the allocation. 127 NV_STATUS uvm_conf_computing_dma_buffer_alloc(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 128 uvm_conf_computing_dma_buffer_t **out_dma_buffer, 129 uvm_tracker_t *out_tracker); 130 131 // Free a DMA buffer to the DMA allocation pool. All DMA buffers must be freed 132 // prior to GPU deinit. 133 // 134 // The tracker is optional and a NULL tracker indicates that no new operation 135 // has been pushed for the buffer. A non-NULL tracker indicates any additional 136 // pending operations on the buffer pushed by the caller that need to be 137 // synchronized before freeing or re-using the buffer. 138 void uvm_conf_computing_dma_buffer_free(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 139 uvm_conf_computing_dma_buffer_t *dma_buffer, 140 uvm_tracker_t *tracker); 141 142 // Synchronize trackers in all entries in the GPU's DMA pool 143 void uvm_conf_computing_dma_buffer_pool_sync(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool); 144 145 146 // Initialization and deinitialization of Confidential Computing data structures 147 // for the given GPU. 148 NV_STATUS uvm_conf_computing_gpu_init(uvm_gpu_t *gpu); 149 void uvm_conf_computing_gpu_deinit(uvm_gpu_t *gpu); 150 151 // Logs encryption information from the GPU and returns the IV. 152 void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv); 153 154 // Acquires next CPU encryption IV and returns it. 155 void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *iv); 156 157 // CPU side encryption helper with explicit IV, which is obtained from 158 // uvm_conf_computing_acquire_encryption_iv. Without an explicit IV 159 // the function uses the next IV in order. Encrypts data in src_plain and 160 // write the cipher text in dst_cipher. src_plain and dst_cipher can't overlap. 161 // The IV is invalidated and can't be used again after this operation. 162 void uvm_conf_computing_cpu_encrypt(uvm_channel_t *channel, 163 void *dst_cipher, 164 const void *src_plain, 165 UvmCslIv *encrypt_iv, 166 size_t size, 167 void *auth_tag_buffer); 168 169 // CPU side decryption helper. Decrypts data from src_cipher and writes the 170 // plain text in dst_plain. src_cipher and dst_plain can't overlap. IV obtained 171 // from uvm_conf_computing_log_gpu_encryption() needs to be be passed to src_iv. 172 NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel, 173 void *dst_plain, 174 const void *src_cipher, 175 const UvmCslIv *src_iv, 176 size_t size, 177 const void *auth_tag_buffer); 178 179 // CPU decryption of a single replayable fault, encrypted by GSP-RM. 180 // 181 // Replayable fault decryption depends not only on the encrypted fault contents, 182 // and the authentication tag, but also on the plaintext valid bit associated 183 // with the fault. 184 // 185 // When decrypting data previously encrypted by the Copy Engine, use 186 // uvm_conf_computing_cpu_decrypt instead. 187 // 188 // Locking: this function must be invoked while holding the replayable ISR lock. 189 NV_STATUS uvm_conf_computing_fault_decrypt(uvm_parent_gpu_t *parent_gpu, 190 void *dst_plain, 191 const void *src_cipher, 192 const void *auth_tag_buffer, 193 NvU8 valid); 194 195 // Increment the CPU-side decrypt IV of the CSL context associated with 196 // replayable faults. The function is a no-op if the given increment is zero. 197 // 198 // The IV associated with a fault CSL context is a 64-bit counter. 199 // 200 // Locking: this function must be invoked while holding the replayable ISR lock. 201 void uvm_conf_computing_fault_increment_decrypt_iv(uvm_parent_gpu_t *parent_gpu, NvU64 increment); 202 #endif // __UVM_CONF_COMPUTING_H__ 203