1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #ifndef _CHANNEL_UTILS_H_ 25 #define _CHANNEL_UTILS_H_ 26 27 #include "core/core.h" 28 #include "gpu/gpu.h" 29 #include "gpu/mem_mgr/mem_mgr.h" 30 #include "gpu/ce/kernel_ce.h" 31 #include "gpu/bus/kern_bus.h" 32 #include "core/prelude.h" 33 #include "rmapi/rs_utils.h" 34 #include "nvos.h" 35 36 #include "class/cl906f.h" 37 #include "class/cl906f.h" 38 #include "class/cl906fsw.h" 39 40 #include "class/clb0b5.h" // MAXWELL_DMA_COPY_A 41 #include "class/clc0b5.h" // PASCAL_DMA_COPY_A 42 #include "class/clc1b5.h" // PASCAL_DMA_COPY_B 43 #include "class/clc3b5.h" // VOLTA_DMA_COPY_A 44 #include "class/clc5b5.h" // TURING_DMA_COPY_A 45 #include "class/clc6b5.h" // AMPERE_DMA_COPY_A 46 #include "class/clc7b5.h" // AMPERE_DMA_COPY_B 47 48 #include "class/clc8b5.h" // HOPPER_DMA_COPY_A 49 50 #include "class/clc86f.h" // HOPPER_CHANNEL_GPFIFO_A 51 52 #include "gpu/conf_compute/ccsl.h" 53 54 #include "nvctassert.h" 55 #include "vgpu/vgpu_guest_pma_scrubber.h" 56 57 #define RM_SUBCHANNEL 0x0 58 59 #define NV_PUSH_METHOD(OpType, SubCh, Method, Count) \ 60 (DRF_DEF(906F, _DMA, _SEC_OP, OpType) | \ 61 DRF_NUM(906F, _DMA, _METHOD_ADDRESS, (Method) >> 2) | \ 62 DRF_NUM(906F, _DMA, _METHOD_SUBCHANNEL, (SubCh)) | \ 63 DRF_NUM(906F, _DMA, _METHOD_COUNT, (Count))) 64 65 #define _NV_ASSERT_CONTIGUOUS_METHOD(a1, a2) NV_ASSERT((a2) - (a1) == 4) 66 67 #define NV_PUSH_DATA(Data) MEM_WR32(pPtr++, (Data)) 68 69 #define _NV_PUSH_INC_1U(SubCh, a1, d1, Count) \ 70 do \ 71 { \ 72 NV_PUSH_DATA(NV_PUSH_METHOD(_INC_METHOD, SubCh, a1, Count)); \ 73 NV_PUSH_DATA(d1); \ 74 } while (0) 75 76 #define NV_PUSH_INC_1U(SubCh, a1, d1) \ 77 do \ 78 { \ 79 _NV_PUSH_INC_1U (SubCh, a1, d1, 1); \ 80 } while (0) 81 82 #define NV_PUSH_INC_2U(SubCh, a1, d1, a2, d2) \ 83 do \ 84 { \ 85 _NV_ASSERT_CONTIGUOUS_METHOD(a1, a2); \ 86 _NV_PUSH_INC_1U(SubCh, a1, d1, 2); \ 87 NV_PUSH_DATA(d2); \ 88 } while (0) 89 90 #define NV_PUSH_INC_3U(SubCh, a1, d1, a2, d2, a3, d3) \ 91 do \ 92 { \ 93 _NV_ASSERT_CONTIGUOUS_METHOD(a1, a2); \ 94 _NV_ASSERT_CONTIGUOUS_METHOD(a2, a3); \ 95 _NV_PUSH_INC_1U(SubCh, a1, d1, 3); \ 96 NV_PUSH_DATA(d2); \ 97 NV_PUSH_DATA(d3); \ 98 } while (0) 99 100 #define NV_PUSH_INC_4U(SubCh, a1, d1, a2, d2, a3, d3, a4, d4) \ 101 do \ 102 { \ 103 _NV_ASSERT_CONTIGUOUS_METHOD(a1, a2); \ 104 _NV_ASSERT_CONTIGUOUS_METHOD(a2, a3); \ 105 _NV_ASSERT_CONTIGUOUS_METHOD(a3, a4); \ 106 _NV_PUSH_INC_1U(SubCh, a1, d1, 4); \ 107 NV_PUSH_DATA(d2); \ 108 NV_PUSH_DATA(d3); \ 109 NV_PUSH_DATA(d4); \ 110 } while (0) 111 112 #define READ_CHANNEL_PAYLOAD_SEMA(channel) channelReadChannelMemdesc(channel, channel->finishPayloadOffset) 113 #define READ_CHANNEL_PB_SEMA(channel) channelReadChannelMemdesc(channel, channel->semaOffset) 114 115 // 116 // This struct contains parameters needed to send a pushbuffer for a CE 117 // operation. This interface only supports contiguous operations. 118 // 119 typedef struct 120 { 121 NvBool bCeMemcopy; // Whether this is a CE memcopy; 122 // If set to false, this will be a memset operation 123 NvU64 dstAddr; // Physical address of the source address 124 NvU64 srcAddr; // Physical address of the source address; only valid for memcopy 125 NvU32 size; 126 NvU32 pattern; // Fixed pattern to memset to. Only valid for memset 127 NvU32 payload; // Payload value used to release semaphore 128 NvU64 clientSemaAddr; 129 NV_ADDRESS_SPACE dstAddressSpace; 130 NV_ADDRESS_SPACE srcAddressSpace; 131 NvU32 dstCpuCacheAttrib; 132 NvU32 srcCpuCacheAttrib; 133 134 NvBool bSecureCopy; // The copy encrypts/decrypts protected memory 135 NvBool bEncrypt; // encrypt/decrypt 136 NvU64 authTagAddr; 137 NvU64 encryptIvAddr; 138 139 } CHANNEL_PB_INFO; 140 141 NV_STATUS channelSetupIDs(OBJCHANNEL *pChannel, OBJGPU *pGpu, NvBool bUseVasForCeCopy, NvBool bMIGInUse); 142 NV_STATUS channelAllocSubdevice(OBJGPU *pGpu, OBJCHANNEL *pChannel); 143 void channelSetupChannelBufferSizes(OBJCHANNEL *pChannel); 144 NvU32 channelReadChannelMemdesc(OBJCHANNEL *pChannel, NvU32 offset); 145 146 // Needed for pushbuffer management 147 NV_STATUS channelWaitForFreeEntry(OBJCHANNEL *pChannel, NvU32 *pPutIndex); 148 NV_STATUS channelFillGpFifo(OBJCHANNEL *pChannel, NvU32 putIndex, NvU32 methodsLength); 149 NvU32 channelFillCePb(OBJCHANNEL *pChannel, NvU32 putIndex, NvBool bPipelined, 150 NvBool bInsertFinishPayload, CHANNEL_PB_INFO *pChannelPbInfo); 151 NvU32 channelFillPbFastScrub(OBJCHANNEL *pChannel, NvU32 putIndex, NvBool bPipelined, 152 NvBool bInsertFinishPayload, CHANNEL_PB_INFO *pChannelPbInfo); 153 154 NV_STATUS channelFillSec2Pb(OBJCHANNEL *pChannel, NvU32 putIndex, NvBool bInsertFinishPayload, 155 CHANNEL_PB_INFO *pChannelPbInfo, CCSL_CONTEXT *pCcslCtx, 156 MEMORY_DESCRIPTOR *pScrubMemDesc, MEMORY_DESCRIPTOR *pSemaMemDesc, 157 NvU64 scrubMthdAuthTagBufGpuVA, NvU32 scrubAuthTagBufIndex, 158 NvU64 semaMthdAuthTagBufGpuVA, NvU32 semaAuthTagBufIndex, NvU32* methodLength); 159 160 // Needed for work tracking 161 NV_STATUS channelWaitForFinishPayload(OBJCHANNEL *pChannel, NvU64 targetPayload); 162 NvU64 channelGetFinishPayload(OBJCHANNEL *pChannel); 163 164 void channelServiceScrubberInterrupts(OBJCHANNEL *pChannel); 165 166 #endif // _CHANNEL_UTILS_H_ 167