1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #ifndef NVLINK_INBAND_MSG_HDR_H 25 #define NVLINK_INBAND_MSG_HDR_H 26 27 /* 28 * Messages do not have individual versioning, instead a strict ABI is maintained. When a change is 29 * required on existing message, instead of modifying corresponding message structure, a completely 30 * new message type (like INBAND_MSG_TYPE_XXX_V1, INBAND_MSG_TYPE_XXX_V2) and corresponding message 31 * definition structure needs to be added. Do not modify existing structs in any way. 32 * 33 * Messages may contain fields which are debug only and must be used for logging purpose. Such 34 * fields shouldn't be trusted. 35 * 36 * - Avoid use of enums or bitfields. Always use fixed types. 37 * - Avoid conditional fields in the structs. 38 * - Avoid nested and complex structs. Keep them simple and flat for ease of encoding and decoding. 39 * - Avoid embedded pointers. Flexible arrays at the end of the struct are allowed. 40 * - Always use the packed struct to typecast inband messages. More details: 41 * - Always have reserved flags or fields to CYA given the stable ABI conditions. 42 */ 43 44 /* Align to byte boundaries */ 45 #pragma pack(push, 1) 46 47 #include "nvtypes.h" 48 #include "nvmisc.h" 49 #include "nvCpuUuid.h" 50 #include "nvstatus.h" 51 #include "nvstatuscodes.h" 52 53 #define NVLINK_INBAND_MAX_MSG_SIZE 4096 54 #define NVLINK_INBAND_MSG_MAGIC_ID_FM 0xadbc 55 56 /* Nvlink Inband messages types */ 57 #define NVLINK_INBAND_MSG_TYPE_GPU_PROBE_REQ 0 58 #define NVLINK_INBAND_MSG_TYPE_GPU_PROBE_RSP 1 59 #define NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ 2 60 #define NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_RSP 3 61 #define NVLINK_INBAND_MSG_TYPE_MC_TEAM_RELEASE_REQ 4 62 #define NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ_V2 5 63 #define NVLINK_INBAND_MSG_TYPE_MAX 6 64 65 /* Nvlink Inband message packet header */ 66 typedef struct 67 { 68 NvU16 magicId; /* Identifier to represent in-band msg, will be NVLINK_INBAND_MSG_MAGIC_ID */ 69 NvU64 requestId; /* Unique Id for a request and response will carry same id */ 70 NV_STATUS status; /* High level status of the message/request */ 71 NvU16 type; /* Type of encoded message. One of NVLINK_INBAND_MSG_TYPE_xxx */ 72 NvU32 length; /* Length of encoded message */ 73 NvU8 reserved[8]; /* For future use. Must be initialized to zero */ 74 } nvlink_inband_msg_header_t; 75 76 #define NVLINK_INBAND_GPU_PROBE_CAPS_SRIOV_ENABLED NVBIT(0) 77 78 /* Add more caps as need in the future */ 79 80 #define NVLINK_INBAND_BW_MODE_FULL 0 81 #define NVLINK_INBAND_BW_MODE_OFF 1 82 #define NVLINK_INBAND_BW_MODE_MIN 2 83 #define NVLINK_INBAND_BW_MODE_HALF 3 84 #define NVLINK_INBAND_BW_MODE_3QUARTER 4 85 86 typedef struct 87 { 88 NvU64 pciInfo; /* Encoded as Domain(63:32):Bus(15:8):Device(0:7). (debug only) */ 89 NvU8 moduleId; /* GPIO based physical/module ID of the GPU. (debug only) */ 90 NvUuid gpuUuid; /* UUID of the GPU. (debug only) */ 91 NvU64 discoveredLinkMask; /* GPU's discovered NVLink mask info. (debug only) */ 92 NvU64 enabledLinkMask; /* GPU's currently enabled NvLink mask info. (debug only) */ 93 94 NvU32 gpuCapMask; /* GPU capabilities, one of NVLINK_INBAND_GPU_PROBE_CAPS */ 95 NvU8 bwMode; /* NVLink bandwidth mode, one of NVLINK_INBAND_BW_MODE */ 96 NvU8 reserved[31]; /* For future use. Must be initialized to zero */ 97 } nvlink_inband_gpu_probe_req_t; 98 99 typedef struct 100 { 101 nvlink_inband_msg_header_t msgHdr; 102 nvlink_inband_gpu_probe_req_t probeReq; 103 } nvlink_inband_gpu_probe_req_msg_t; 104 105 #define NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V1 NVBIT64(0) 106 #define NVLINK_INBAND_FM_CAPS_MC_TEAM_RELEASE_V1 NVBIT64(1) 107 #define NVLINK_INBAND_FM_CAPS_BW_MODE_MIN NVBIT64(2) 108 #define NVLINK_INBAND_FM_CAPS_BW_MODE_HALF NVBIT64(3) 109 #define NVLINK_INBAND_FM_CAPS_BW_MODE_3QUARTER NVBIT64(4) 110 #define NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V2 NVBIT64(5) 111 112 typedef struct 113 { 114 NvU64 gpuHandle; /* Unique handle assigned by initialization entity for this GPU */ 115 NvU32 gfId; /* GFID which supports NVLink */ 116 NvU64 fmCaps; /* Capability of FM e.g. what features FM support. */ 117 NvUuid clusterUuid; /* Cluster UUID to which this node belongs */ 118 NvU16 fabricPartitionId; /* Partition ID if the GPU belongs to a fabric partition */ 119 NvU64 gpaAddress; /* GPA starting address for the GPU */ 120 NvU64 gpaAddressRange; /* GPU GPA address range */ 121 NvU64 flaAddress; /* FLA starting address for the GPU */ 122 NvU64 flaAddressRange; /* GPU FLA address range */ 123 NvU32 linkMaskToBeReduced; /* bit mask of unused NVLink ports for P2P */ 124 NvU32 cliqueId; /* Fabric Clique Id */ 125 NvU8 reserved[24]; /* For future use. Must be initialized to zero */ 126 } nvlink_inband_gpu_probe_rsp_t; 127 128 typedef struct 129 { 130 nvlink_inband_msg_header_t msgHdr; 131 nvlink_inband_gpu_probe_rsp_t probeRsp; 132 } nvlink_inband_gpu_probe_rsp_msg_t; 133 134 typedef struct 135 { 136 NvU64 mcAllocSize; /* Multicast allocation size requested */ 137 NvU32 flags; /* For future use. Must be initialized to zero */ 138 NvU8 reserved[8]; /* For future use. Must be initialized to zero */ 139 NvU16 numGpuHandles; /* Number of GPUs in this team */ 140 NvU64 gpuHandles[]; /* Array of probed handles, should be last */ 141 } nvlink_inband_mc_team_setup_req_t; 142 143 typedef struct 144 { 145 nvlink_inband_msg_header_t msgHdr; 146 nvlink_inband_mc_team_setup_req_t mcTeamSetupReq; 147 } nvlink_inband_mc_team_setup_req_msg_t; 148 149 typedef struct 150 { 151 NvU64 mcAllocSize; /* Multicast allocation size requested */ 152 NvU32 flags; /* For future use. Must be initialized to zero */ 153 NvU8 reserved[8]; /* For future use. Must be initialized to zero */ 154 NvU16 numGpuHandles; /* Number of GPUs in this team */ 155 NvU16 numKeys; /* Number of keys (a.k.a request ID) used by FM to send response */ 156 NvU64 gpuHandlesAndKeys[]; /* Array of probed handles and keys, should be last */ 157 158 /* 159 * The array will be grouped and ordered as: <allGpuHandlesOfNodeA, allGpuHandlesOfNodeB,... 160 * keyForNodeA, keyForNodeB>. The first group of gpuHandles will belong to the exporter node, 161 * which will be followed by the importer nodes. 162 * 163 * Test case: If the exporter and importer nodes are same, then the message will 164 * have multiple keys belonging to the same node as: <allGpuHandlesOfNodeA,... 165 * key1ForNodeA, key2ForNodeA>. Even though all gpuHandles belong to the same node, the 166 * first key should be considered from the exporter node and the rest from the importer 167 * nodes. 168 */ 169 } nvlink_inband_mc_team_setup_req_v2_t; 170 171 typedef struct 172 { 173 nvlink_inband_msg_header_t msgHdr; 174 nvlink_inband_mc_team_setup_req_v2_t mcTeamSetupReq; 175 } nvlink_inband_mc_team_setup_req_v2_msg_t; 176 177 typedef struct 178 { 179 NvU64 mcTeamHandle; /* Unique handle assigned for this Multicast team */ 180 /* Should be zero if the response is sent to the importer nodes */ 181 NvU32 flags; /* For future use. Must be initialized to zero */ 182 NvU8 reserved[8]; /* For future use. Must be initialized to zero */ 183 NvU64 mcAddressBase; /* FLA starting address assigned for the Multicast slot */ 184 NvU64 mcAddressSize; /* Should be same as mcAllocSize */ 185 } nvlink_inband_mc_team_setup_rsp_t; 186 187 typedef struct 188 { 189 nvlink_inband_msg_header_t msgHdr; 190 nvlink_inband_mc_team_setup_rsp_t mcTeamSetupRsp; 191 } nvlink_inband_mc_team_setup_rsp_msg_t; 192 193 typedef struct 194 { 195 NvU64 mcTeamHandle; /* Unique handle assigned for the Multicast team */ 196 NvU32 flags; /* For future use. Must be initialized to zero */ 197 NvU8 reserved[8]; /* For future use. Must be initialized to zero */ 198 } nvlink_inband_mc_team_release_req_t; 199 200 typedef struct 201 { 202 nvlink_inband_msg_header_t msgHdr; 203 nvlink_inband_mc_team_release_req_t mcTeamReleaseReq; 204 } nvlink_inband_mc_team_release_req_msg_t; 205 206 #pragma pack(pop) 207 208 /********************* Don't add any message structs after this line ******************************/ 209 210 /* Helpers */ 211 static NV_INLINE void nvlinkInitInbandMsgHdr 212 ( 213 nvlink_inband_msg_header_t *pMsgHdr, 214 NvU16 type, 215 NvU32 len, 216 NvU64 requestId 217 ) 218 { 219 NvU8 i; 220 221 pMsgHdr->requestId = requestId; 222 pMsgHdr->magicId = NVLINK_INBAND_MSG_MAGIC_ID_FM; 223 pMsgHdr->type = type; 224 pMsgHdr->length = len; 225 pMsgHdr->status = NV_OK; 226 227 for (i = 0; i < sizeof(pMsgHdr->reserved); i++) 228 pMsgHdr->reserved[i] = 0; 229 } 230 231 #endif 232