1 /******************************************************************************* 2 Copyright (c) 2015-2023 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #ifndef __UVM_PUSH_MACROS_H__ 25 #define __UVM_PUSH_MACROS_H__ 26 27 #include "uvm_extern_decl.h" 28 #include "uvm_forward_decl.h" 29 #include "uvm_channel.h" 30 #include "nvtypes.h" 31 #include "nvmisc.h" 32 #include "cla06fsubch.h" 33 #include "cla16f.h" 34 #include "clb06f.h" 35 36 #define HWMASK(d, r, f) DRF_MASK(NV ## d ## _ ## r ## _ ## f) 37 #define HWSHIFT(d, r, f) DRF_SHIFT(NV ## d ## _ ## r ## _ ## f) 38 #define HWSHIFTMASK(d, r, f) DRF_SHIFTMASK(NV ## d ## _ ## r ## _ ## f) 39 #define HWSIZE(d, r, f) DRF_SIZE(NV ## d ## _ ## r ## _ ## f) 40 #define HWCONST(d, r, f, c) DRF_DEF(d, _ ## r, _ ## f, _ ## c) 41 #define HWVALUE(d, r, f, v) ({ \ 42 NvU32 _v = (v); \ 43 NvU32 val = DRF_NUM(d, _ ## r, _ ## f, _v); \ 44 UVM_ASSERT_MSG(_v == DRF_VAL(d, _ ## r, _ ## f, val), "v 0x%x mask 0x%x\n", _v, HWMASK(d, r, f)); \ 45 val; \ 46 }) 47 48 #define HWMASK64(d, r, f) DRF_MASK64(NV ## d ## _ ## r ## _ ## f) 49 #define HWCONST64(d, r, f, c) DRF_DEF64(d, _ ## r, _ ## f, _ ## c) 50 #define HWVALUE64(d, r, f, v) ({ \ 51 NvU64 _v = (v); \ 52 NvU64 val = DRF_NUM64(d, _ ## r, _ ## f, _v); \ 53 UVM_ASSERT_MSG(_v == DRF_VAL64(d, _ ## r, _ ## f, val), "v 0x%llx mask 0x%llx\n", _v, HWMASK64(d, r, f)); \ 54 val; \ 55 }) 56 57 #define HWMASK_MW(d, r, f) DRF_MASK_MW(NV ## d ## _ ## r ## _ ## f) 58 #define HWSIZE_MW(d, r, f) DRF_SIZE_MW(NV ## d ## _ ## r ## _ ## f) 59 #define HWCONST_MW(d, r, f, c) DRF_DEF_MW(d, _ ## r, _ ## f, _ ## c) 60 #define HWVALUE_MW(d, r, f, v) ({ \ 61 NvU32 _v = (v); \ 62 NvU32 val = DRF_NUM_MW(d, _ ## r, _ ## f, _v); \ 63 UVM_ASSERT_MSG(_v == DRF_VAL_MW(d, _ ## r, _ ## f, val), "v 0x%x mask 0x%x\n", _v, HWMASK_MW(d, r, f)); \ 64 val; \ 65 }) 66 67 #define WRITE_HWCONST(v, d, r, f, c) FLD_SET_DRF(d,_##r,_##f,_##c, v) 68 #define WRITE_HWVALUE(v, d, r, f, n) FLD_SET_DRF_NUM(d,_##r,_##f, n, v) 69 70 // nvmisc.h has FLD_SET_DRF_NUM64, but not FLD_SET_DRF64 71 #define UVM_FLD_SET_DRF64(d,r,f,c,v) ((((NvU64)(v)) & ~DRF_SHIFTMASK64(NV##d##r##f)) | DRF_DEF64(d,r,f,c)) 72 73 #define WRITE_HWCONST64(v, d, r, f, c) UVM_FLD_SET_DRF64(d,_##r,_##f,_##c, v) 74 #define WRITE_HWVALUE64(v, d, r, f, n) FLD_SET_DRF_NUM64(d,_##r,_##f, n, v) 75 76 #define READ_HWVALUE(v, d, r, f) DRF_VAL(d,_##r,_##f, v) 77 #define READ_HWVALUE_MW(v, d, r, f) DRF_VAL_MW(d,_##r,_##f, v) 78 #define WRITE_HWCONST_MW(v, d, r, f, c) FLD_ASSIGN_MW(NV##d##_##r##_##f, DRF_DEF_MW(d,_##r,_##f,_##c), v) 79 #define WRITE_HWVALUE_MW(v, d, r, f, n) FLD_ASSIGN_MW(NV##d##_##r##_##f, DRF_NUM_MW(d,_##r,_##f,n), v) 80 81 // Host methods ignore the subchannel, just use 0 82 #define UVM_SUBCHANNEL_HOST 0 83 84 // Starting with Kepler HW settled on using a fixed subchannel for CE. 85 #define UVM_SUBCHANNEL_CE NVA06F_SUBCHANNEL_COPY_ENGINE 86 87 #define UVM_SUBCHANNEL_A16F UVM_SUBCHANNEL_HOST 88 89 #define UVM_SUBCHANNEL_B06F UVM_SUBCHANNEL_HOST 90 #define UVM_SUBCHANNEL_B0B5 UVM_SUBCHANNEL_CE 91 92 #define UVM_SUBCHANNEL_C06F UVM_SUBCHANNEL_HOST 93 #define UVM_SUBCHANNEL_C0B5 UVM_SUBCHANNEL_CE 94 95 #define UVM_SUBCHANNEL_C36F UVM_SUBCHANNEL_HOST 96 #define UVM_SUBCHANNEL_C3B5 UVM_SUBCHANNEL_CE 97 98 #define UVM_SUBCHANNEL_C46F UVM_SUBCHANNEL_HOST 99 100 #define UVM_SUBCHANNEL_C56F UVM_SUBCHANNEL_HOST 101 #define UVM_SUBCHANNEL_C6B5 UVM_SUBCHANNEL_CE 102 103 #define UVM_SUBCHANNEL_C86F UVM_SUBCHANNEL_HOST 104 #define UVM_SUBCHANNEL_C8B5 UVM_SUBCHANNEL_CE 105 106 // Channel for UVM SW methods. This is defined in nv_uvm_types.h. RM does not 107 // care about the specific number as long as it's bigger than the largest HW 108 // value. For example, Kepler reserves subchannels 5-7 for software objects. 109 #define UVM_SUBCHANNEL_C076 UVM_SW_OBJ_SUBCHANNEL 110 111 // NVA06F_SUBCHANNEL_COMPUTE is a semi-arbitrary value for UVM_SUBCHANNEL_SEC2. 112 // We need a "unique" subchannel across all subchannels UVM submits work. This 113 // is used when we are post-processing a pushbuffer and we need to extract SEC2 114 // methods from a it, having a unique subchannel facilitates the SEC2 method 115 // identification. 116 #define UVM_SUBCHANNEL_SEC2 NVA06F_SUBCHANNEL_COMPUTE 117 #define UVM_SUBCHANNEL_CBA2 UVM_SUBCHANNEL_SEC2 118 119 #define UVM_METHOD_SIZE 4 120 #define UVM_METHOD_COUNT_MAX HWMASK(B06F, DMA, INCR_COUNT) 121 #if HWMASK(B06F, DMA, INCR_COUNT) != HWMASK(B06F, DMA, NONINCR_COUNT) 122 #error "Unable to define UVM_METHOD_COUNT_MAX" 123 #endif 124 125 #define UVM_METHOD_INC(subch, address, count) \ 126 (HWCONST(B06F, DMA, SEC_OP, INC_METHOD) | \ 127 HWVALUE(B06F, DMA, INCR_ADDRESS, (address) >> 2) | \ 128 HWVALUE(B06F, DMA, INCR_SUBCHANNEL, (subch)) | \ 129 HWVALUE(B06F, DMA, INCR_COUNT, (count))) 130 131 #define UVM_METHOD_NONINC(subch, address, count) \ 132 (HWCONST(B06F, DMA, SEC_OP, NON_INC_METHOD) | \ 133 HWVALUE(B06F, DMA, NONINCR_ADDRESS, (address) >> 2) | \ 134 HWVALUE(B06F, DMA, NONINCR_SUBCHANNEL, (subch)) | \ 135 HWVALUE(B06F, DMA, NONINCR_COUNT, (count))) 136 137 #define __UVM_ASSERT_CONTIGUOUS_METHODS(a1, a2) BUILD_BUG_ON((a2) - (a1) != UVM_METHOD_SIZE) 138 139 // __NV_PUSH_*U support being called recursively from the N+1 sized method with 140 // the _0U doing all the common things. 141 // Notably all the push macros assume that symbol "push" of type uvm_push_t * is 142 // in scope. 143 #define __NV_PUSH_0U(subch, count, a1) \ 144 do { \ 145 UVM_ASSERT(!uvm_global_is_suspended()); \ 146 UVM_ASSERT(uvm_push_get_size(push) + (count + 1) * UVM_METHOD_SIZE <= UVM_MAX_PUSH_SIZE); \ 147 UVM_ASSERT_MSG(IS_ALIGNED(a1, UVM_METHOD_SIZE), "Address %u\n", a1); \ 148 \ 149 push->next[0] = UVM_METHOD_INC(subch, a1, count); \ 150 ++push->next; \ 151 } while (0) 152 153 #define __NV_PUSH_1U(subch, count, a1,d1) \ 154 do { \ 155 __NV_PUSH_0U(subch, count, a1); \ 156 UVM_ASSERT_MSG(uvm_push_method_is_valid(push, subch, a1, d1), \ 157 "Method validation failed in channel %s\n", \ 158 push->channel->name); \ 159 push->next[0] = d1; \ 160 ++push->next; \ 161 } while (0) 162 163 #define __NV_PUSH_2U(subch, count, a1,d1, a2,d2) \ 164 do { \ 165 __UVM_ASSERT_CONTIGUOUS_METHODS(a1, a2); \ 166 __NV_PUSH_1U(subch, count, a1,d1); \ 167 UVM_ASSERT_MSG(uvm_push_method_is_valid(push, subch, a2, d2), \ 168 "Method validation failed in channel %s\n", \ 169 push->channel->name); \ 170 push->next[0] = d2; \ 171 ++push->next; \ 172 } while (0) 173 174 #define __NV_PUSH_3U(subch, count, a1,d1, a2,d2, a3,d3) \ 175 do { \ 176 __UVM_ASSERT_CONTIGUOUS_METHODS(a2, a3); \ 177 __NV_PUSH_2U(subch, count, a1,d1, a2,d2); \ 178 UVM_ASSERT_MSG(uvm_push_method_is_valid(push, subch, a3, d3), \ 179 "Method validation failed in channel %s\n", \ 180 push->channel->name); \ 181 push->next[0] = d3; \ 182 ++push->next; \ 183 } while (0) 184 185 #define __NV_PUSH_4U(subch, count, a1,d1, a2,d2, a3,d3, a4,d4) \ 186 do { \ 187 __UVM_ASSERT_CONTIGUOUS_METHODS(a3, a4); \ 188 __NV_PUSH_3U(subch, count, a1,d1, a2,d2, a3,d3); \ 189 UVM_ASSERT_MSG(uvm_push_method_is_valid(push, subch, a4, d4), \ 190 "Method validation failed in channel %s\n", \ 191 push->channel->name); \ 192 push->next[0] = d4; \ 193 ++push->next; \ 194 } while (0) 195 196 #define __NV_PUSH_5U(subch, count, a1,d1, a2,d2, a3,d3, a4,d4, a5,d5) \ 197 do { \ 198 __UVM_ASSERT_CONTIGUOUS_METHODS(a4, a5); \ 199 __NV_PUSH_4U(subch, count, a1,d1, a2,d2, a3,d3, a4,d4); \ 200 UVM_ASSERT_MSG(uvm_push_method_is_valid(push, subch, a5, d5), \ 201 "Method validation failed in channel %s\n", \ 202 push->channel->name); \ 203 push->next[0] = d5; \ 204 ++push->next; \ 205 } while (0) 206 207 #define __NV_PUSH_6U(subch, count, a1,d1, a2,d2, a3,d3, a4,d4, a5,d5, a6,d6) \ 208 do { \ 209 __UVM_ASSERT_CONTIGUOUS_METHODS(a5, a6); \ 210 __NV_PUSH_5U(subch, count, a1,d1, a2,d2, a3,d3, a4,d4, a5,d5); \ 211 UVM_ASSERT_MSG(uvm_push_method_is_valid(push, subch, a6, d6), \ 212 "Method validation failed in channel %s\n", \ 213 push->channel->name); \ 214 push->next[0] = d6; \ 215 ++push->next; \ 216 } while (0) 217 218 #define NV_PUSH_1U(class, a1,d1) \ 219 __NV_PUSH_1U(UVM_SUBCHANNEL_ ## class, 1, \ 220 NV ## class ## _ ## a1, d1) 221 222 #define NV_PUSH_2U(class, a1,d1, a2,d2) \ 223 __NV_PUSH_2U(UVM_SUBCHANNEL_ ## class, 2, \ 224 NV ## class ## _ ## a1, d1, \ 225 NV ## class ## _ ## a2, d2) 226 227 #define NV_PUSH_3U(class, a1,d1, a2,d2, a3,d3) \ 228 __NV_PUSH_3U(UVM_SUBCHANNEL_ ## class, 3, \ 229 NV ## class ## _ ## a1, d1, \ 230 NV ## class ## _ ## a2, d2, \ 231 NV ## class ## _ ## a3, d3) 232 233 #define NV_PUSH_4U(class, a1,d1, a2,d2, a3,d3, a4,d4) \ 234 __NV_PUSH_4U(UVM_SUBCHANNEL_ ## class, 4, \ 235 NV ## class ## _ ## a1, d1, \ 236 NV ## class ## _ ## a2, d2, \ 237 NV ## class ## _ ## a3, d3, \ 238 NV ## class ## _ ## a4, d4) 239 240 #define NV_PUSH_5U(class, a1,d1, a2,d2, a3,d3, a4,d4, a5,d5) \ 241 __NV_PUSH_5U(UVM_SUBCHANNEL_ ## class, 5, \ 242 NV ## class ## _ ## a1, d1, \ 243 NV ## class ## _ ## a2, d2, \ 244 NV ## class ## _ ## a3, d3, \ 245 NV ## class ## _ ## a4, d4, \ 246 NV ## class ## _ ## a5, d5) 247 248 #define NV_PUSH_6U(class, a1,d1, a2,d2, a3,d3, a4,d4, a5,d5, a6,d6) \ 249 __NV_PUSH_6U(UVM_SUBCHANNEL_ ## class, 6, \ 250 NV ## class ## _ ## a1, d1, \ 251 NV ## class ## _ ## a2, d2, \ 252 NV ## class ## _ ## a3, d3, \ 253 NV ## class ## _ ## a4, d4, \ 254 NV ## class ## _ ## a5, d5, \ 255 NV ## class ## _ ## a6, d6) 256 257 // Non-incrementing method with count data fields following it. The data is left 258 // untouched and hence it's primarily useful for a NOP method. 259 #define __NV_PUSH_NU_NONINC(subch, count, address) \ 260 do { \ 261 UVM_ASSERT(!uvm_global_is_suspended()); \ 262 UVM_ASSERT(uvm_push_get_size(push) + (count + 1) * UVM_METHOD_SIZE <= UVM_MAX_PUSH_SIZE); \ 263 UVM_ASSERT_MSG(IS_ALIGNED(address, UVM_METHOD_SIZE), "Address %u\n", address); \ 264 push->next[0] = UVM_METHOD_NONINC(subch, address, count); \ 265 push->next += count + 1; \ 266 } while (0) 267 268 #define NV_PUSH_NU_NONINC(class, a1, count) \ 269 __NV_PUSH_NU_NONINC(UVM_SUBCHANNEL_ ## class, count, \ 270 NV ## class ## _ ## a1) 271 272 #endif // __UVM_PUSH_MACROS_H__ 273