1//===--- BuiltinsPTX.def - PTX Builtin function database ----*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the PTX-specific builtin function database. Users of 10// this file must define the BUILTIN macro to make use of this information. 11// 12//===----------------------------------------------------------------------===// 13 14// The format of this database matches clang/Basic/Builtins.def. 15 16#if defined(BUILTIN) && !defined(TARGET_BUILTIN) 17# define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS) 18#endif 19 20#pragma push_macro("SM_70") 21#pragma push_macro("SM_72") 22#pragma push_macro("SM_75") 23#pragma push_macro("SM_80") 24#define SM_80 "sm_80" 25#define SM_75 "sm_75|" SM_80 26#define SM_72 "sm_72|" SM_75 27#define SM_70 "sm_70|" SM_72 28 29#pragma push_macro("SM_60") 30#define SM_60 "sm_60|sm_61|sm_62|" SM_70 31 32#pragma push_macro("PTX60") 33#pragma push_macro("PTX61") 34#pragma push_macro("PTX63") 35#pragma push_macro("PTX64") 36#pragma push_macro("PTX65") 37#pragma push_macro("PTX70") 38#define PTX70 "ptx70" 39#define PTX65 "ptx65|" PTX70 40#define PTX64 "ptx64|" PTX65 41#define PTX63 "ptx63|" PTX64 42#define PTX61 "ptx61|" PTX63 43#define PTX60 "ptx60|" PTX61 44 45#pragma push_macro("AND") 46#define AND(a, b) a "," b 47 48// Special Registers 49 50BUILTIN(__nvvm_read_ptx_sreg_tid_x, "i", "nc") 51BUILTIN(__nvvm_read_ptx_sreg_tid_y, "i", "nc") 52BUILTIN(__nvvm_read_ptx_sreg_tid_z, "i", "nc") 53BUILTIN(__nvvm_read_ptx_sreg_tid_w, "i", "nc") 54 55BUILTIN(__nvvm_read_ptx_sreg_ntid_x, "i", "nc") 56BUILTIN(__nvvm_read_ptx_sreg_ntid_y, "i", "nc") 57BUILTIN(__nvvm_read_ptx_sreg_ntid_z, "i", "nc") 58BUILTIN(__nvvm_read_ptx_sreg_ntid_w, "i", "nc") 59 60BUILTIN(__nvvm_read_ptx_sreg_ctaid_x, "i", "nc") 61BUILTIN(__nvvm_read_ptx_sreg_ctaid_y, "i", "nc") 62BUILTIN(__nvvm_read_ptx_sreg_ctaid_z, "i", "nc") 63BUILTIN(__nvvm_read_ptx_sreg_ctaid_w, "i", "nc") 64 65BUILTIN(__nvvm_read_ptx_sreg_nctaid_x, "i", "nc") 66BUILTIN(__nvvm_read_ptx_sreg_nctaid_y, "i", "nc") 67BUILTIN(__nvvm_read_ptx_sreg_nctaid_z, "i", "nc") 68BUILTIN(__nvvm_read_ptx_sreg_nctaid_w, "i", "nc") 69 70BUILTIN(__nvvm_read_ptx_sreg_laneid, "i", "nc") 71BUILTIN(__nvvm_read_ptx_sreg_warpid, "i", "nc") 72BUILTIN(__nvvm_read_ptx_sreg_nwarpid, "i", "nc") 73 74BUILTIN(__nvvm_read_ptx_sreg_smid, "i", "nc") 75BUILTIN(__nvvm_read_ptx_sreg_nsmid, "i", "nc") 76BUILTIN(__nvvm_read_ptx_sreg_gridid, "i", "nc") 77 78BUILTIN(__nvvm_read_ptx_sreg_lanemask_eq, "i", "nc") 79BUILTIN(__nvvm_read_ptx_sreg_lanemask_le, "i", "nc") 80BUILTIN(__nvvm_read_ptx_sreg_lanemask_lt, "i", "nc") 81BUILTIN(__nvvm_read_ptx_sreg_lanemask_ge, "i", "nc") 82BUILTIN(__nvvm_read_ptx_sreg_lanemask_gt, "i", "nc") 83 84BUILTIN(__nvvm_read_ptx_sreg_clock, "i", "n") 85BUILTIN(__nvvm_read_ptx_sreg_clock64, "LLi", "n") 86 87BUILTIN(__nvvm_read_ptx_sreg_pm0, "i", "n") 88BUILTIN(__nvvm_read_ptx_sreg_pm1, "i", "n") 89BUILTIN(__nvvm_read_ptx_sreg_pm2, "i", "n") 90BUILTIN(__nvvm_read_ptx_sreg_pm3, "i", "n") 91 92// MISC 93 94BUILTIN(__nvvm_prmt, "UiUiUiUi", "") 95 96// Min Max 97 98BUILTIN(__nvvm_fmax_ftz_f, "fff", "") 99BUILTIN(__nvvm_fmax_f, "fff", "") 100BUILTIN(__nvvm_fmin_ftz_f, "fff", "") 101BUILTIN(__nvvm_fmin_f, "fff", "") 102 103BUILTIN(__nvvm_fmax_d, "ddd", "") 104BUILTIN(__nvvm_fmin_d, "ddd", "") 105 106// Multiplication 107 108BUILTIN(__nvvm_mulhi_i, "iii", "") 109BUILTIN(__nvvm_mulhi_ui, "UiUiUi", "") 110BUILTIN(__nvvm_mulhi_ll, "LLiLLiLLi", "") 111BUILTIN(__nvvm_mulhi_ull, "ULLiULLiULLi", "") 112 113BUILTIN(__nvvm_mul_rn_ftz_f, "fff", "") 114BUILTIN(__nvvm_mul_rn_f, "fff", "") 115BUILTIN(__nvvm_mul_rz_ftz_f, "fff", "") 116BUILTIN(__nvvm_mul_rz_f, "fff", "") 117BUILTIN(__nvvm_mul_rm_ftz_f, "fff", "") 118BUILTIN(__nvvm_mul_rm_f, "fff", "") 119BUILTIN(__nvvm_mul_rp_ftz_f, "fff", "") 120BUILTIN(__nvvm_mul_rp_f, "fff", "") 121 122BUILTIN(__nvvm_mul_rn_d, "ddd", "") 123BUILTIN(__nvvm_mul_rz_d, "ddd", "") 124BUILTIN(__nvvm_mul_rm_d, "ddd", "") 125BUILTIN(__nvvm_mul_rp_d, "ddd", "") 126 127BUILTIN(__nvvm_mul24_i, "iii", "") 128BUILTIN(__nvvm_mul24_ui, "UiUiUi", "") 129 130// Div 131 132BUILTIN(__nvvm_div_approx_ftz_f, "fff", "") 133BUILTIN(__nvvm_div_approx_f, "fff", "") 134 135BUILTIN(__nvvm_div_rn_ftz_f, "fff", "") 136BUILTIN(__nvvm_div_rn_f, "fff", "") 137BUILTIN(__nvvm_div_rz_ftz_f, "fff", "") 138BUILTIN(__nvvm_div_rz_f, "fff", "") 139BUILTIN(__nvvm_div_rm_ftz_f, "fff", "") 140BUILTIN(__nvvm_div_rm_f, "fff", "") 141BUILTIN(__nvvm_div_rp_ftz_f, "fff", "") 142BUILTIN(__nvvm_div_rp_f, "fff", "") 143 144BUILTIN(__nvvm_div_rn_d, "ddd", "") 145BUILTIN(__nvvm_div_rz_d, "ddd", "") 146BUILTIN(__nvvm_div_rm_d, "ddd", "") 147BUILTIN(__nvvm_div_rp_d, "ddd", "") 148 149// Sad 150 151BUILTIN(__nvvm_sad_i, "iiii", "") 152BUILTIN(__nvvm_sad_ui, "UiUiUiUi", "") 153 154// Floor, Ceil 155 156BUILTIN(__nvvm_floor_ftz_f, "ff", "") 157BUILTIN(__nvvm_floor_f, "ff", "") 158BUILTIN(__nvvm_floor_d, "dd", "") 159 160BUILTIN(__nvvm_ceil_ftz_f, "ff", "") 161BUILTIN(__nvvm_ceil_f, "ff", "") 162BUILTIN(__nvvm_ceil_d, "dd", "") 163 164// Abs 165 166BUILTIN(__nvvm_fabs_ftz_f, "ff", "") 167BUILTIN(__nvvm_fabs_f, "ff", "") 168BUILTIN(__nvvm_fabs_d, "dd", "") 169 170// Round 171 172BUILTIN(__nvvm_round_ftz_f, "ff", "") 173BUILTIN(__nvvm_round_f, "ff", "") 174BUILTIN(__nvvm_round_d, "dd", "") 175 176// Trunc 177 178BUILTIN(__nvvm_trunc_ftz_f, "ff", "") 179BUILTIN(__nvvm_trunc_f, "ff", "") 180BUILTIN(__nvvm_trunc_d, "dd", "") 181 182// Saturate 183 184BUILTIN(__nvvm_saturate_ftz_f, "ff", "") 185BUILTIN(__nvvm_saturate_f, "ff", "") 186BUILTIN(__nvvm_saturate_d, "dd", "") 187 188// Exp2, Log2 189 190BUILTIN(__nvvm_ex2_approx_ftz_f, "ff", "") 191BUILTIN(__nvvm_ex2_approx_f, "ff", "") 192BUILTIN(__nvvm_ex2_approx_d, "dd", "") 193 194BUILTIN(__nvvm_lg2_approx_ftz_f, "ff", "") 195BUILTIN(__nvvm_lg2_approx_f, "ff", "") 196BUILTIN(__nvvm_lg2_approx_d, "dd", "") 197 198// Sin, Cos 199 200BUILTIN(__nvvm_sin_approx_ftz_f, "ff", "") 201BUILTIN(__nvvm_sin_approx_f, "ff", "") 202 203BUILTIN(__nvvm_cos_approx_ftz_f, "ff", "") 204BUILTIN(__nvvm_cos_approx_f, "ff", "") 205 206// Fma 207 208BUILTIN(__nvvm_fma_rn_ftz_f, "ffff", "") 209BUILTIN(__nvvm_fma_rn_f, "ffff", "") 210BUILTIN(__nvvm_fma_rz_ftz_f, "ffff", "") 211BUILTIN(__nvvm_fma_rz_f, "ffff", "") 212BUILTIN(__nvvm_fma_rm_ftz_f, "ffff", "") 213BUILTIN(__nvvm_fma_rm_f, "ffff", "") 214BUILTIN(__nvvm_fma_rp_ftz_f, "ffff", "") 215BUILTIN(__nvvm_fma_rp_f, "ffff", "") 216BUILTIN(__nvvm_fma_rn_d, "dddd", "") 217BUILTIN(__nvvm_fma_rz_d, "dddd", "") 218BUILTIN(__nvvm_fma_rm_d, "dddd", "") 219BUILTIN(__nvvm_fma_rp_d, "dddd", "") 220 221// Rcp 222 223BUILTIN(__nvvm_rcp_rn_ftz_f, "ff", "") 224BUILTIN(__nvvm_rcp_rn_f, "ff", "") 225BUILTIN(__nvvm_rcp_rz_ftz_f, "ff", "") 226BUILTIN(__nvvm_rcp_rz_f, "ff", "") 227BUILTIN(__nvvm_rcp_rm_ftz_f, "ff", "") 228BUILTIN(__nvvm_rcp_rm_f, "ff", "") 229BUILTIN(__nvvm_rcp_rp_ftz_f, "ff", "") 230BUILTIN(__nvvm_rcp_rp_f, "ff", "") 231 232BUILTIN(__nvvm_rcp_rn_d, "dd", "") 233BUILTIN(__nvvm_rcp_rz_d, "dd", "") 234BUILTIN(__nvvm_rcp_rm_d, "dd", "") 235BUILTIN(__nvvm_rcp_rp_d, "dd", "") 236BUILTIN(__nvvm_rcp_approx_ftz_d, "dd", "") 237 238// Sqrt 239 240BUILTIN(__nvvm_sqrt_rn_ftz_f, "ff", "") 241BUILTIN(__nvvm_sqrt_rn_f, "ff", "") 242BUILTIN(__nvvm_sqrt_rz_ftz_f, "ff", "") 243BUILTIN(__nvvm_sqrt_rz_f, "ff", "") 244BUILTIN(__nvvm_sqrt_rm_ftz_f, "ff", "") 245BUILTIN(__nvvm_sqrt_rm_f, "ff", "") 246BUILTIN(__nvvm_sqrt_rp_ftz_f, "ff", "") 247BUILTIN(__nvvm_sqrt_rp_f, "ff", "") 248BUILTIN(__nvvm_sqrt_approx_ftz_f, "ff", "") 249BUILTIN(__nvvm_sqrt_approx_f, "ff", "") 250 251BUILTIN(__nvvm_sqrt_rn_d, "dd", "") 252BUILTIN(__nvvm_sqrt_rz_d, "dd", "") 253BUILTIN(__nvvm_sqrt_rm_d, "dd", "") 254BUILTIN(__nvvm_sqrt_rp_d, "dd", "") 255 256// Rsqrt 257 258BUILTIN(__nvvm_rsqrt_approx_ftz_f, "ff", "") 259BUILTIN(__nvvm_rsqrt_approx_f, "ff", "") 260BUILTIN(__nvvm_rsqrt_approx_d, "dd", "") 261 262// Add 263 264BUILTIN(__nvvm_add_rn_ftz_f, "fff", "") 265BUILTIN(__nvvm_add_rn_f, "fff", "") 266BUILTIN(__nvvm_add_rz_ftz_f, "fff", "") 267BUILTIN(__nvvm_add_rz_f, "fff", "") 268BUILTIN(__nvvm_add_rm_ftz_f, "fff", "") 269BUILTIN(__nvvm_add_rm_f, "fff", "") 270BUILTIN(__nvvm_add_rp_ftz_f, "fff", "") 271BUILTIN(__nvvm_add_rp_f, "fff", "") 272 273BUILTIN(__nvvm_add_rn_d, "ddd", "") 274BUILTIN(__nvvm_add_rz_d, "ddd", "") 275BUILTIN(__nvvm_add_rm_d, "ddd", "") 276BUILTIN(__nvvm_add_rp_d, "ddd", "") 277 278// Convert 279 280BUILTIN(__nvvm_d2f_rn_ftz, "fd", "") 281BUILTIN(__nvvm_d2f_rn, "fd", "") 282BUILTIN(__nvvm_d2f_rz_ftz, "fd", "") 283BUILTIN(__nvvm_d2f_rz, "fd", "") 284BUILTIN(__nvvm_d2f_rm_ftz, "fd", "") 285BUILTIN(__nvvm_d2f_rm, "fd", "") 286BUILTIN(__nvvm_d2f_rp_ftz, "fd", "") 287BUILTIN(__nvvm_d2f_rp, "fd", "") 288 289BUILTIN(__nvvm_d2i_rn, "id", "") 290BUILTIN(__nvvm_d2i_rz, "id", "") 291BUILTIN(__nvvm_d2i_rm, "id", "") 292BUILTIN(__nvvm_d2i_rp, "id", "") 293 294BUILTIN(__nvvm_d2ui_rn, "Uid", "") 295BUILTIN(__nvvm_d2ui_rz, "Uid", "") 296BUILTIN(__nvvm_d2ui_rm, "Uid", "") 297BUILTIN(__nvvm_d2ui_rp, "Uid", "") 298 299BUILTIN(__nvvm_i2d_rn, "di", "") 300BUILTIN(__nvvm_i2d_rz, "di", "") 301BUILTIN(__nvvm_i2d_rm, "di", "") 302BUILTIN(__nvvm_i2d_rp, "di", "") 303 304BUILTIN(__nvvm_ui2d_rn, "dUi", "") 305BUILTIN(__nvvm_ui2d_rz, "dUi", "") 306BUILTIN(__nvvm_ui2d_rm, "dUi", "") 307BUILTIN(__nvvm_ui2d_rp, "dUi", "") 308 309BUILTIN(__nvvm_f2i_rn_ftz, "if", "") 310BUILTIN(__nvvm_f2i_rn, "if", "") 311BUILTIN(__nvvm_f2i_rz_ftz, "if", "") 312BUILTIN(__nvvm_f2i_rz, "if", "") 313BUILTIN(__nvvm_f2i_rm_ftz, "if", "") 314BUILTIN(__nvvm_f2i_rm, "if", "") 315BUILTIN(__nvvm_f2i_rp_ftz, "if", "") 316BUILTIN(__nvvm_f2i_rp, "if", "") 317 318BUILTIN(__nvvm_f2ui_rn_ftz, "Uif", "") 319BUILTIN(__nvvm_f2ui_rn, "Uif", "") 320BUILTIN(__nvvm_f2ui_rz_ftz, "Uif", "") 321BUILTIN(__nvvm_f2ui_rz, "Uif", "") 322BUILTIN(__nvvm_f2ui_rm_ftz, "Uif", "") 323BUILTIN(__nvvm_f2ui_rm, "Uif", "") 324BUILTIN(__nvvm_f2ui_rp_ftz, "Uif", "") 325BUILTIN(__nvvm_f2ui_rp, "Uif", "") 326 327BUILTIN(__nvvm_i2f_rn, "fi", "") 328BUILTIN(__nvvm_i2f_rz, "fi", "") 329BUILTIN(__nvvm_i2f_rm, "fi", "") 330BUILTIN(__nvvm_i2f_rp, "fi", "") 331 332BUILTIN(__nvvm_ui2f_rn, "fUi", "") 333BUILTIN(__nvvm_ui2f_rz, "fUi", "") 334BUILTIN(__nvvm_ui2f_rm, "fUi", "") 335BUILTIN(__nvvm_ui2f_rp, "fUi", "") 336 337BUILTIN(__nvvm_lohi_i2d, "dii", "") 338 339BUILTIN(__nvvm_d2i_lo, "id", "") 340BUILTIN(__nvvm_d2i_hi, "id", "") 341 342BUILTIN(__nvvm_f2ll_rn_ftz, "LLif", "") 343BUILTIN(__nvvm_f2ll_rn, "LLif", "") 344BUILTIN(__nvvm_f2ll_rz_ftz, "LLif", "") 345BUILTIN(__nvvm_f2ll_rz, "LLif", "") 346BUILTIN(__nvvm_f2ll_rm_ftz, "LLif", "") 347BUILTIN(__nvvm_f2ll_rm, "LLif", "") 348BUILTIN(__nvvm_f2ll_rp_ftz, "LLif", "") 349BUILTIN(__nvvm_f2ll_rp, "LLif", "") 350 351BUILTIN(__nvvm_f2ull_rn_ftz, "ULLif", "") 352BUILTIN(__nvvm_f2ull_rn, "ULLif", "") 353BUILTIN(__nvvm_f2ull_rz_ftz, "ULLif", "") 354BUILTIN(__nvvm_f2ull_rz, "ULLif", "") 355BUILTIN(__nvvm_f2ull_rm_ftz, "ULLif", "") 356BUILTIN(__nvvm_f2ull_rm, "ULLif", "") 357BUILTIN(__nvvm_f2ull_rp_ftz, "ULLif", "") 358BUILTIN(__nvvm_f2ull_rp, "ULLif", "") 359 360BUILTIN(__nvvm_d2ll_rn, "LLid", "") 361BUILTIN(__nvvm_d2ll_rz, "LLid", "") 362BUILTIN(__nvvm_d2ll_rm, "LLid", "") 363BUILTIN(__nvvm_d2ll_rp, "LLid", "") 364 365BUILTIN(__nvvm_d2ull_rn, "ULLid", "") 366BUILTIN(__nvvm_d2ull_rz, "ULLid", "") 367BUILTIN(__nvvm_d2ull_rm, "ULLid", "") 368BUILTIN(__nvvm_d2ull_rp, "ULLid", "") 369 370BUILTIN(__nvvm_ll2f_rn, "fLLi", "") 371BUILTIN(__nvvm_ll2f_rz, "fLLi", "") 372BUILTIN(__nvvm_ll2f_rm, "fLLi", "") 373BUILTIN(__nvvm_ll2f_rp, "fLLi", "") 374 375BUILTIN(__nvvm_ull2f_rn, "fULLi", "") 376BUILTIN(__nvvm_ull2f_rz, "fULLi", "") 377BUILTIN(__nvvm_ull2f_rm, "fULLi", "") 378BUILTIN(__nvvm_ull2f_rp, "fULLi", "") 379 380BUILTIN(__nvvm_ll2d_rn, "dLLi", "") 381BUILTIN(__nvvm_ll2d_rz, "dLLi", "") 382BUILTIN(__nvvm_ll2d_rm, "dLLi", "") 383BUILTIN(__nvvm_ll2d_rp, "dLLi", "") 384 385BUILTIN(__nvvm_ull2d_rn, "dULLi", "") 386BUILTIN(__nvvm_ull2d_rz, "dULLi", "") 387BUILTIN(__nvvm_ull2d_rm, "dULLi", "") 388BUILTIN(__nvvm_ull2d_rp, "dULLi", "") 389 390BUILTIN(__nvvm_f2h_rn_ftz, "Usf", "") 391BUILTIN(__nvvm_f2h_rn, "Usf", "") 392 393// Bitcast 394 395BUILTIN(__nvvm_bitcast_f2i, "if", "") 396BUILTIN(__nvvm_bitcast_i2f, "fi", "") 397 398BUILTIN(__nvvm_bitcast_ll2d, "dLLi", "") 399BUILTIN(__nvvm_bitcast_d2ll, "LLid", "") 400 401// FNS 402TARGET_BUILTIN(__nvvm_fns, "UiUiUii", "n", PTX60) 403 404// Sync 405 406BUILTIN(__syncthreads, "v", "") 407BUILTIN(__nvvm_bar0_popc, "ii", "") 408BUILTIN(__nvvm_bar0_and, "ii", "") 409BUILTIN(__nvvm_bar0_or, "ii", "") 410BUILTIN(__nvvm_bar_sync, "vi", "n") 411TARGET_BUILTIN(__nvvm_bar_warp_sync, "vUi", "n", PTX60) 412TARGET_BUILTIN(__nvvm_barrier_sync, "vUi", "n", PTX60) 413TARGET_BUILTIN(__nvvm_barrier_sync_cnt, "vUiUi", "n", PTX60) 414 415// Shuffle 416 417BUILTIN(__nvvm_shfl_down_i32, "iiii", "") 418BUILTIN(__nvvm_shfl_down_f32, "ffii", "") 419BUILTIN(__nvvm_shfl_up_i32, "iiii", "") 420BUILTIN(__nvvm_shfl_up_f32, "ffii", "") 421BUILTIN(__nvvm_shfl_bfly_i32, "iiii", "") 422BUILTIN(__nvvm_shfl_bfly_f32, "ffii", "") 423BUILTIN(__nvvm_shfl_idx_i32, "iiii", "") 424BUILTIN(__nvvm_shfl_idx_f32, "ffii", "") 425 426TARGET_BUILTIN(__nvvm_shfl_sync_down_i32, "iUiiii", "", PTX60) 427TARGET_BUILTIN(__nvvm_shfl_sync_down_f32, "fUifii", "", PTX60) 428TARGET_BUILTIN(__nvvm_shfl_sync_up_i32, "iUiiii", "", PTX60) 429TARGET_BUILTIN(__nvvm_shfl_sync_up_f32, "fUifii", "", PTX60) 430TARGET_BUILTIN(__nvvm_shfl_sync_bfly_i32, "iUiiii", "", PTX60) 431TARGET_BUILTIN(__nvvm_shfl_sync_bfly_f32, "fUifii", "", PTX60) 432TARGET_BUILTIN(__nvvm_shfl_sync_idx_i32, "iUiiii", "", PTX60) 433TARGET_BUILTIN(__nvvm_shfl_sync_idx_f32, "fUifii", "", PTX60) 434 435// Vote 436BUILTIN(__nvvm_vote_all, "bb", "") 437BUILTIN(__nvvm_vote_any, "bb", "") 438BUILTIN(__nvvm_vote_uni, "bb", "") 439BUILTIN(__nvvm_vote_ballot, "Uib", "") 440 441TARGET_BUILTIN(__nvvm_vote_all_sync, "bUib", "", PTX60) 442TARGET_BUILTIN(__nvvm_vote_any_sync, "bUib", "", PTX60) 443TARGET_BUILTIN(__nvvm_vote_uni_sync, "bUib", "", PTX60) 444TARGET_BUILTIN(__nvvm_vote_ballot_sync, "UiUib", "", PTX60) 445 446// Match 447TARGET_BUILTIN(__nvvm_match_any_sync_i32, "UiUiUi", "", PTX60) 448TARGET_BUILTIN(__nvvm_match_any_sync_i64, "WiUiWi", "", PTX60) 449// These return a pair {value, predicate}, which requires custom lowering. 450TARGET_BUILTIN(__nvvm_match_all_sync_i32p, "UiUiUii*", "", PTX60) 451TARGET_BUILTIN(__nvvm_match_all_sync_i64p, "WiUiWii*", "", PTX60) 452 453// Membar 454 455BUILTIN(__nvvm_membar_cta, "v", "") 456BUILTIN(__nvvm_membar_gl, "v", "") 457BUILTIN(__nvvm_membar_sys, "v", "") 458 459// Memcpy, Memset 460 461BUILTIN(__nvvm_memcpy, "vUc*Uc*zi","") 462BUILTIN(__nvvm_memset, "vUc*Uczi","") 463 464// Image 465 466BUILTIN(__builtin_ptx_read_image2Dfi_, "V4fiiii", "") 467BUILTIN(__builtin_ptx_read_image2Dff_, "V4fiiff", "") 468BUILTIN(__builtin_ptx_read_image2Dii_, "V4iiiii", "") 469BUILTIN(__builtin_ptx_read_image2Dif_, "V4iiiff", "") 470 471BUILTIN(__builtin_ptx_read_image3Dfi_, "V4fiiiiii", "") 472BUILTIN(__builtin_ptx_read_image3Dff_, "V4fiiffff", "") 473BUILTIN(__builtin_ptx_read_image3Dii_, "V4iiiiiii", "") 474BUILTIN(__builtin_ptx_read_image3Dif_, "V4iiiffff", "") 475 476BUILTIN(__builtin_ptx_write_image2Df_, "viiiffff", "") 477BUILTIN(__builtin_ptx_write_image2Di_, "viiiiiii", "") 478BUILTIN(__builtin_ptx_write_image2Dui_, "viiiUiUiUiUi", "") 479BUILTIN(__builtin_ptx_get_image_depthi_, "ii", "") 480BUILTIN(__builtin_ptx_get_image_heighti_, "ii", "") 481BUILTIN(__builtin_ptx_get_image_widthi_, "ii", "") 482BUILTIN(__builtin_ptx_get_image_channel_data_typei_, "ii", "") 483BUILTIN(__builtin_ptx_get_image_channel_orderi_, "ii", "") 484 485// Atomic 486// 487// We need the atom intrinsics because 488// - they are used in converging analysis 489// - they are used in address space analysis and optimization 490// So it does not hurt to expose them as builtins. 491// 492BUILTIN(__nvvm_atom_add_gen_i, "iiD*i", "n") 493TARGET_BUILTIN(__nvvm_atom_cta_add_gen_i, "iiD*i", "n", SM_60) 494TARGET_BUILTIN(__nvvm_atom_sys_add_gen_i, "iiD*i", "n", SM_60) 495BUILTIN(__nvvm_atom_add_gen_l, "LiLiD*Li", "n") 496TARGET_BUILTIN(__nvvm_atom_cta_add_gen_l, "LiLiD*Li", "n", SM_60) 497TARGET_BUILTIN(__nvvm_atom_sys_add_gen_l, "LiLiD*Li", "n", SM_60) 498BUILTIN(__nvvm_atom_add_gen_ll, "LLiLLiD*LLi", "n") 499TARGET_BUILTIN(__nvvm_atom_cta_add_gen_ll, "LLiLLiD*LLi", "n", SM_60) 500TARGET_BUILTIN(__nvvm_atom_sys_add_gen_ll, "LLiLLiD*LLi", "n", SM_60) 501BUILTIN(__nvvm_atom_add_gen_f, "ffD*f", "n") 502TARGET_BUILTIN(__nvvm_atom_cta_add_gen_f, "ffD*f", "n", SM_60) 503TARGET_BUILTIN(__nvvm_atom_sys_add_gen_f, "ffD*f", "n", SM_60) 504TARGET_BUILTIN(__nvvm_atom_add_gen_d, "ddD*d", "n", SM_60) 505TARGET_BUILTIN(__nvvm_atom_cta_add_gen_d, "ddD*d", "n", SM_60) 506TARGET_BUILTIN(__nvvm_atom_sys_add_gen_d, "ddD*d", "n", SM_60) 507 508BUILTIN(__nvvm_atom_sub_gen_i, "iiD*i", "n") 509BUILTIN(__nvvm_atom_sub_gen_l, "LiLiD*Li", "n") 510BUILTIN(__nvvm_atom_sub_gen_ll, "LLiLLiD*LLi", "n") 511 512BUILTIN(__nvvm_atom_xchg_gen_i, "iiD*i", "n") 513TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_i, "iiD*i", "n", SM_60) 514TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_i, "iiD*i", "n", SM_60) 515BUILTIN(__nvvm_atom_xchg_gen_l, "LiLiD*Li", "n") 516TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_l, "LiLiD*Li", "n", SM_60) 517TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_l, "LiLiD*Li", "n", SM_60) 518BUILTIN(__nvvm_atom_xchg_gen_ll, "LLiLLiD*LLi", "n") 519TARGET_BUILTIN(__nvvm_atom_cta_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60) 520TARGET_BUILTIN(__nvvm_atom_sys_xchg_gen_ll, "LLiLLiD*LLi", "n", SM_60) 521 522BUILTIN(__nvvm_atom_max_gen_i, "iiD*i", "n") 523TARGET_BUILTIN(__nvvm_atom_cta_max_gen_i, "iiD*i", "n", SM_60) 524TARGET_BUILTIN(__nvvm_atom_sys_max_gen_i, "iiD*i", "n", SM_60) 525BUILTIN(__nvvm_atom_max_gen_ui, "UiUiD*Ui", "n") 526TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ui, "UiUiD*Ui", "n", SM_60) 527TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ui, "UiUiD*Ui", "n", SM_60) 528BUILTIN(__nvvm_atom_max_gen_l, "LiLiD*Li", "n") 529TARGET_BUILTIN(__nvvm_atom_cta_max_gen_l, "LiLiD*Li", "n", SM_60) 530TARGET_BUILTIN(__nvvm_atom_sys_max_gen_l, "LiLiD*Li", "n", SM_60) 531BUILTIN(__nvvm_atom_max_gen_ul, "ULiULiD*ULi", "n") 532TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ul, "ULiULiD*ULi", "n", SM_60) 533TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ul, "ULiULiD*ULi", "n", SM_60) 534BUILTIN(__nvvm_atom_max_gen_ll, "LLiLLiD*LLi", "n") 535TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ll, "LLiLLiD*LLi", "n", SM_60) 536TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ll, "LLiLLiD*LLi", "n", SM_60) 537BUILTIN(__nvvm_atom_max_gen_ull, "ULLiULLiD*ULLi", "n") 538TARGET_BUILTIN(__nvvm_atom_cta_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60) 539TARGET_BUILTIN(__nvvm_atom_sys_max_gen_ull, "ULLiULLiD*ULLi", "n", SM_60) 540 541BUILTIN(__nvvm_atom_min_gen_i, "iiD*i", "n") 542TARGET_BUILTIN(__nvvm_atom_cta_min_gen_i, "iiD*i", "n", SM_60) 543TARGET_BUILTIN(__nvvm_atom_sys_min_gen_i, "iiD*i", "n", SM_60) 544BUILTIN(__nvvm_atom_min_gen_ui, "UiUiD*Ui", "n") 545TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ui, "UiUiD*Ui", "n", SM_60) 546TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ui, "UiUiD*Ui", "n", SM_60) 547BUILTIN(__nvvm_atom_min_gen_l, "LiLiD*Li", "n") 548TARGET_BUILTIN(__nvvm_atom_cta_min_gen_l, "LiLiD*Li", "n", SM_60) 549TARGET_BUILTIN(__nvvm_atom_sys_min_gen_l, "LiLiD*Li", "n", SM_60) 550BUILTIN(__nvvm_atom_min_gen_ul, "ULiULiD*ULi", "n") 551TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ul, "ULiULiD*ULi", "n", SM_60) 552TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ul, "ULiULiD*ULi", "n", SM_60) 553BUILTIN(__nvvm_atom_min_gen_ll, "LLiLLiD*LLi", "n") 554TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ll, "LLiLLiD*LLi", "n", SM_60) 555TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ll, "LLiLLiD*LLi", "n", SM_60) 556BUILTIN(__nvvm_atom_min_gen_ull, "ULLiULLiD*ULLi", "n") 557TARGET_BUILTIN(__nvvm_atom_cta_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60) 558TARGET_BUILTIN(__nvvm_atom_sys_min_gen_ull, "ULLiULLiD*ULLi", "n", SM_60) 559 560BUILTIN(__nvvm_atom_inc_gen_ui, "UiUiD*Ui", "n") 561TARGET_BUILTIN(__nvvm_atom_cta_inc_gen_ui, "UiUiD*Ui", "n", SM_60) 562TARGET_BUILTIN(__nvvm_atom_sys_inc_gen_ui, "UiUiD*Ui", "n", SM_60) 563BUILTIN(__nvvm_atom_dec_gen_ui, "UiUiD*Ui", "n") 564TARGET_BUILTIN(__nvvm_atom_cta_dec_gen_ui, "UiUiD*Ui", "n", SM_60) 565TARGET_BUILTIN(__nvvm_atom_sys_dec_gen_ui, "UiUiD*Ui", "n", SM_60) 566 567BUILTIN(__nvvm_atom_and_gen_i, "iiD*i", "n") 568TARGET_BUILTIN(__nvvm_atom_cta_and_gen_i, "iiD*i", "n", SM_60) 569TARGET_BUILTIN(__nvvm_atom_sys_and_gen_i, "iiD*i", "n", SM_60) 570BUILTIN(__nvvm_atom_and_gen_l, "LiLiD*Li", "n") 571TARGET_BUILTIN(__nvvm_atom_cta_and_gen_l, "LiLiD*Li", "n", SM_60) 572TARGET_BUILTIN(__nvvm_atom_sys_and_gen_l, "LiLiD*Li", "n", SM_60) 573BUILTIN(__nvvm_atom_and_gen_ll, "LLiLLiD*LLi", "n") 574TARGET_BUILTIN(__nvvm_atom_cta_and_gen_ll, "LLiLLiD*LLi", "n", SM_60) 575TARGET_BUILTIN(__nvvm_atom_sys_and_gen_ll, "LLiLLiD*LLi", "n", SM_60) 576 577BUILTIN(__nvvm_atom_or_gen_i, "iiD*i", "n") 578TARGET_BUILTIN(__nvvm_atom_cta_or_gen_i, "iiD*i", "n", SM_60) 579TARGET_BUILTIN(__nvvm_atom_sys_or_gen_i, "iiD*i", "n", SM_60) 580BUILTIN(__nvvm_atom_or_gen_l, "LiLiD*Li", "n") 581TARGET_BUILTIN(__nvvm_atom_cta_or_gen_l, "LiLiD*Li", "n", SM_60) 582TARGET_BUILTIN(__nvvm_atom_sys_or_gen_l, "LiLiD*Li", "n", SM_60) 583BUILTIN(__nvvm_atom_or_gen_ll, "LLiLLiD*LLi", "n") 584TARGET_BUILTIN(__nvvm_atom_cta_or_gen_ll, "LLiLLiD*LLi", "n", SM_60) 585TARGET_BUILTIN(__nvvm_atom_sys_or_gen_ll, "LLiLLiD*LLi", "n", SM_60) 586 587BUILTIN(__nvvm_atom_xor_gen_i, "iiD*i", "n") 588TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_i, "iiD*i", "n", SM_60) 589TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_i, "iiD*i", "n", SM_60) 590BUILTIN(__nvvm_atom_xor_gen_l, "LiLiD*Li", "n") 591TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_l, "LiLiD*Li", "n", SM_60) 592TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_l, "LiLiD*Li", "n", SM_60) 593BUILTIN(__nvvm_atom_xor_gen_ll, "LLiLLiD*LLi", "n") 594TARGET_BUILTIN(__nvvm_atom_cta_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60) 595TARGET_BUILTIN(__nvvm_atom_sys_xor_gen_ll, "LLiLLiD*LLi", "n", SM_60) 596 597BUILTIN(__nvvm_atom_cas_gen_i, "iiD*ii", "n") 598TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_i, "iiD*ii", "n", SM_60) 599TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_i, "iiD*ii", "n", SM_60) 600BUILTIN(__nvvm_atom_cas_gen_l, "LiLiD*LiLi", "n") 601TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_l, "LiLiD*LiLi", "n", SM_60) 602TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_l, "LiLiD*LiLi", "n", SM_60) 603BUILTIN(__nvvm_atom_cas_gen_ll, "LLiLLiD*LLiLLi", "n") 604TARGET_BUILTIN(__nvvm_atom_cta_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60) 605TARGET_BUILTIN(__nvvm_atom_sys_cas_gen_ll, "LLiLLiD*LLiLLi", "n", SM_60) 606 607// Compiler Error Warn 608BUILTIN(__nvvm_compiler_error, "vcC*4", "n") 609BUILTIN(__nvvm_compiler_warn, "vcC*4", "n") 610 611// __ldg. This is not implemented as a builtin by nvcc. 612BUILTIN(__nvvm_ldg_c, "ccC*", "") 613BUILTIN(__nvvm_ldg_s, "ssC*", "") 614BUILTIN(__nvvm_ldg_i, "iiC*", "") 615BUILTIN(__nvvm_ldg_l, "LiLiC*", "") 616BUILTIN(__nvvm_ldg_ll, "LLiLLiC*", "") 617 618BUILTIN(__nvvm_ldg_uc, "UcUcC*", "") 619BUILTIN(__nvvm_ldg_us, "UsUsC*", "") 620BUILTIN(__nvvm_ldg_ui, "UiUiC*", "") 621BUILTIN(__nvvm_ldg_ul, "ULiULiC*", "") 622BUILTIN(__nvvm_ldg_ull, "ULLiULLiC*", "") 623 624BUILTIN(__nvvm_ldg_f, "ffC*", "") 625BUILTIN(__nvvm_ldg_d, "ddC*", "") 626 627BUILTIN(__nvvm_ldg_c2, "E2cE2cC*", "") 628BUILTIN(__nvvm_ldg_c4, "E4cE4cC*", "") 629BUILTIN(__nvvm_ldg_s2, "E2sE2sC*", "") 630BUILTIN(__nvvm_ldg_s4, "E4sE4sC*", "") 631BUILTIN(__nvvm_ldg_i2, "E2iE2iC*", "") 632BUILTIN(__nvvm_ldg_i4, "E4iE4iC*", "") 633BUILTIN(__nvvm_ldg_ll2, "E2LLiE2LLiC*", "") 634 635BUILTIN(__nvvm_ldg_uc2, "E2UcE2UcC*", "") 636BUILTIN(__nvvm_ldg_uc4, "E4UcE4UcC*", "") 637BUILTIN(__nvvm_ldg_us2, "E2UsE2UsC*", "") 638BUILTIN(__nvvm_ldg_us4, "E4UsE4UsC*", "") 639BUILTIN(__nvvm_ldg_ui2, "E2UiE2UiC*", "") 640BUILTIN(__nvvm_ldg_ui4, "E4UiE4UiC*", "") 641BUILTIN(__nvvm_ldg_ull2, "E2ULLiE2ULLiC*", "") 642 643BUILTIN(__nvvm_ldg_f2, "E2fE2fC*", "") 644BUILTIN(__nvvm_ldg_f4, "E4fE4fC*", "") 645BUILTIN(__nvvm_ldg_d2, "E2dE2dC*", "") 646 647// Builtins to support WMMA instructions on sm_70 648TARGET_BUILTIN(__hmma_m16n16k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX60)) 649TARGET_BUILTIN(__hmma_m16n16k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX60)) 650TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60)) 651TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60)) 652TARGET_BUILTIN(__hmma_m16n16k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX60)) 653TARGET_BUILTIN(__hmma_m16n16k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX60)) 654 655TARGET_BUILTIN(__hmma_m32n8k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61)) 656TARGET_BUILTIN(__hmma_m32n8k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61)) 657TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61)) 658TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61)) 659TARGET_BUILTIN(__hmma_m32n8k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61)) 660TARGET_BUILTIN(__hmma_m32n8k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61)) 661 662TARGET_BUILTIN(__hmma_m8n32k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61)) 663TARGET_BUILTIN(__hmma_m8n32k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61)) 664TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61)) 665TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61)) 666TARGET_BUILTIN(__hmma_m8n32k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61)) 667TARGET_BUILTIN(__hmma_m8n32k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61)) 668 669TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX60)) 670TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX60)) 671TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX60)) 672TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX60)) 673 674TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61)) 675TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61)) 676TARGET_BUILTIN(__hmma_m32n8k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61)) 677TARGET_BUILTIN(__hmma_m32n8k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61)) 678 679TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX61)) 680TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX61)) 681TARGET_BUILTIN(__hmma_m8n32k16_mma_f32f32, "vf*iC*iC*fC*IiIi", "", AND(SM_70,PTX61)) 682TARGET_BUILTIN(__hmma_m8n32k16_mma_f16f32, "vi*iC*iC*fC*IiIi", "", AND(SM_70,PTX61)) 683 684// Builtins to support integer and sub-integer WMMA instructions on sm_72/sm_75 685TARGET_BUILTIN(__bmma_m8n8k128_ld_a_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 686TARGET_BUILTIN(__bmma_m8n8k128_ld_b_b1, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 687TARGET_BUILTIN(__bmma_m8n8k128_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 688TARGET_BUILTIN(__bmma_m8n8k128_mma_xor_popc_b1, "vi*iC*iC*iC*Ii", "", AND(SM_75,PTX63)) 689TARGET_BUILTIN(__bmma_m8n8k128_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 690TARGET_BUILTIN(__imma_m16n16k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 691TARGET_BUILTIN(__imma_m16n16k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 692TARGET_BUILTIN(__imma_m16n16k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 693TARGET_BUILTIN(__imma_m16n16k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 694TARGET_BUILTIN(__imma_m16n16k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 695TARGET_BUILTIN(__imma_m16n16k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63)) 696TARGET_BUILTIN(__imma_m16n16k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63)) 697TARGET_BUILTIN(__imma_m16n16k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 698TARGET_BUILTIN(__imma_m32n8k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 699TARGET_BUILTIN(__imma_m32n8k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 700TARGET_BUILTIN(__imma_m32n8k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 701TARGET_BUILTIN(__imma_m32n8k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 702TARGET_BUILTIN(__imma_m32n8k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 703TARGET_BUILTIN(__imma_m32n8k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63)) 704TARGET_BUILTIN(__imma_m32n8k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63)) 705TARGET_BUILTIN(__imma_m32n8k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 706TARGET_BUILTIN(__imma_m8n32k16_ld_a_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 707TARGET_BUILTIN(__imma_m8n32k16_ld_a_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 708TARGET_BUILTIN(__imma_m8n32k16_ld_b_s8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 709TARGET_BUILTIN(__imma_m8n32k16_ld_b_u8, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 710TARGET_BUILTIN(__imma_m8n32k16_ld_c, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 711TARGET_BUILTIN(__imma_m8n32k16_mma_s8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63)) 712TARGET_BUILTIN(__imma_m8n32k16_mma_u8, "vi*iC*iC*iC*IiIi", "", AND(SM_72,PTX63)) 713TARGET_BUILTIN(__imma_m8n32k16_st_c_i32, "vi*iC*UiIi", "", AND(SM_72,PTX63)) 714TARGET_BUILTIN(__imma_m8n8k32_ld_a_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 715TARGET_BUILTIN(__imma_m8n8k32_ld_a_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 716TARGET_BUILTIN(__imma_m8n8k32_ld_b_s4, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 717TARGET_BUILTIN(__imma_m8n8k32_ld_b_u4, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 718TARGET_BUILTIN(__imma_m8n8k32_ld_c, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 719TARGET_BUILTIN(__imma_m8n8k32_mma_s4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63)) 720TARGET_BUILTIN(__imma_m8n8k32_mma_u4, "vi*iC*iC*iC*IiIi", "", AND(SM_75,PTX63)) 721TARGET_BUILTIN(__imma_m8n8k32_st_c_i32, "vi*iC*UiIi", "", AND(SM_75,PTX63)) 722 723#undef BUILTIN 724#undef TARGET_BUILTIN 725#pragma pop_macro("AND") 726#pragma pop_macro("SM_60") 727#pragma pop_macro("SM_70") 728#pragma pop_macro("SM_72") 729#pragma pop_macro("SM_75") 730#pragma pop_macro("SM_80") 731#pragma pop_macro("PTX60") 732#pragma pop_macro("PTX61") 733#pragma pop_macro("PTX63") 734#pragma pop_macro("PTX64") 735#pragma pop_macro("PTX65") 736#pragma pop_macro("PTX70") 737