1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines all of the AARCH64-specific intrinsics. 10// 11//===----------------------------------------------------------------------===// 12 13let TargetPrefix = "aarch64" in { 14 15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty], 16 [IntrNoFree, IntrWillReturn]>; 17def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty], 18 [IntrNoFree, IntrWillReturn]>; 19def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty], 20 [IntrNoFree, IntrWillReturn]>; 21def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty], 22 [IntrNoFree, IntrWillReturn]>; 23 24def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty], 25 [IntrNoFree, IntrWillReturn]>; 26def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty], 27 [IntrNoFree, IntrWillReturn]>; 28def int_aarch64_stxp : Intrinsic<[llvm_i32_ty], 29 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], 30 [IntrNoFree, IntrWillReturn]>; 31def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty], 32 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], 33 [IntrNoFree, IntrWillReturn]>; 34 35def int_aarch64_clrex : Intrinsic<[]>; 36 37def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 38 LLVMMatchType<0>], [IntrNoMem]>; 39def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 40 LLVMMatchType<0>], [IntrNoMem]>; 41 42def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; 43 44def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 45def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; 46 47def int_aarch64_frint32z 48 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 49 [ IntrNoMem ]>; 50def int_aarch64_frint64z 51 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 52 [ IntrNoMem ]>; 53def int_aarch64_frint32x 54 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 55 [ IntrNoMem ]>; 56def int_aarch64_frint64x 57 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 58 [ IntrNoMem ]>; 59 60//===----------------------------------------------------------------------===// 61// HINT 62 63def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>; 64 65def int_aarch64_break : Intrinsic<[], [llvm_i32_ty], 66 [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>; 67 68//===----------------------------------------------------------------------===// 69// Data Barrier Instructions 70 71def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, 72 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 73def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, 74 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 75def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, 76 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 77 78// A space-consuming intrinsic primarily for testing block and jump table 79// placements. The first argument is the number of bytes this "instruction" 80// takes up, the second and return value are essentially chains, used to force 81// ordering during ISel. 82def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>; 83 84} 85 86//===----------------------------------------------------------------------===// 87// Advanced SIMD (NEON) 88 89let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 90 class AdvSIMD_2Scalar_Float_Intrinsic 91 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 92 [IntrNoMem]>; 93 94 class AdvSIMD_FPToIntRounding_Intrinsic 95 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>; 96 97 class AdvSIMD_1IntArg_Intrinsic 98 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>; 99 class AdvSIMD_1FloatArg_Intrinsic 100 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; 101 class AdvSIMD_1VectorArg_Intrinsic 102 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>; 103 class AdvSIMD_1VectorArg_Expand_Intrinsic 104 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>; 105 class AdvSIMD_1VectorArg_Long_Intrinsic 106 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>; 107 class AdvSIMD_1IntArg_Narrow_Intrinsic 108 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>; 109 class AdvSIMD_1VectorArg_Narrow_Intrinsic 110 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>; 111 class AdvSIMD_1VectorArg_Int_Across_Intrinsic 112 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>; 113 class AdvSIMD_1VectorArg_Float_Across_Intrinsic 114 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>; 115 116 class AdvSIMD_2IntArg_Intrinsic 117 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 118 [IntrNoMem]>; 119 class AdvSIMD_2FloatArg_Intrinsic 120 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 121 [IntrNoMem]>; 122 class AdvSIMD_2VectorArg_Intrinsic 123 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 124 [IntrNoMem]>; 125 class AdvSIMD_2VectorArg_Compare_Intrinsic 126 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>], 127 [IntrNoMem]>; 128 class AdvSIMD_2Arg_FloatCompare_Intrinsic 129 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>], 130 [IntrNoMem]>; 131 class AdvSIMD_2VectorArg_Long_Intrinsic 132 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 133 [LLVMTruncatedType<0>, LLVMTruncatedType<0>], 134 [IntrNoMem]>; 135 class AdvSIMD_2VectorArg_Wide_Intrinsic 136 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 137 [LLVMMatchType<0>, LLVMTruncatedType<0>], 138 [IntrNoMem]>; 139 class AdvSIMD_2VectorArg_Narrow_Intrinsic 140 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 141 [LLVMExtendedType<0>, LLVMExtendedType<0>], 142 [IntrNoMem]>; 143 class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic 144 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 145 [LLVMExtendedType<0>, llvm_i32_ty], 146 [IntrNoMem]>; 147 class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic 148 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 149 [llvm_anyvector_ty], 150 [IntrNoMem]>; 151 class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic 152 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 153 [LLVMTruncatedType<0>], 154 [IntrNoMem]>; 155 class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic 156 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 157 [LLVMTruncatedType<0>, llvm_i32_ty], 158 [IntrNoMem]>; 159 class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic 160 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 161 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty], 162 [IntrNoMem]>; 163 class AdvSIMD_2VectorArg_Lane_Intrinsic 164 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 165 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty], 166 [IntrNoMem]>; 167 168 class AdvSIMD_3IntArg_Intrinsic 169 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 170 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 171 [IntrNoMem]>; 172 class AdvSIMD_3VectorArg_Intrinsic 173 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 174 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 175 [IntrNoMem]>; 176 class AdvSIMD_3VectorArg_Scalar_Intrinsic 177 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 178 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], 179 [IntrNoMem]>; 180 class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic 181 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 182 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, 183 LLVMMatchType<1>], [IntrNoMem]>; 184 class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic 185 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 186 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty], 187 [IntrNoMem]>; 188 class AdvSIMD_CvtFxToFP_Intrinsic 189 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], 190 [IntrNoMem]>; 191 class AdvSIMD_CvtFPToFx_Intrinsic 192 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty], 193 [IntrNoMem]>; 194 195 class AdvSIMD_1Arg_Intrinsic 196 : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>; 197 198 class AdvSIMD_Dot_Intrinsic 199 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 200 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 201 [IntrNoMem]>; 202 203 class AdvSIMD_FP16FML_Intrinsic 204 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 205 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 206 [IntrNoMem]>; 207 208 class AdvSIMD_MatMul_Intrinsic 209 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 210 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 211 [IntrNoMem]>; 212 213 class AdvSIMD_FML_Intrinsic 214 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 215 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 216 [IntrNoMem]>; 217 218 class AdvSIMD_BF16FML_Intrinsic 219 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], 220 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 221 [IntrNoMem]>; 222} 223 224// Arithmetic ops 225 226let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { 227 // Vector Add Across Lanes 228 def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 229 def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 230 def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 231 232 // Vector Long Add Across Lanes 233 def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 234 def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 235 236 // Vector Halving Add 237 def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic; 238 def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic; 239 240 // Vector Rounding Halving Add 241 def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic; 242 def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic; 243 244 // Vector Saturating Add 245 def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic; 246 def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic; 247 def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic; 248 def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic; 249 250 // Vector Add High-Half 251 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 252 // header is no longer supported. 253 def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 254 255 // Vector Rounding Add High-Half 256 def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 257 258 // Vector Saturating Doubling Multiply High 259 def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic; 260 def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 261 def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 262 263 // Vector Saturating Rounding Doubling Multiply High 264 def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic; 265 def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 266 def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 267 268 def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic; 269 def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic; 270 271 // Vector Polynominal Multiply 272 def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic; 273 274 // Vector Long Multiply 275 def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic; 276 def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic; 277 def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic; 278 279 // 64-bit polynomial multiply really returns an i128, which is not legal. Fake 280 // it with a v16i8. 281 def int_aarch64_neon_pmull64 : 282 DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 283 284 // Vector Extending Multiply 285 def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic { 286 let IntrProperties = [IntrNoMem, Commutative]; 287 } 288 289 // Vector Saturating Doubling Long Multiply 290 def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic; 291 def int_aarch64_neon_sqdmulls_scalar 292 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; 293 294 // Vector Halving Subtract 295 def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic; 296 def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic; 297 298 // Vector Saturating Subtract 299 def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic; 300 def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic; 301 302 // Vector Subtract High-Half 303 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 304 // header is no longer supported. 305 def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 306 307 // Vector Rounding Subtract High-Half 308 def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 309 310 // Vector Compare Absolute Greater-than-or-equal 311 def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic; 312 313 // Vector Compare Absolute Greater-than 314 def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic; 315 316 // Vector Absolute Difference 317 def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic; 318 def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic; 319 def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic; 320 321 // Scalar Absolute Difference 322 def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic; 323 324 // Vector Max 325 def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic; 326 def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic; 327 def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic; 328 def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic; 329 330 // Vector Max Across Lanes 331 def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 332 def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 333 def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 334 def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 335 336 // Vector Min 337 def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic; 338 def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic; 339 def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic; 340 def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic; 341 342 // Vector Min/Max Number 343 def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic; 344 def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic; 345 346 // Vector Min Across Lanes 347 def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 348 def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 349 def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 350 def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 351 352 // Pairwise Add 353 def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic; 354 def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic; 355 356 // Long Pairwise Add 357 // FIXME: In theory, we shouldn't need intrinsics for saddlp or 358 // uaddlp, but tblgen's type inference currently can't handle the 359 // pattern fragments this ends up generating. 360 def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 361 def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 362 363 // Folding Maximum 364 def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic; 365 def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic; 366 def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic; 367 368 // Folding Minimum 369 def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic; 370 def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic; 371 def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic; 372 373 // Reciprocal Estimate/Step 374 def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic; 375 def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic; 376 377 // Reciprocal Exponent 378 def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic; 379 380 // Vector Saturating Shift Left 381 def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic; 382 def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic; 383 384 // Vector Rounding Shift Left 385 def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic; 386 def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic; 387 388 // Vector Saturating Rounding Shift Left 389 def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic; 390 def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic; 391 392 // Vector Signed->Unsigned Shift Left by Constant 393 def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic; 394 395 // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant 396 def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 397 398 // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const 399 def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 400 401 // Vector Narrowing Shift Right by Constant 402 def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 403 def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 404 405 // Vector Rounding Narrowing Shift Right by Constant 406 def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 407 408 // Vector Rounding Narrowing Saturating Shift Right by Constant 409 def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 410 def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 411 412 // Vector Shift Left 413 def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic; 414 def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic; 415 416 // Vector Widening Shift Left by Constant 417 def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic; 418 def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 419 def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 420 421 // Vector Shift Right by Constant and Insert 422 def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic; 423 424 // Vector Shift Left by Constant and Insert 425 def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic; 426 427 // Vector Saturating Narrow 428 def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic; 429 def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic; 430 def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 431 def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 432 433 // Vector Saturating Extract and Unsigned Narrow 434 def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic; 435 def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic; 436 437 // Vector Absolute Value 438 def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic; 439 440 // Vector Saturating Absolute Value 441 def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic; 442 443 // Vector Saturating Negation 444 def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic; 445 446 // Vector Count Leading Sign Bits 447 def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic; 448 449 // Vector Reciprocal Estimate 450 def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic; 451 def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic; 452 453 // Vector Square Root Estimate 454 def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic; 455 def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic; 456 457 // Vector Conversions Between Half-Precision and Single-Precision. 458 def int_aarch64_neon_vcvtfp2hf 459 : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 460 def int_aarch64_neon_vcvthf2fp 461 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>; 462 463 // Vector Conversions Between Floating-point and Fixed-point. 464 def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic; 465 def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic; 466 def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic; 467 def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic; 468 469 // Vector FP->Int Conversions 470 def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic; 471 def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic; 472 def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic; 473 def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic; 474 def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic; 475 def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic; 476 def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic; 477 def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic; 478 def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic; 479 def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic; 480 481 // v8.5-A Vector FP Rounding 482 def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic; 483 def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic; 484 def int_aarch64_neon_frint64x : AdvSIMD_1FloatArg_Intrinsic; 485 def int_aarch64_neon_frint64z : AdvSIMD_1FloatArg_Intrinsic; 486 487 // Scalar FP->Int conversions 488 489 // Vector FP Inexact Narrowing 490 def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic; 491 492 // Scalar FP Inexact Narrowing 493 def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], 494 [IntrNoMem]>; 495 496 // v8.2-A Dot Product 497 def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic; 498 def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic; 499 500 // v8.6-A Matrix Multiply Intrinsics 501 def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic; 502 def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic; 503 def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic; 504 def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic; 505 def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic; 506 def int_aarch64_neon_bfmmla 507 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], 508 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 509 [IntrNoMem]>; 510 def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic; 511 def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic; 512 513 514 // v8.6-A Bfloat Intrinsics 515 def int_aarch64_neon_bfcvt 516 : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>; 517 def int_aarch64_neon_bfcvtn 518 : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 519 def int_aarch64_neon_bfcvtn2 520 : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], 521 [llvm_v8bf16_ty, llvm_v4f32_ty], 522 [IntrNoMem]>; 523 524 // v8.2-A FP16 Fused Multiply-Add Long 525 def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic; 526 def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic; 527 def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic; 528 def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic; 529 530 // v8.3-A Floating-point complex add 531 def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic; 532 def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic; 533 534 def int_aarch64_neon_vcmla_rot0 : AdvSIMD_3VectorArg_Intrinsic; 535 def int_aarch64_neon_vcmla_rot90 : AdvSIMD_3VectorArg_Intrinsic; 536 def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic; 537 def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic; 538} 539 540let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 541 class AdvSIMD_2Vector2Index_Intrinsic 542 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 543 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty], 544 [IntrNoMem]>; 545} 546 547// Vector element to element moves 548def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic; 549 550let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 551 class AdvSIMD_1Vec_Load_Intrinsic 552 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>], 553 [IntrReadMem, IntrArgMemOnly]>; 554 class AdvSIMD_1Vec_Store_Lane_Intrinsic 555 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], 556 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 557 558 class AdvSIMD_2Vec_Load_Intrinsic 559 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], 560 [LLVMAnyPointerType<LLVMMatchType<0>>], 561 [IntrReadMem, IntrArgMemOnly]>; 562 class AdvSIMD_2Vec_Load_Lane_Intrinsic 563 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>], 564 [LLVMMatchType<0>, llvm_anyvector_ty, 565 llvm_i64_ty, llvm_anyptr_ty], 566 [IntrReadMem, IntrArgMemOnly]>; 567 class AdvSIMD_2Vec_Store_Intrinsic 568 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 569 LLVMAnyPointerType<LLVMMatchType<0>>], 570 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 571 class AdvSIMD_2Vec_Store_Lane_Intrinsic 572 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 573 llvm_i64_ty, llvm_anyptr_ty], 574 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 575 576 class AdvSIMD_3Vec_Load_Intrinsic 577 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty], 578 [LLVMAnyPointerType<LLVMMatchType<0>>], 579 [IntrReadMem, IntrArgMemOnly]>; 580 class AdvSIMD_3Vec_Load_Lane_Intrinsic 581 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 582 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, 583 llvm_i64_ty, llvm_anyptr_ty], 584 [IntrReadMem, IntrArgMemOnly]>; 585 class AdvSIMD_3Vec_Store_Intrinsic 586 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 587 LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>], 588 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 589 class AdvSIMD_3Vec_Store_Lane_Intrinsic 590 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, 591 LLVMMatchType<0>, LLVMMatchType<0>, 592 llvm_i64_ty, llvm_anyptr_ty], 593 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 594 595 class AdvSIMD_4Vec_Load_Intrinsic 596 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 597 LLVMMatchType<0>, llvm_anyvector_ty], 598 [LLVMAnyPointerType<LLVMMatchType<0>>], 599 [IntrReadMem, IntrArgMemOnly]>; 600 class AdvSIMD_4Vec_Load_Lane_Intrinsic 601 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 602 LLVMMatchType<0>, LLVMMatchType<0>], 603 [LLVMMatchType<0>, LLVMMatchType<0>, 604 LLVMMatchType<0>, llvm_anyvector_ty, 605 llvm_i64_ty, llvm_anyptr_ty], 606 [IntrReadMem, IntrArgMemOnly]>; 607 class AdvSIMD_4Vec_Store_Intrinsic 608 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 609 LLVMMatchType<0>, LLVMMatchType<0>, 610 LLVMAnyPointerType<LLVMMatchType<0>>], 611 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 612 class AdvSIMD_4Vec_Store_Lane_Intrinsic 613 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 614 LLVMMatchType<0>, LLVMMatchType<0>, 615 llvm_i64_ty, llvm_anyptr_ty], 616 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 617} 618 619// Memory ops 620 621def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic; 622def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic; 623def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic; 624 625def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic; 626def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic; 627def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic; 628 629def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic; 630def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic; 631def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic; 632 633def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic; 634def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic; 635def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic; 636 637def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic; 638def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic; 639def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic; 640 641def int_aarch64_neon_st2 : AdvSIMD_2Vec_Store_Intrinsic; 642def int_aarch64_neon_st3 : AdvSIMD_3Vec_Store_Intrinsic; 643def int_aarch64_neon_st4 : AdvSIMD_4Vec_Store_Intrinsic; 644 645def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic; 646def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic; 647def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic; 648 649let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 650 class AdvSIMD_Tbl1_Intrinsic 651 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>], 652 [IntrNoMem]>; 653 class AdvSIMD_Tbl2_Intrinsic 654 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 655 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>; 656 class AdvSIMD_Tbl3_Intrinsic 657 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 658 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 659 LLVMMatchType<0>], 660 [IntrNoMem]>; 661 class AdvSIMD_Tbl4_Intrinsic 662 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 663 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 664 LLVMMatchType<0>], 665 [IntrNoMem]>; 666 667 class AdvSIMD_Tbx1_Intrinsic 668 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 669 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>], 670 [IntrNoMem]>; 671 class AdvSIMD_Tbx2_Intrinsic 672 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 673 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 674 LLVMMatchType<0>], 675 [IntrNoMem]>; 676 class AdvSIMD_Tbx3_Intrinsic 677 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 678 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 679 llvm_v16i8_ty, LLVMMatchType<0>], 680 [IntrNoMem]>; 681 class AdvSIMD_Tbx4_Intrinsic 682 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 683 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 684 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], 685 [IntrNoMem]>; 686} 687def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic; 688def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic; 689def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic; 690def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic; 691 692def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic; 693def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic; 694def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic; 695def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic; 696 697let TargetPrefix = "aarch64" in { 698 class FPCR_Get_Intrinsic 699 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>; 700 class FPCR_Set_Intrinsic 701 : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; 702 class RNDR_Intrinsic 703 : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>; 704} 705 706// FPCR 707def int_aarch64_get_fpcr : FPCR_Get_Intrinsic; 708def int_aarch64_set_fpcr : FPCR_Set_Intrinsic; 709 710// Armv8.5-A Random number generation intrinsics 711def int_aarch64_rndr : RNDR_Intrinsic; 712def int_aarch64_rndrrs : RNDR_Intrinsic; 713 714let TargetPrefix = "aarch64" in { 715 class Crypto_AES_DataKey_Intrinsic 716 : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; 717 718 class Crypto_AES_Data_Intrinsic 719 : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; 720 721 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 722 // (v4i32). 723 class Crypto_SHA_5Hash4Schedule_Intrinsic 724 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], 725 [IntrNoMem]>; 726 727 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 728 // (v4i32). 729 class Crypto_SHA_1Hash_Intrinsic 730 : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 731 732 // SHA intrinsic taking 8 words of the schedule 733 class Crypto_SHA_8Schedule_Intrinsic 734 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 735 736 // SHA intrinsic taking 12 words of the schedule 737 class Crypto_SHA_12Schedule_Intrinsic 738 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 739 [IntrNoMem]>; 740 741 // SHA intrinsic taking 8 words of the hash and 4 of the schedule. 742 class Crypto_SHA_8Hash4Schedule_Intrinsic 743 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 744 [IntrNoMem]>; 745 746 // SHA512 intrinsic taking 2 arguments 747 class Crypto_SHA512_2Arg_Intrinsic 748 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; 749 750 // SHA512 intrinsic taking 3 Arguments 751 class Crypto_SHA512_3Arg_Intrinsic 752 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], 753 [IntrNoMem]>; 754 755 // SHA3 Intrinsics taking 3 arguments 756 class Crypto_SHA3_3Arg_Intrinsic 757 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 758 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 759 [IntrNoMem]>; 760 761 // SHA3 Intrinsic taking 2 arguments 762 class Crypto_SHA3_2Arg_Intrinsic 763 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], 764 [IntrNoMem]>; 765 766 // SHA3 Intrinsic taking 3 Arguments 1 immediate 767 class Crypto_SHA3_2ArgImm_Intrinsic 768 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i64_ty], 769 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 770 771 class Crypto_SM3_3Vector_Intrinsic 772 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 773 [IntrNoMem]>; 774 775 class Crypto_SM3_3VectorIndexed_Intrinsic 776 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i64_ty], 777 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 778 779 class Crypto_SM4_2Vector_Intrinsic 780 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 781} 782 783// AES 784def int_aarch64_crypto_aese : Crypto_AES_DataKey_Intrinsic; 785def int_aarch64_crypto_aesd : Crypto_AES_DataKey_Intrinsic; 786def int_aarch64_crypto_aesmc : Crypto_AES_Data_Intrinsic; 787def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic; 788 789// SHA1 790def int_aarch64_crypto_sha1c : Crypto_SHA_5Hash4Schedule_Intrinsic; 791def int_aarch64_crypto_sha1p : Crypto_SHA_5Hash4Schedule_Intrinsic; 792def int_aarch64_crypto_sha1m : Crypto_SHA_5Hash4Schedule_Intrinsic; 793def int_aarch64_crypto_sha1h : Crypto_SHA_1Hash_Intrinsic; 794 795def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic; 796def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic; 797 798// SHA256 799def int_aarch64_crypto_sha256h : Crypto_SHA_8Hash4Schedule_Intrinsic; 800def int_aarch64_crypto_sha256h2 : Crypto_SHA_8Hash4Schedule_Intrinsic; 801def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic; 802def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic; 803 804//SHA3 805def int_aarch64_crypto_eor3s : Crypto_SHA3_3Arg_Intrinsic; 806def int_aarch64_crypto_eor3u : Crypto_SHA3_3Arg_Intrinsic; 807def int_aarch64_crypto_bcaxs : Crypto_SHA3_3Arg_Intrinsic; 808def int_aarch64_crypto_bcaxu : Crypto_SHA3_3Arg_Intrinsic; 809def int_aarch64_crypto_rax1 : Crypto_SHA3_2Arg_Intrinsic; 810def int_aarch64_crypto_xar : Crypto_SHA3_2ArgImm_Intrinsic; 811 812// SHA512 813def int_aarch64_crypto_sha512h : Crypto_SHA512_3Arg_Intrinsic; 814def int_aarch64_crypto_sha512h2 : Crypto_SHA512_3Arg_Intrinsic; 815def int_aarch64_crypto_sha512su0 : Crypto_SHA512_2Arg_Intrinsic; 816def int_aarch64_crypto_sha512su1 : Crypto_SHA512_3Arg_Intrinsic; 817 818//SM3 & SM4 819def int_aarch64_crypto_sm3partw1 : Crypto_SM3_3Vector_Intrinsic; 820def int_aarch64_crypto_sm3partw2 : Crypto_SM3_3Vector_Intrinsic; 821def int_aarch64_crypto_sm3ss1 : Crypto_SM3_3Vector_Intrinsic; 822def int_aarch64_crypto_sm3tt1a : Crypto_SM3_3VectorIndexed_Intrinsic; 823def int_aarch64_crypto_sm3tt1b : Crypto_SM3_3VectorIndexed_Intrinsic; 824def int_aarch64_crypto_sm3tt2a : Crypto_SM3_3VectorIndexed_Intrinsic; 825def int_aarch64_crypto_sm3tt2b : Crypto_SM3_3VectorIndexed_Intrinsic; 826def int_aarch64_crypto_sm4e : Crypto_SM4_2Vector_Intrinsic; 827def int_aarch64_crypto_sm4ekey : Crypto_SM4_2Vector_Intrinsic; 828 829//===----------------------------------------------------------------------===// 830// CRC32 831 832let TargetPrefix = "aarch64" in { 833 834def int_aarch64_crc32b : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 835 [IntrNoMem]>; 836def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 837 [IntrNoMem]>; 838def int_aarch64_crc32h : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 839 [IntrNoMem]>; 840def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 841 [IntrNoMem]>; 842def int_aarch64_crc32w : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 843 [IntrNoMem]>; 844def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 845 [IntrNoMem]>; 846def int_aarch64_crc32x : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 847 [IntrNoMem]>; 848def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 849 [IntrNoMem]>; 850} 851 852//===----------------------------------------------------------------------===// 853// Memory Tagging Extensions (MTE) Intrinsics 854let TargetPrefix = "aarch64" in { 855def int_aarch64_irg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 856 [IntrNoMem, IntrHasSideEffects]>; 857def int_aarch64_addg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 858 [IntrNoMem]>; 859def int_aarch64_gmi : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], 860 [IntrNoMem]>; 861def int_aarch64_ldg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], 862 [IntrReadMem]>; 863def int_aarch64_stg : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], 864 [IntrWriteMem]>; 865def int_aarch64_subp : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], 866 [IntrNoMem]>; 867 868// The following are codegen-only intrinsics for stack instrumentation. 869 870// Generate a randomly tagged stack base pointer. 871def int_aarch64_irg_sp : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty], 872 [IntrNoMem, IntrHasSideEffects]>; 873 874// Transfer pointer tag with offset. 875// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where 876// * address is the address in ptr0 877// * tag is a function of (tag in baseptr, tag_offset). 878// ** Beware, this is not the same function as implemented by the ADDG instruction! 879// Backend optimizations may change tag_offset; the only guarantee is that calls 880// to tagp with the same pair of (baseptr, tag_offset) will produce pointers 881// with the same tag value, assuming the set of excluded tags has not changed. 882// Address bits in baseptr and tag bits in ptr0 are ignored. 883// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as 884// ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset 885// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp. 886def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty], 887 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 888 889// Update allocation tags for the memory range to match the tag in the pointer argument. 890def int_aarch64_settag : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 891 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 892 893// Update allocation tags for the memory range to match the tag in the pointer argument, 894// and set memory contents to zero. 895def int_aarch64_settag_zero : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 896 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 897 898// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values. 899def int_aarch64_stgp : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], 900 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 901} 902 903//===----------------------------------------------------------------------===// 904// Memory Operations (MOPS) Intrinsics 905let TargetPrefix = "aarch64" in { 906 // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64 907 def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty], 908 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 909} 910 911// Transactional Memory Extension (TME) Intrinsics 912let TargetPrefix = "aarch64" in { 913def int_aarch64_tstart : ClangBuiltin<"__builtin_arm_tstart">, 914 Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>; 915 916def int_aarch64_tcommit : ClangBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>; 917 918def int_aarch64_tcancel : ClangBuiltin<"__builtin_arm_tcancel">, 919 Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>; 920 921def int_aarch64_ttest : ClangBuiltin<"__builtin_arm_ttest">, 922 Intrinsic<[llvm_i64_ty], [], 923 [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; 924 925// Armv8.7-A load/store 64-byte intrinsics 926defvar data512 = !listsplat(llvm_i64_ty, 8); 927def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>; 928def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>; 929def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; 930def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; 931 932} 933 934def llvm_nxv1i1_ty : LLVMType<nxv1i1>; 935def llvm_nxv2i1_ty : LLVMType<nxv2i1>; 936def llvm_nxv4i1_ty : LLVMType<nxv4i1>; 937def llvm_nxv8i1_ty : LLVMType<nxv8i1>; 938def llvm_nxv16i1_ty : LLVMType<nxv16i1>; 939def llvm_nxv16i8_ty : LLVMType<nxv16i8>; 940def llvm_nxv4i32_ty : LLVMType<nxv4i32>; 941def llvm_nxv2i64_ty : LLVMType<nxv2i64>; 942def llvm_nxv8f16_ty : LLVMType<nxv8f16>; 943def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>; 944def llvm_nxv4f32_ty : LLVMType<nxv4f32>; 945def llvm_nxv2f64_ty : LLVMType<nxv2f64>; 946 947let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 948 949 class AdvSIMD_SVE_Create_2Vector_Tuple 950 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 951 [llvm_anyvector_ty, LLVMMatchType<1>], 952 [IntrReadMem]>; 953 954 class AdvSIMD_SVE_Create_3Vector_Tuple 955 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 956 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>], 957 [IntrReadMem]>; 958 959 class AdvSIMD_SVE_Create_4Vector_Tuple 960 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 961 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, 962 LLVMMatchType<1>], 963 [IntrReadMem]>; 964 965 class AdvSIMD_SVE_Set_Vector_Tuple 966 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 967 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty], 968 [IntrReadMem, ImmArg<ArgIndex<1>>]>; 969 970 class AdvSIMD_SVE_Get_Vector_Tuple 971 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty], 972 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>; 973 974 class AdvSIMD_ManyVec_PredLoad_Intrinsic 975 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>], 976 [IntrReadMem, IntrArgMemOnly]>; 977 978 class AdvSIMD_1Vec_PredLoad_Intrinsic 979 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 980 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 981 LLVMPointerToElt<0>], 982 [IntrReadMem, IntrArgMemOnly]>; 983 984 class AdvSIMD_2Vec_PredLoad_Intrinsic 985 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 986 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 987 LLVMPointerToElt<0>], 988 [IntrReadMem, IntrArgMemOnly]>; 989 990 class AdvSIMD_3Vec_PredLoad_Intrinsic 991 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], 992 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 993 LLVMPointerToElt<0>], 994 [IntrReadMem, IntrArgMemOnly]>; 995 996 class AdvSIMD_4Vec_PredLoad_Intrinsic 997 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 998 LLVMMatchType<0>], 999 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1000 LLVMPointerToElt<0>], 1001 [IntrReadMem, IntrArgMemOnly]>; 1002 1003 class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic 1004 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1005 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1006 LLVMPointerToElt<0>], 1007 [IntrInaccessibleMemOrArgMemOnly]>; 1008 1009 class AdvSIMD_1Vec_PredStore_Intrinsic 1010 : DefaultAttrsIntrinsic<[], 1011 [llvm_anyvector_ty, 1012 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1013 LLVMPointerToElt<0>], 1014 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 1015 1016 class AdvSIMD_2Vec_PredStore_Intrinsic 1017 : DefaultAttrsIntrinsic<[], 1018 [llvm_anyvector_ty, LLVMMatchType<0>, 1019 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 1020 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 1021 1022 class AdvSIMD_3Vec_PredStore_Intrinsic 1023 : DefaultAttrsIntrinsic<[], 1024 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 1025 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 1026 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 1027 1028 class AdvSIMD_4Vec_PredStore_Intrinsic 1029 : DefaultAttrsIntrinsic<[], 1030 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 1031 LLVMMatchType<0>, 1032 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 1033 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 1034 1035 class AdvSIMD_SVE_Index_Intrinsic 1036 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1037 [LLVMVectorElementType<0>, 1038 LLVMVectorElementType<0>], 1039 [IntrNoMem]>; 1040 1041 class AdvSIMD_Merged1VectorArg_Intrinsic 1042 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1043 [LLVMMatchType<0>, 1044 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1045 LLVMMatchType<0>], 1046 [IntrNoMem]>; 1047 1048 class AdvSIMD_2VectorArgIndexed_Intrinsic 1049 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1050 [LLVMMatchType<0>, 1051 LLVMMatchType<0>, 1052 llvm_i32_ty], 1053 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1054 1055 class AdvSIMD_3VectorArgIndexed_Intrinsic 1056 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1057 [LLVMMatchType<0>, 1058 LLVMMatchType<0>, 1059 LLVMMatchType<0>, 1060 llvm_i32_ty], 1061 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1062 1063 class AdvSIMD_Pred1VectorArg_Intrinsic 1064 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1065 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1066 LLVMMatchType<0>], 1067 [IntrNoMem]>; 1068 1069 class AdvSIMD_Pred2VectorArg_Intrinsic 1070 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1071 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1072 LLVMMatchType<0>, 1073 LLVMMatchType<0>], 1074 [IntrNoMem]>; 1075 1076 class AdvSIMD_Pred3VectorArg_Intrinsic 1077 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1078 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1079 LLVMMatchType<0>, 1080 LLVMMatchType<0>, 1081 LLVMMatchType<0>], 1082 [IntrNoMem]>; 1083 1084 class AdvSIMD_SVE_Compare_Intrinsic 1085 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1086 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1087 llvm_anyvector_ty, 1088 LLVMMatchType<0>], 1089 [IntrNoMem]>; 1090 1091 class AdvSIMD_SVE_CompareWide_Intrinsic 1092 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1093 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1094 llvm_anyvector_ty, 1095 llvm_nxv2i64_ty], 1096 [IntrNoMem]>; 1097 1098 class AdvSIMD_SVE_Saturating_Intrinsic 1099 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1100 [LLVMMatchType<0>, 1101 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1102 [IntrNoMem]>; 1103 1104 class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic 1105 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1106 [LLVMMatchType<0>, 1107 llvm_i32_ty, 1108 llvm_i32_ty], 1109 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 1110 1111 class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T> 1112 : DefaultAttrsIntrinsic<[T], 1113 [T, llvm_anyvector_ty], 1114 [IntrNoMem]>; 1115 1116 class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T> 1117 : DefaultAttrsIntrinsic<[T], 1118 [T, llvm_i32_ty, llvm_i32_ty], 1119 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 1120 1121 class AdvSIMD_SVE_CNT_Intrinsic 1122 : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>], 1123 [LLVMVectorOfBitcastsToInt<0>, 1124 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1125 llvm_anyvector_ty], 1126 [IntrNoMem]>; 1127 1128 class AdvSIMD_SVE_ReduceWithInit_Intrinsic 1129 : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], 1130 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1131 LLVMVectorElementType<0>, 1132 llvm_anyvector_ty], 1133 [IntrNoMem]>; 1134 1135 class AdvSIMD_SVE_ShiftByImm_Intrinsic 1136 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1137 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1138 LLVMMatchType<0>, 1139 llvm_i32_ty], 1140 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1141 1142 class AdvSIMD_SVE_ShiftWide_Intrinsic 1143 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1144 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1145 LLVMMatchType<0>, 1146 llvm_nxv2i64_ty], 1147 [IntrNoMem]>; 1148 1149 class AdvSIMD_SVE_Unpack_Intrinsic 1150 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1151 [LLVMSubdivide2VectorType<0>], 1152 [IntrNoMem]>; 1153 1154 class AdvSIMD_SVE_CADD_Intrinsic 1155 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1156 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1157 LLVMMatchType<0>, 1158 LLVMMatchType<0>, 1159 llvm_i32_ty], 1160 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1161 1162 class AdvSIMD_SVE_CMLA_Intrinsic 1163 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1164 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1165 LLVMMatchType<0>, 1166 LLVMMatchType<0>, 1167 LLVMMatchType<0>, 1168 llvm_i32_ty], 1169 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 1170 1171 class AdvSIMD_SVE_CMLA_LANE_Intrinsic 1172 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1173 [LLVMMatchType<0>, 1174 LLVMMatchType<0>, 1175 LLVMMatchType<0>, 1176 llvm_i32_ty, 1177 llvm_i32_ty], 1178 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1179 1180 class AdvSIMD_SVE_DUP_Intrinsic 1181 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1182 [LLVMMatchType<0>, 1183 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1184 LLVMVectorElementType<0>], 1185 [IntrNoMem]>; 1186 1187 class AdvSIMD_SVE_DUP_Unpred_Intrinsic 1188 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>], 1189 [IntrNoMem]>; 1190 1191 class AdvSIMD_SVE_DUPQ_Intrinsic 1192 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1193 [LLVMMatchType<0>, 1194 llvm_i64_ty], 1195 [IntrNoMem]>; 1196 1197 class AdvSIMD_SVE_EXPA_Intrinsic 1198 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1199 [LLVMVectorOfBitcastsToInt<0>], 1200 [IntrNoMem]>; 1201 1202 class AdvSIMD_SVE_FCVT_Intrinsic 1203 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1204 [LLVMMatchType<0>, 1205 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1206 llvm_anyvector_ty], 1207 [IntrNoMem]>; 1208 1209 class AdvSIMD_SVE_FCVTZS_Intrinsic 1210 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1211 [LLVMVectorOfBitcastsToInt<0>, 1212 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1213 llvm_anyvector_ty], 1214 [IntrNoMem]>; 1215 1216 class AdvSIMD_SVE_INSR_Intrinsic 1217 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1218 [LLVMMatchType<0>, 1219 LLVMVectorElementType<0>], 1220 [IntrNoMem]>; 1221 1222 class AdvSIMD_SVE_PTRUE_Intrinsic 1223 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1224 [llvm_i32_ty], 1225 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1226 1227 class AdvSIMD_SVE_PUNPKHI_Intrinsic 1228 : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>], 1229 [llvm_anyvector_ty], 1230 [IntrNoMem]>; 1231 1232 class AdvSIMD_SVE_SCALE_Intrinsic 1233 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1234 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1235 LLVMMatchType<0>, 1236 LLVMVectorOfBitcastsToInt<0>], 1237 [IntrNoMem]>; 1238 1239 class AdvSIMD_SVE_SCVTF_Intrinsic 1240 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1241 [LLVMMatchType<0>, 1242 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1243 llvm_anyvector_ty], 1244 [IntrNoMem]>; 1245 1246 class AdvSIMD_SVE_TSMUL_Intrinsic 1247 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1248 [LLVMMatchType<0>, 1249 LLVMVectorOfBitcastsToInt<0>], 1250 [IntrNoMem]>; 1251 1252 class AdvSIMD_SVE_CNTB_Intrinsic 1253 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1254 [llvm_i32_ty], 1255 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1256 1257 class AdvSIMD_SVE_CNTP_Intrinsic 1258 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1259 [llvm_anyvector_ty, LLVMMatchType<0>], 1260 [IntrNoMem]>; 1261 1262 class AdvSIMD_SVE_DOT_Intrinsic 1263 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1264 [LLVMMatchType<0>, 1265 LLVMSubdivide4VectorType<0>, 1266 LLVMSubdivide4VectorType<0>], 1267 [IntrNoMem]>; 1268 1269 class AdvSIMD_SVE_DOT_Indexed_Intrinsic 1270 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1271 [LLVMMatchType<0>, 1272 LLVMSubdivide4VectorType<0>, 1273 LLVMSubdivide4VectorType<0>, 1274 llvm_i32_ty], 1275 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1276 1277 class AdvSIMD_SVE_PTEST_Intrinsic 1278 : DefaultAttrsIntrinsic<[llvm_i1_ty], 1279 [llvm_anyvector_ty, 1280 LLVMMatchType<0>], 1281 [IntrNoMem]>; 1282 1283 class AdvSIMD_SVE_TBL_Intrinsic 1284 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1285 [LLVMMatchType<0>, 1286 LLVMVectorOfBitcastsToInt<0>], 1287 [IntrNoMem]>; 1288 1289 class AdvSIMD_SVE2_TBX_Intrinsic 1290 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1291 [LLVMMatchType<0>, 1292 LLVMMatchType<0>, 1293 LLVMVectorOfBitcastsToInt<0>], 1294 [IntrNoMem]>; 1295 1296 class SVE2_1VectorArg_Long_Intrinsic 1297 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1298 [LLVMSubdivide2VectorType<0>, 1299 llvm_i32_ty], 1300 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1301 1302 class SVE2_2VectorArg_Long_Intrinsic 1303 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1304 [LLVMSubdivide2VectorType<0>, 1305 LLVMSubdivide2VectorType<0>], 1306 [IntrNoMem]>; 1307 1308 class SVE2_2VectorArgIndexed_Long_Intrinsic 1309 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1310 [LLVMSubdivide2VectorType<0>, 1311 LLVMSubdivide2VectorType<0>, 1312 llvm_i32_ty], 1313 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1314 1315 class SVE2_2VectorArg_Wide_Intrinsic 1316 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1317 [LLVMMatchType<0>, 1318 LLVMSubdivide2VectorType<0>], 1319 [IntrNoMem]>; 1320 1321 class SVE2_2VectorArg_Pred_Long_Intrinsic 1322 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1323 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1324 LLVMMatchType<0>, 1325 LLVMSubdivide2VectorType<0>], 1326 [IntrNoMem]>; 1327 1328 class SVE2_3VectorArg_Long_Intrinsic 1329 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1330 [LLVMMatchType<0>, 1331 LLVMSubdivide2VectorType<0>, 1332 LLVMSubdivide2VectorType<0>], 1333 [IntrNoMem]>; 1334 1335 class SVE2_3VectorArgIndexed_Long_Intrinsic 1336 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1337 [LLVMMatchType<0>, 1338 LLVMSubdivide2VectorType<0>, 1339 LLVMSubdivide2VectorType<0>, 1340 llvm_i32_ty], 1341 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1342 1343 class SVE2_1VectorArg_Narrowing_Intrinsic 1344 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1345 [llvm_anyvector_ty], 1346 [IntrNoMem]>; 1347 1348 class SVE2_Merged1VectorArg_Narrowing_Intrinsic 1349 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1350 [LLVMSubdivide2VectorType<0>, 1351 llvm_anyvector_ty], 1352 [IntrNoMem]>; 1353 class SVE2_2VectorArg_Narrowing_Intrinsic 1354 : DefaultAttrsIntrinsic< 1355 [LLVMSubdivide2VectorType<0>], 1356 [llvm_anyvector_ty, LLVMMatchType<0>], 1357 [IntrNoMem]>; 1358 1359 class SVE2_Merged2VectorArg_Narrowing_Intrinsic 1360 : DefaultAttrsIntrinsic< 1361 [LLVMSubdivide2VectorType<0>], 1362 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>], 1363 [IntrNoMem]>; 1364 1365 class SVE2_1VectorArg_Imm_Narrowing_Intrinsic 1366 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1367 [llvm_anyvector_ty, llvm_i32_ty], 1368 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1369 1370 class SVE2_2VectorArg_Imm_Narrowing_Intrinsic 1371 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1372 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, 1373 llvm_i32_ty], 1374 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1375 1376 class SVE2_CONFLICT_DETECT_Intrinsic 1377 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1378 [LLVMAnyPointerType<llvm_any_ty>, 1379 LLVMMatchType<1>]>; 1380 1381 class SVE2_3VectorArg_Indexed_Intrinsic 1382 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1383 [LLVMMatchType<0>, 1384 LLVMSubdivide2VectorType<0>, 1385 LLVMSubdivide2VectorType<0>, 1386 llvm_i32_ty], 1387 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1388 1389 class AdvSIMD_SVE_CDOT_LANE_Intrinsic 1390 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1391 [LLVMMatchType<0>, 1392 LLVMSubdivide4VectorType<0>, 1393 LLVMSubdivide4VectorType<0>, 1394 llvm_i32_ty, 1395 llvm_i32_ty], 1396 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1397 1398 // NOTE: There is no relationship between these intrinsics beyond an attempt 1399 // to reuse currently identical class definitions. 1400 class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; 1401 class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic; 1402 class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic; 1403 1404 // This class of intrinsics are not intended to be useful within LLVM IR but 1405 // are instead here to support some of the more regid parts of the ACLE. 1406 class Builtin_SVCVT<LLVMType OUT, LLVMType PRED, LLVMType IN> 1407 : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>; 1408} 1409 1410//===----------------------------------------------------------------------===// 1411// SVE 1412 1413let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 1414 1415class AdvSIMD_SVE_Reduce_Intrinsic 1416 : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], 1417 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1418 llvm_anyvector_ty], 1419 [IntrNoMem]>; 1420 1421class AdvSIMD_SVE_SADDV_Reduce_Intrinsic 1422 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1423 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1424 llvm_anyvector_ty], 1425 [IntrNoMem]>; 1426 1427class AdvSIMD_SVE_WHILE_Intrinsic 1428 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1429 [llvm_anyint_ty, LLVMMatchType<1>], 1430 [IntrNoMem]>; 1431 1432class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic 1433 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1434 [ 1435 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1436 LLVMPointerToElt<0>, 1437 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1438 ], 1439 [IntrReadMem, IntrArgMemOnly]>; 1440 1441class AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic 1442 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1443 [ 1444 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1445 LLVMPointerToElt<0>, 1446 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1447 ], 1448 [IntrInaccessibleMemOrArgMemOnly]>; 1449 1450class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic 1451 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1452 [ 1453 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1454 LLVMPointerToElt<0>, 1455 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1456 ], 1457 [IntrReadMem, IntrArgMemOnly]>; 1458 1459class AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic 1460 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1461 [ 1462 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1463 LLVMPointerToElt<0>, 1464 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1465 ], 1466 [IntrInaccessibleMemOrArgMemOnly]>; 1467 1468class AdvSIMD_GatherLoad_VS_Intrinsic 1469 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1470 [ 1471 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1472 llvm_anyvector_ty, 1473 llvm_i64_ty 1474 ], 1475 [IntrReadMem]>; 1476 1477class AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic 1478 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1479 [ 1480 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1481 llvm_anyvector_ty, 1482 llvm_i64_ty 1483 ], 1484 [IntrInaccessibleMemOrArgMemOnly]>; 1485 1486class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic 1487 : DefaultAttrsIntrinsic<[], 1488 [ 1489 llvm_anyvector_ty, 1490 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1491 LLVMPointerToElt<0>, 1492 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1493 ], 1494 [IntrWriteMem, IntrArgMemOnly]>; 1495 1496class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic 1497 : DefaultAttrsIntrinsic<[], 1498 [ 1499 llvm_anyvector_ty, 1500 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1501 LLVMPointerToElt<0>, 1502 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1503 ], 1504 [IntrWriteMem, IntrArgMemOnly]>; 1505 1506class AdvSIMD_ScatterStore_VS_Intrinsic 1507 : DefaultAttrsIntrinsic<[], 1508 [ 1509 llvm_anyvector_ty, 1510 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1511 llvm_anyvector_ty, llvm_i64_ty 1512 ], 1513 [IntrWriteMem]>; 1514 1515 1516class SVE_gather_prf_SV 1517 : DefaultAttrsIntrinsic<[], 1518 [ 1519 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1520 llvm_ptr_ty, // Base address 1521 llvm_anyvector_ty, // Offsets 1522 llvm_i32_ty // Prfop 1523 ], 1524 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>; 1525 1526class SVE_gather_prf_VS 1527 : DefaultAttrsIntrinsic<[], 1528 [ 1529 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1530 llvm_anyvector_ty, // Base addresses 1531 llvm_i64_ty, // Scalar offset 1532 llvm_i32_ty // Prfop 1533 ], 1534 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>; 1535 1536class SVE_MatMul_Intrinsic 1537 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1538 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], 1539 [IntrNoMem]>; 1540 1541class SVE_4Vec_BF16 1542 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 1543 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 1544 [IntrNoMem]>; 1545 1546class SVE_4Vec_BF16_Indexed 1547 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 1548 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty], 1549 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1550 1551// 1552// Vector tuple creation intrinsics (ACLE) 1553// 1554 1555def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple; 1556def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple; 1557def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple; 1558 1559// 1560// Vector tuple insertion/extraction intrinsics (ACLE) 1561// 1562 1563def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple; 1564def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple; 1565 1566// 1567// Loads 1568// 1569 1570def int_aarch64_sve_ld1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1571 1572def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1573def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1574def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1575 1576def int_aarch64_sve_ld2_sret : AdvSIMD_2Vec_PredLoad_Intrinsic; 1577def int_aarch64_sve_ld3_sret : AdvSIMD_3Vec_PredLoad_Intrinsic; 1578def int_aarch64_sve_ld4_sret : AdvSIMD_4Vec_PredLoad_Intrinsic; 1579 1580def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1581def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; 1582def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; 1583 1584def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic; 1585def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic; 1586 1587// 1588// Stores 1589// 1590 1591def int_aarch64_sve_st1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1592def int_aarch64_sve_st2 : AdvSIMD_2Vec_PredStore_Intrinsic; 1593def int_aarch64_sve_st3 : AdvSIMD_3Vec_PredStore_Intrinsic; 1594def int_aarch64_sve_st4 : AdvSIMD_4Vec_PredStore_Intrinsic; 1595 1596def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1597 1598// 1599// Prefetches 1600// 1601 1602def int_aarch64_sve_prf 1603 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty], 1604 [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>; 1605 1606// Scalar + 32-bit scaled offset vector, zero extend, packed and 1607// unpacked. 1608def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV; 1609def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV; 1610def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV; 1611def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV; 1612 1613// Scalar + 32-bit scaled offset vector, sign extend, packed and 1614// unpacked. 1615def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV; 1616def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV; 1617def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV; 1618def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV; 1619 1620// Scalar + 64-bit scaled offset vector. 1621def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV; 1622def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV; 1623def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV; 1624def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV; 1625 1626// Vector + scalar. 1627def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS; 1628def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS; 1629def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS; 1630def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS; 1631 1632// 1633// Scalar to vector operations 1634// 1635 1636def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic; 1637def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic; 1638 1639def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic; 1640 1641// 1642// Address calculation 1643// 1644 1645def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic; 1646def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic; 1647def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic; 1648def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic; 1649 1650// 1651// Integer arithmetic 1652// 1653 1654def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic; 1655def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; 1656def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic; 1657 1658def int_aarch64_sve_pmul : AdvSIMD_2VectorArg_Intrinsic; 1659 1660def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic; 1661def int_aarch64_sve_mul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1662def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1663def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1664 1665def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1666def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1667def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1668def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1669 1670def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic; 1671def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic; 1672def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic; 1673def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic; 1674def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1675def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1676 1677def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; 1678def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; 1679def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; 1680def int_aarch64_sve_mla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1681def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; 1682def int_aarch64_sve_mls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1683 1684def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1685def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1686 1687def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1688def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1689def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic; 1690def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic; 1691 1692def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic; 1693def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic; 1694def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic; 1695 1696def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; 1697def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; 1698 1699def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic; 1700def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1701 1702def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; 1703def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1704 1705def int_aarch64_sve_sqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1706def int_aarch64_sve_sqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1707def int_aarch64_sve_uqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1708def int_aarch64_sve_uqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1709 1710// Shifts 1711 1712def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic; 1713def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1714def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic; 1715def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic; 1716def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic; 1717def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1718def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic; 1719def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1720 1721// 1722// Integer comparisons 1723// 1724 1725def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1726def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic; 1727def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1728def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic; 1729def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic; 1730def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic; 1731 1732def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1733def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1734def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1735def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1736def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1737def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1738def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1739def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1740def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1741def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1742 1743// 1744// Counting bits 1745// 1746 1747def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic; 1748def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic; 1749def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; 1750 1751// 1752// Counting elements 1753// 1754 1755def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic; 1756def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic; 1757def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic; 1758def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic; 1759 1760def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; 1761 1762// 1763// FFR manipulation 1764// 1765 1766def int_aarch64_sve_rdffr : ClangBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; 1767def int_aarch64_sve_rdffr_z : ClangBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>; 1768def int_aarch64_sve_setffr : ClangBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>; 1769def int_aarch64_sve_wrffr : ClangBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>; 1770 1771// 1772// Saturating scalar arithmetic 1773// 1774 1775def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1776def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1777def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1778def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1779 1780def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1781def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1782def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1783def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1784def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1785def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1786def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1787def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1788def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1789def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1790 1791def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1792def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1793def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1794def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1795 1796def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1797def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1798def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1799def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1800def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1801def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1802def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1803def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1804def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1805def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1806 1807def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1808def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1809def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1810def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1811 1812def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1813def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1814def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1815def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1816def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1817def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1818def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1819def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1820def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1821def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1822 1823def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1824def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1825def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1826def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1827 1828def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1829def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1830def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1831def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1832def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1833def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1834def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1835def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1836def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1837def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1838 1839// 1840// Reversal 1841// 1842 1843def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic; 1844def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic; 1845def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic; 1846def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic; 1847 1848// 1849// Permutations and selection 1850// 1851 1852def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic; 1853def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1854def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic; 1855def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1856def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic; 1857def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic; 1858def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic; 1859def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic; 1860def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; 1861def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; 1862def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; 1863def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; 1864def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1865def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1866def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; 1867def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; 1868def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; 1869def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic; 1870def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic; 1871def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1872def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1873def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; 1874def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; 1875def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic; 1876def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic; 1877def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; 1878def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; 1879def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic; 1880def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic; 1881 1882// 1883// Logical operations 1884// 1885 1886def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic; 1887def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic; 1888def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic; 1889def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic; 1890def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic; 1891def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic; 1892 1893// 1894// Conversion 1895// 1896 1897def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1898def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1899def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1900def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1901def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1902def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1903 1904// 1905// While comparisons 1906// 1907 1908def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic; 1909def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic; 1910def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic; 1911def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic; 1912def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic; 1913def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic; 1914def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic; 1915def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic; 1916 1917// 1918// Floating-point arithmetic 1919// 1920 1921def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1922def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic; 1923def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic; 1924def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic; 1925def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic; 1926def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 1927def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1928def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1929def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic; 1930def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic; 1931def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic; 1932def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic; 1933def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic; 1934def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic; 1935def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic; 1936def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1937def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic; 1938def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1939def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 1940def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic; 1941def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic; 1942def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic; 1943def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1944def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic; 1945def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic; 1946def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic; 1947def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 1948def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic; 1949def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic; 1950def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic; 1951def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic; 1952def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic; 1953def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic; 1954def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic; 1955def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic; 1956def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic; 1957def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic; 1958def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic; 1959def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic; 1960def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic; 1961def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic; 1962def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic; 1963def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 1964def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic; 1965def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic; 1966def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic; 1967 1968// 1969// Floating-point reductions 1970// 1971 1972def int_aarch64_sve_fadda : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1973def int_aarch64_sve_faddv : AdvSIMD_SVE_Reduce_Intrinsic; 1974def int_aarch64_sve_fmaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1975def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic; 1976def int_aarch64_sve_fminv : AdvSIMD_SVE_Reduce_Intrinsic; 1977def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic; 1978 1979// 1980// Floating-point conversions 1981// 1982 1983def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic; 1984def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic; 1985def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic; 1986def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 1987def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 1988 1989// 1990// Floating-point comparisons 1991// 1992 1993def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic; 1994def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic; 1995 1996def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1997def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic; 1998def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1999def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic; 2000def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic; 2001 2002def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2003def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2004def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2005def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2006 2007def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 2008def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 2009 2010def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2011def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2012def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2013def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2014 2015def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 2016def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2017def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2018 2019def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2020def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2021def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2022 2023def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2024def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2025def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 2026def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2027 2028def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2029def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2030 2031def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 2032def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2033def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2034def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 2035 2036def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 2037def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2038def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2039def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 2040 2041// 2042// Predicate creation 2043// 2044 2045def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic; 2046 2047// 2048// Predicate operations 2049// 2050 2051def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2052def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2053def int_aarch64_sve_brka : AdvSIMD_Merged1VectorArg_Intrinsic; 2054def int_aarch64_sve_brka_z : AdvSIMD_Pred1VectorArg_Intrinsic; 2055def int_aarch64_sve_brkb : AdvSIMD_Merged1VectorArg_Intrinsic; 2056def int_aarch64_sve_brkb_z : AdvSIMD_Pred1VectorArg_Intrinsic; 2057def int_aarch64_sve_brkn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2058def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2059def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2060def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2061def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2062def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2063def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2064def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2065def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic; 2066def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic; 2067def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; 2068def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; 2069 2070// 2071// Testing predicates 2072// 2073 2074def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic; 2075def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic; 2076def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic; 2077 2078// 2079// Reinterpreting data 2080// 2081 2082def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2083 [llvm_nxv16i1_ty], 2084 [IntrNoMem]>; 2085 2086def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 2087 [llvm_anyvector_ty], 2088 [IntrNoMem]>; 2089 2090// 2091// Gather loads: scalar base + vector offsets 2092// 2093 2094// 64 bit unscaled offsets 2095def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2096 2097// 64 bit scaled offsets 2098def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2099 2100// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2101def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2102def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2103 2104// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2105def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2106def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2107 2108// 2109// Gather loads: vector base + scalar offset 2110// 2111 2112def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 2113 2114 2115// 2116// First-faulting gather loads: scalar base + vector offsets 2117// 2118 2119// 64 bit unscaled offsets 2120def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; 2121 2122// 64 bit scaled offsets 2123def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; 2124 2125// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 2126def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2127def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2128 2129// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 2130def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2131def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2132 2133// 2134// First-faulting gather loads: vector base + scalar offset 2135// 2136 2137def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic; 2138 2139 2140// 2141// Non-temporal gather loads: scalar base + vector offsets 2142// 2143 2144// 64 bit unscaled offsets 2145def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2146 2147// 64 bit indices 2148def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2149 2150// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2151def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2152 2153// 2154// Non-temporal gather loads: vector base + scalar offset 2155// 2156 2157def int_aarch64_sve_ldnt1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 2158 2159// 2160// Scatter stores: scalar base + vector offsets 2161// 2162 2163// 64 bit unscaled offsets 2164def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2165 2166// 64 bit scaled offsets 2167def int_aarch64_sve_st1_scatter_index 2168 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2169 2170// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2171def int_aarch64_sve_st1_scatter_sxtw 2172 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2173 2174def int_aarch64_sve_st1_scatter_uxtw 2175 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2176 2177// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2178def int_aarch64_sve_st1_scatter_sxtw_index 2179 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2180 2181def int_aarch64_sve_st1_scatter_uxtw_index 2182 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2183 2184// 2185// Scatter stores: vector base + scalar offset 2186// 2187 2188def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2189 2190// 2191// Non-temporal scatter stores: scalar base + vector offsets 2192// 2193 2194// 64 bit unscaled offsets 2195def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2196 2197// 64 bit indices 2198def int_aarch64_sve_stnt1_scatter_index 2199 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2200 2201// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2202def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2203 2204// 2205// Non-temporal scatter stores: vector base + scalar offset 2206// 2207 2208def int_aarch64_sve_stnt1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2209 2210// 2211// SVE2 - Uniform DSP operations 2212// 2213 2214def int_aarch64_sve_saba : AdvSIMD_3VectorArg_Intrinsic; 2215def int_aarch64_sve_shadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2216def int_aarch64_sve_shsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2217def int_aarch64_sve_shsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2218def int_aarch64_sve_sli : AdvSIMD_2VectorArgIndexed_Intrinsic; 2219def int_aarch64_sve_sqabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2220def int_aarch64_sve_sqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2221def int_aarch64_sve_sqdmulh : AdvSIMD_2VectorArg_Intrinsic; 2222def int_aarch64_sve_sqdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2223def int_aarch64_sve_sqneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2224def int_aarch64_sve_sqrdmlah : AdvSIMD_3VectorArg_Intrinsic; 2225def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2226def int_aarch64_sve_sqrdmlsh : AdvSIMD_3VectorArg_Intrinsic; 2227def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2228def int_aarch64_sve_sqrdmulh : AdvSIMD_2VectorArg_Intrinsic; 2229def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2230def int_aarch64_sve_sqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2231def int_aarch64_sve_sqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2232def int_aarch64_sve_sqshlu : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2233def int_aarch64_sve_sqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2234def int_aarch64_sve_sqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2235def int_aarch64_sve_srhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2236def int_aarch64_sve_sri : AdvSIMD_2VectorArgIndexed_Intrinsic; 2237def int_aarch64_sve_srshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2238def int_aarch64_sve_srshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2239def int_aarch64_sve_srsra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2240def int_aarch64_sve_ssra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2241def int_aarch64_sve_suqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2242def int_aarch64_sve_uaba : AdvSIMD_3VectorArg_Intrinsic; 2243def int_aarch64_sve_uhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2244def int_aarch64_sve_uhsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2245def int_aarch64_sve_uhsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2246def int_aarch64_sve_uqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2247def int_aarch64_sve_uqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2248def int_aarch64_sve_uqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2249def int_aarch64_sve_uqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2250def int_aarch64_sve_uqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2251def int_aarch64_sve_urecpe : AdvSIMD_Merged1VectorArg_Intrinsic; 2252def int_aarch64_sve_urhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2253def int_aarch64_sve_urshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2254def int_aarch64_sve_urshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2255def int_aarch64_sve_ursqrte : AdvSIMD_Merged1VectorArg_Intrinsic; 2256def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2257def int_aarch64_sve_usqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2258def int_aarch64_sve_usra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2259 2260// 2261// SVE2 - Widening DSP operations 2262// 2263 2264def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic; 2265def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic; 2266def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic; 2267def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic; 2268def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic; 2269def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic; 2270def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic; 2271def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic; 2272def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic; 2273def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic; 2274def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic; 2275def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic; 2276def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic; 2277def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic; 2278def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic; 2279def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic; 2280def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic; 2281def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic; 2282def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic; 2283def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic; 2284def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic; 2285def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic; 2286def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic; 2287def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic; 2288def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic; 2289def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic; 2290def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic; 2291def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic; 2292 2293// 2294// SVE2 - Non-widening pairwise arithmetic 2295// 2296 2297def int_aarch64_sve_addp : AdvSIMD_Pred2VectorArg_Intrinsic; 2298def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic; 2299def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2300def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2301def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2302def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2303def int_aarch64_sve_smaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2304def int_aarch64_sve_sminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2305def int_aarch64_sve_umaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2306def int_aarch64_sve_uminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2307 2308// 2309// SVE2 - Widening pairwise arithmetic 2310// 2311 2312def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2313def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2314 2315// 2316// SVE2 - Uniform complex integer arithmetic 2317// 2318 2319def int_aarch64_sve_cadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2320def int_aarch64_sve_sqcadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2321def int_aarch64_sve_cmla_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2322def int_aarch64_sve_cmla_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2323def int_aarch64_sve_sqrdcmlah_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2324def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2325 2326// 2327// SVE2 - Widening complex integer arithmetic 2328// 2329 2330def int_aarch64_sve_saddlbt : SVE2_2VectorArg_Long_Intrinsic; 2331def int_aarch64_sve_ssublbt : SVE2_2VectorArg_Long_Intrinsic; 2332def int_aarch64_sve_ssubltb : SVE2_2VectorArg_Long_Intrinsic; 2333 2334// 2335// SVE2 - Widening complex integer dot product 2336// 2337 2338def int_aarch64_sve_cdot : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2339def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic; 2340 2341// 2342// SVE2 - Floating-point widening multiply-accumulate 2343// 2344 2345def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic; 2346def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2347def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic; 2348def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2349def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic; 2350def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2351def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic; 2352def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2353 2354// 2355// SVE2 - Floating-point integer binary logarithm 2356// 2357 2358def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic; 2359 2360// 2361// SVE2 - Vector histogram count 2362// 2363 2364def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic; 2365def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic; 2366 2367// 2368// SVE2 - Character match 2369// 2370 2371def int_aarch64_sve_match : AdvSIMD_SVE_Compare_Intrinsic; 2372def int_aarch64_sve_nmatch : AdvSIMD_SVE_Compare_Intrinsic; 2373 2374// 2375// SVE2 - Unary narrowing operations 2376// 2377 2378def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2379def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2380def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic; 2381def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2382def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2383def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2384 2385// 2386// SVE2 - Binary narrowing DSP operations 2387// 2388def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2389def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2390 2391def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2392def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2393 2394def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2395def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2396 2397def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2398def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2399 2400// Narrowing shift right 2401def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2402def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2403 2404def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2405def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2406 2407// Saturating shift right - signed input/output 2408def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2409def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2410 2411def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2412def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2413 2414// Saturating shift right - unsigned input/output 2415def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2416def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2417 2418def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2419def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2420 2421// Saturating shift right - signed input, unsigned output 2422def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2423def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2424 2425def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2426def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2427 2428// SVE2 MLA LANE. 2429def int_aarch64_sve_smlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2430def int_aarch64_sve_smlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2431def int_aarch64_sve_umlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2432def int_aarch64_sve_umlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2433def int_aarch64_sve_smlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2434def int_aarch64_sve_smlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2435def int_aarch64_sve_umlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2436def int_aarch64_sve_umlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2437def int_aarch64_sve_smullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2438def int_aarch64_sve_smullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2439def int_aarch64_sve_umullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2440def int_aarch64_sve_umullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2441def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2442def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2443def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2444def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2445def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2446def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2447 2448// SVE2 MLA Unpredicated. 2449def int_aarch64_sve_smlalb : SVE2_3VectorArg_Long_Intrinsic; 2450def int_aarch64_sve_smlalt : SVE2_3VectorArg_Long_Intrinsic; 2451def int_aarch64_sve_umlalb : SVE2_3VectorArg_Long_Intrinsic; 2452def int_aarch64_sve_umlalt : SVE2_3VectorArg_Long_Intrinsic; 2453def int_aarch64_sve_smlslb : SVE2_3VectorArg_Long_Intrinsic; 2454def int_aarch64_sve_smlslt : SVE2_3VectorArg_Long_Intrinsic; 2455def int_aarch64_sve_umlslb : SVE2_3VectorArg_Long_Intrinsic; 2456def int_aarch64_sve_umlslt : SVE2_3VectorArg_Long_Intrinsic; 2457def int_aarch64_sve_smullb : SVE2_2VectorArg_Long_Intrinsic; 2458def int_aarch64_sve_smullt : SVE2_2VectorArg_Long_Intrinsic; 2459def int_aarch64_sve_umullb : SVE2_2VectorArg_Long_Intrinsic; 2460def int_aarch64_sve_umullt : SVE2_2VectorArg_Long_Intrinsic; 2461 2462def int_aarch64_sve_sqdmlalb : SVE2_3VectorArg_Long_Intrinsic; 2463def int_aarch64_sve_sqdmlalt : SVE2_3VectorArg_Long_Intrinsic; 2464def int_aarch64_sve_sqdmlslb : SVE2_3VectorArg_Long_Intrinsic; 2465def int_aarch64_sve_sqdmlslt : SVE2_3VectorArg_Long_Intrinsic; 2466def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic; 2467def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic; 2468def int_aarch64_sve_sqdmlalbt : SVE2_3VectorArg_Long_Intrinsic; 2469def int_aarch64_sve_sqdmlslbt : SVE2_3VectorArg_Long_Intrinsic; 2470 2471// SVE2 ADDSUB Long Unpredicated. 2472def int_aarch64_sve_adclb : AdvSIMD_3VectorArg_Intrinsic; 2473def int_aarch64_sve_adclt : AdvSIMD_3VectorArg_Intrinsic; 2474def int_aarch64_sve_sbclb : AdvSIMD_3VectorArg_Intrinsic; 2475def int_aarch64_sve_sbclt : AdvSIMD_3VectorArg_Intrinsic; 2476 2477// 2478// SVE2 - Polynomial arithmetic 2479// 2480def int_aarch64_sve_eorbt : AdvSIMD_3VectorArg_Intrinsic; 2481def int_aarch64_sve_eortb : AdvSIMD_3VectorArg_Intrinsic; 2482def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic; 2483def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic; 2484 2485// 2486// SVE2 bitwise ternary operations. 2487// 2488def int_aarch64_sve_eor3 : AdvSIMD_3VectorArg_Intrinsic; 2489def int_aarch64_sve_bcax : AdvSIMD_3VectorArg_Intrinsic; 2490def int_aarch64_sve_bsl : AdvSIMD_3VectorArg_Intrinsic; 2491def int_aarch64_sve_bsl1n : AdvSIMD_3VectorArg_Intrinsic; 2492def int_aarch64_sve_bsl2n : AdvSIMD_3VectorArg_Intrinsic; 2493def int_aarch64_sve_nbsl : AdvSIMD_3VectorArg_Intrinsic; 2494def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic; 2495 2496// 2497// SVE2 - Optional AES, SHA-3 and SM4 2498// 2499 2500def int_aarch64_sve_aesd : ClangBuiltin<"__builtin_sve_svaesd_u8">, 2501 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2502 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2503 [IntrNoMem]>; 2504def int_aarch64_sve_aesimc : ClangBuiltin<"__builtin_sve_svaesimc_u8">, 2505 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2506 [llvm_nxv16i8_ty], 2507 [IntrNoMem]>; 2508def int_aarch64_sve_aese : ClangBuiltin<"__builtin_sve_svaese_u8">, 2509 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2510 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2511 [IntrNoMem]>; 2512def int_aarch64_sve_aesmc : ClangBuiltin<"__builtin_sve_svaesmc_u8">, 2513 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2514 [llvm_nxv16i8_ty], 2515 [IntrNoMem]>; 2516def int_aarch64_sve_rax1 : ClangBuiltin<"__builtin_sve_svrax1_u64">, 2517 DefaultAttrsIntrinsic<[llvm_nxv2i64_ty], 2518 [llvm_nxv2i64_ty, llvm_nxv2i64_ty], 2519 [IntrNoMem]>; 2520def int_aarch64_sve_sm4e : ClangBuiltin<"__builtin_sve_svsm4e_u32">, 2521 DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], 2522 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2523 [IntrNoMem]>; 2524def int_aarch64_sve_sm4ekey : ClangBuiltin<"__builtin_sve_svsm4ekey_u32">, 2525 DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], 2526 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2527 [IntrNoMem]>; 2528// 2529// SVE2 - Extended table lookup/permute 2530// 2531 2532def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic; 2533def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic; 2534 2535// 2536// SVE2 - Optional bit permutation 2537// 2538 2539def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic; 2540def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic; 2541def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic; 2542 2543 2544// 2545// SVE ACLE: 7.3. INT8 matrix multiply extensions 2546// 2547def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic; 2548def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic; 2549def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic; 2550 2551def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic; 2552def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2553def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2554 2555// 2556// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions 2557// 2558def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; 2559 2560// 2561// SVE ACLE: 7.2. BFloat16 extensions 2562// 2563 2564def int_aarch64_sve_bfdot : SVE_4Vec_BF16; 2565def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16; 2566def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16; 2567 2568def int_aarch64_sve_bfmmla : SVE_4Vec_BF16; 2569 2570def int_aarch64_sve_bfdot_lane : SVE_4Vec_BF16_Indexed; 2571def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed; 2572def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed; 2573} 2574 2575// 2576// SVE2 - Contiguous conflict detection 2577// 2578 2579def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic; 2580def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic; 2581def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic; 2582def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic; 2583def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic; 2584def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic; 2585def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic; 2586def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic; 2587 2588// Scalable Matrix Extension (SME) Intrinsics 2589let TargetPrefix = "aarch64" in { 2590 class SME_Load_Store_Intrinsic<LLVMType pred_ty> 2591 : DefaultAttrsIntrinsic<[], 2592 [pred_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], []>; 2593 2594 // Loads 2595 def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2596 def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2597 def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2598 def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2599 def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2600 def int_aarch64_sme_ld1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2601 def int_aarch64_sme_ld1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2602 def int_aarch64_sme_ld1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2603 def int_aarch64_sme_ld1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2604 def int_aarch64_sme_ld1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2605 2606 // Stores 2607 def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2608 def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2609 def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2610 def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2611 def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2612 def int_aarch64_sme_st1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2613 def int_aarch64_sme_st1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2614 def int_aarch64_sme_st1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2615 def int_aarch64_sme_st1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2616 def int_aarch64_sme_st1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2617 2618 // Spill + fill 2619 def int_aarch64_sme_ldr : DefaultAttrsIntrinsic< 2620 [], [llvm_i32_ty, llvm_ptr_ty]>; 2621 def int_aarch64_sme_str : DefaultAttrsIntrinsic< 2622 [], [llvm_i32_ty, llvm_ptr_ty]>; 2623 2624 class SME_TileToVector_Intrinsic 2625 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2626 [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i64_ty, llvm_i32_ty]>; 2627 class SME_VectorToTile_Intrinsic 2628 : DefaultAttrsIntrinsic<[], 2629 [llvm_i64_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2630 llvm_anyvector_ty]>; 2631 2632 def int_aarch64_sme_read_horiz : SME_TileToVector_Intrinsic; 2633 def int_aarch64_sme_read_vert : SME_TileToVector_Intrinsic; 2634 def int_aarch64_sme_write_horiz : SME_VectorToTile_Intrinsic; 2635 def int_aarch64_sme_write_vert : SME_VectorToTile_Intrinsic; 2636 2637 def int_aarch64_sme_readq_horiz : SME_TileToVector_Intrinsic; 2638 def int_aarch64_sme_readq_vert : SME_TileToVector_Intrinsic; 2639 def int_aarch64_sme_writeq_horiz : SME_VectorToTile_Intrinsic; 2640 def int_aarch64_sme_writeq_vert : SME_VectorToTile_Intrinsic; 2641 2642 def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i64_ty]>; 2643 2644 class SME_OuterProduct_Intrinsic 2645 : DefaultAttrsIntrinsic<[], 2646 [llvm_i64_ty, 2647 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2648 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2649 LLVMMatchType<0>, 2650 llvm_anyvector_ty]>; 2651 2652 def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic; 2653 def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic; 2654 2655 def int_aarch64_sme_mopa_wide : SME_OuterProduct_Intrinsic; 2656 def int_aarch64_sme_mops_wide : SME_OuterProduct_Intrinsic; 2657 2658 def int_aarch64_sme_smopa_wide : SME_OuterProduct_Intrinsic; 2659 def int_aarch64_sme_smops_wide : SME_OuterProduct_Intrinsic; 2660 def int_aarch64_sme_umopa_wide : SME_OuterProduct_Intrinsic; 2661 def int_aarch64_sme_umops_wide : SME_OuterProduct_Intrinsic; 2662 def int_aarch64_sme_sumopa_wide : SME_OuterProduct_Intrinsic; 2663 def int_aarch64_sme_sumops_wide : SME_OuterProduct_Intrinsic; 2664 def int_aarch64_sme_usmopa_wide : SME_OuterProduct_Intrinsic; 2665 def int_aarch64_sme_usmops_wide : SME_OuterProduct_Intrinsic; 2666 2667 class SME_AddVectorToTile_Intrinsic 2668 : DefaultAttrsIntrinsic<[], 2669 [llvm_i64_ty, 2670 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2671 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2672 llvm_anyvector_ty]>; 2673 2674 def int_aarch64_sme_addha : SME_AddVectorToTile_Intrinsic; 2675 def int_aarch64_sme_addva : SME_AddVectorToTile_Intrinsic; 2676 2677 // 2678 // Counting elements 2679 // 2680 2681 class AdvSIMD_SME_CNTSB_Intrinsic 2682 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem]>; 2683 2684 def int_aarch64_sme_cntsb : AdvSIMD_SME_CNTSB_Intrinsic; 2685 def int_aarch64_sme_cntsh : AdvSIMD_SME_CNTSB_Intrinsic; 2686 def int_aarch64_sme_cntsw : AdvSIMD_SME_CNTSB_Intrinsic; 2687 def int_aarch64_sme_cntsd : AdvSIMD_SME_CNTSB_Intrinsic; 2688 2689 // 2690 // PSTATE Functions 2691 // 2692 2693 def int_aarch64_sme_get_pstatesm 2694 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], 2695 [IntrReadMem, IntrInaccessibleMemOnly]>; 2696 2697 def int_aarch64_sme_get_tpidr2 2698 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], 2699 [IntrNoMem, IntrHasSideEffects]>; 2700 def int_aarch64_sme_set_tpidr2 2701 : DefaultAttrsIntrinsic<[], [llvm_i64_ty], 2702 [IntrNoMem, IntrHasSideEffects]>; 2703 // Clamp 2704 // 2705 2706 def int_aarch64_sve_sclamp : AdvSIMD_3VectorArg_Intrinsic; 2707 def int_aarch64_sve_uclamp : AdvSIMD_3VectorArg_Intrinsic; 2708 2709 // 2710 // Reversal 2711 // 2712 2713 def int_aarch64_sve_revd : AdvSIMD_Merged1VectorArg_Intrinsic; 2714 2715 // 2716 // Predicate selection 2717 // 2718 2719 def int_aarch64_sve_psel 2720 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2721 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2722 LLVMMatchType<0>, llvm_i32_ty]>; 2723} 2724