1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines all of the AARCH64-specific intrinsics. 10// 11//===----------------------------------------------------------------------===// 12 13let TargetPrefix = "aarch64" in { 14 15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; 16def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; 17def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>; 18def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>; 19 20def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>; 21def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>; 22def int_aarch64_stxp : Intrinsic<[llvm_i32_ty], 23 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>; 24def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty], 25 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>; 26 27def int_aarch64_clrex : Intrinsic<[]>; 28 29def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 30 LLVMMatchType<0>], [IntrNoMem]>; 31def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 32 LLVMMatchType<0>], [IntrNoMem]>; 33 34def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; 35 36def int_aarch64_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 37def int_aarch64_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; 38 39//===----------------------------------------------------------------------===// 40// HINT 41 42def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>; 43 44//===----------------------------------------------------------------------===// 45// Data Barrier Instructions 46 47def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>; 48def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>; 49def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>; 50 51// A space-consuming intrinsic primarily for testing block and jump table 52// placements. The first argument is the number of bytes this "instruction" 53// takes up, the second and return value are essentially chains, used to force 54// ordering during ISel. 55def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>; 56 57} 58 59//===----------------------------------------------------------------------===// 60// Advanced SIMD (NEON) 61 62let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 63 class AdvSIMD_2Scalar_Float_Intrinsic 64 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 65 [IntrNoMem]>; 66 67 class AdvSIMD_FPToIntRounding_Intrinsic 68 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>; 69 70 class AdvSIMD_1IntArg_Intrinsic 71 : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>; 72 class AdvSIMD_1FloatArg_Intrinsic 73 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; 74 class AdvSIMD_1VectorArg_Intrinsic 75 : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>; 76 class AdvSIMD_1VectorArg_Expand_Intrinsic 77 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>; 78 class AdvSIMD_1VectorArg_Long_Intrinsic 79 : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>; 80 class AdvSIMD_1IntArg_Narrow_Intrinsic 81 : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>; 82 class AdvSIMD_1VectorArg_Narrow_Intrinsic 83 : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>; 84 class AdvSIMD_1VectorArg_Int_Across_Intrinsic 85 : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>; 86 class AdvSIMD_1VectorArg_Float_Across_Intrinsic 87 : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>; 88 89 class AdvSIMD_2IntArg_Intrinsic 90 : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 91 [IntrNoMem]>; 92 class AdvSIMD_2FloatArg_Intrinsic 93 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 94 [IntrNoMem]>; 95 class AdvSIMD_2VectorArg_Intrinsic 96 : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 97 [IntrNoMem]>; 98 class AdvSIMD_2VectorArg_Compare_Intrinsic 99 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>], 100 [IntrNoMem]>; 101 class AdvSIMD_2Arg_FloatCompare_Intrinsic 102 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>], 103 [IntrNoMem]>; 104 class AdvSIMD_2VectorArg_Long_Intrinsic 105 : Intrinsic<[llvm_anyvector_ty], 106 [LLVMTruncatedType<0>, LLVMTruncatedType<0>], 107 [IntrNoMem]>; 108 class AdvSIMD_2VectorArg_Wide_Intrinsic 109 : Intrinsic<[llvm_anyvector_ty], 110 [LLVMMatchType<0>, LLVMTruncatedType<0>], 111 [IntrNoMem]>; 112 class AdvSIMD_2VectorArg_Narrow_Intrinsic 113 : Intrinsic<[llvm_anyvector_ty], 114 [LLVMExtendedType<0>, LLVMExtendedType<0>], 115 [IntrNoMem]>; 116 class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic 117 : Intrinsic<[llvm_anyint_ty], 118 [LLVMExtendedType<0>, llvm_i32_ty], 119 [IntrNoMem]>; 120 class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic 121 : Intrinsic<[llvm_anyvector_ty], 122 [llvm_anyvector_ty], 123 [IntrNoMem]>; 124 class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic 125 : Intrinsic<[llvm_anyvector_ty], 126 [LLVMTruncatedType<0>], 127 [IntrNoMem]>; 128 class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic 129 : Intrinsic<[llvm_anyvector_ty], 130 [LLVMTruncatedType<0>, llvm_i32_ty], 131 [IntrNoMem]>; 132 class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic 133 : Intrinsic<[llvm_anyvector_ty], 134 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty], 135 [IntrNoMem]>; 136 class AdvSIMD_2VectorArg_Lane_Intrinsic 137 : Intrinsic<[llvm_anyint_ty], 138 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty], 139 [IntrNoMem]>; 140 141 class AdvSIMD_3VectorArg_Intrinsic 142 : Intrinsic<[llvm_anyvector_ty], 143 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 144 [IntrNoMem]>; 145 class AdvSIMD_3VectorArg_Scalar_Intrinsic 146 : Intrinsic<[llvm_anyvector_ty], 147 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], 148 [IntrNoMem]>; 149 class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic 150 : Intrinsic<[llvm_anyvector_ty], 151 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, 152 LLVMMatchType<1>], [IntrNoMem]>; 153 class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic 154 : Intrinsic<[llvm_anyvector_ty], 155 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty], 156 [IntrNoMem]>; 157 class AdvSIMD_CvtFxToFP_Intrinsic 158 : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], 159 [IntrNoMem]>; 160 class AdvSIMD_CvtFPToFx_Intrinsic 161 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty], 162 [IntrNoMem]>; 163 164 class AdvSIMD_1Arg_Intrinsic 165 : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>; 166 167 class AdvSIMD_Dot_Intrinsic 168 : Intrinsic<[llvm_anyvector_ty], 169 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 170 [IntrNoMem]>; 171 172 class AdvSIMD_FP16FML_Intrinsic 173 : Intrinsic<[llvm_anyvector_ty], 174 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 175 [IntrNoMem]>; 176 177 class AdvSIMD_MatMul_Intrinsic 178 : Intrinsic<[llvm_anyvector_ty], 179 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 180 [IntrNoMem]>; 181 182 class AdvSIMD_FML_Intrinsic 183 : Intrinsic<[llvm_anyvector_ty], 184 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 185 [IntrNoMem]>; 186 187} 188 189// Arithmetic ops 190 191let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { 192 // Vector Add Across Lanes 193 def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 194 def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 195 def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 196 197 // Vector Long Add Across Lanes 198 def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 199 def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 200 201 // Vector Halving Add 202 def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic; 203 def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic; 204 205 // Vector Rounding Halving Add 206 def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic; 207 def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic; 208 209 // Vector Saturating Add 210 def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic; 211 def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic; 212 def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic; 213 def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic; 214 215 // Vector Add High-Half 216 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 217 // header is no longer supported. 218 def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 219 220 // Vector Rounding Add High-Half 221 def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 222 223 // Vector Saturating Doubling Multiply High 224 def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic; 225 def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 226 def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 227 228 // Vector Saturating Rounding Doubling Multiply High 229 def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic; 230 def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 231 def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 232 233 // Vector Polynominal Multiply 234 def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic; 235 236 // Vector Long Multiply 237 def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic; 238 def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic; 239 def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic; 240 241 // 64-bit polynomial multiply really returns an i128, which is not legal. Fake 242 // it with a v16i8. 243 def int_aarch64_neon_pmull64 : 244 Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 245 246 // Vector Extending Multiply 247 def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic { 248 let IntrProperties = [IntrNoMem, Commutative]; 249 } 250 251 // Vector Saturating Doubling Long Multiply 252 def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic; 253 def int_aarch64_neon_sqdmulls_scalar 254 : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; 255 256 // Vector Halving Subtract 257 def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic; 258 def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic; 259 260 // Vector Saturating Subtract 261 def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic; 262 def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic; 263 264 // Vector Subtract High-Half 265 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 266 // header is no longer supported. 267 def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 268 269 // Vector Rounding Subtract High-Half 270 def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 271 272 // Vector Compare Absolute Greater-than-or-equal 273 def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic; 274 275 // Vector Compare Absolute Greater-than 276 def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic; 277 278 // Vector Absolute Difference 279 def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic; 280 def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic; 281 def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic; 282 283 // Scalar Absolute Difference 284 def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic; 285 286 // Vector Max 287 def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic; 288 def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic; 289 def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic; 290 def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic; 291 292 // Vector Max Across Lanes 293 def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 294 def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 295 def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 296 def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 297 298 // Vector Min 299 def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic; 300 def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic; 301 def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic; 302 def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic; 303 304 // Vector Min/Max Number 305 def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic; 306 def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic; 307 308 // Vector Min Across Lanes 309 def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 310 def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 311 def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 312 def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 313 314 // Pairwise Add 315 def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic; 316 def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic; 317 318 // Long Pairwise Add 319 // FIXME: In theory, we shouldn't need intrinsics for saddlp or 320 // uaddlp, but tblgen's type inference currently can't handle the 321 // pattern fragments this ends up generating. 322 def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 323 def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 324 325 // Folding Maximum 326 def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic; 327 def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic; 328 def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic; 329 330 // Folding Minimum 331 def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic; 332 def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic; 333 def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic; 334 335 // Reciprocal Estimate/Step 336 def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic; 337 def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic; 338 339 // Reciprocal Exponent 340 def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic; 341 342 // Vector Saturating Shift Left 343 def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic; 344 def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic; 345 346 // Vector Rounding Shift Left 347 def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic; 348 def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic; 349 350 // Vector Saturating Rounding Shift Left 351 def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic; 352 def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic; 353 354 // Vector Signed->Unsigned Shift Left by Constant 355 def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic; 356 357 // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant 358 def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 359 360 // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const 361 def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 362 363 // Vector Narrowing Shift Right by Constant 364 def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 365 def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 366 367 // Vector Rounding Narrowing Shift Right by Constant 368 def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 369 370 // Vector Rounding Narrowing Saturating Shift Right by Constant 371 def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 372 def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 373 374 // Vector Shift Left 375 def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic; 376 def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic; 377 378 // Vector Widening Shift Left by Constant 379 def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic; 380 def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 381 def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 382 383 // Vector Shift Right by Constant and Insert 384 def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic; 385 386 // Vector Shift Left by Constant and Insert 387 def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic; 388 389 // Vector Saturating Narrow 390 def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic; 391 def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic; 392 def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 393 def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 394 395 // Vector Saturating Extract and Unsigned Narrow 396 def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic; 397 def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic; 398 399 // Vector Absolute Value 400 def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic; 401 402 // Vector Saturating Absolute Value 403 def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic; 404 405 // Vector Saturating Negation 406 def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic; 407 408 // Vector Count Leading Sign Bits 409 def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic; 410 411 // Vector Reciprocal Estimate 412 def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic; 413 def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic; 414 415 // Vector Square Root Estimate 416 def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic; 417 def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic; 418 419 // Vector Bitwise Reverse 420 def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic; 421 422 // Vector Conversions Between Half-Precision and Single-Precision. 423 def int_aarch64_neon_vcvtfp2hf 424 : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 425 def int_aarch64_neon_vcvthf2fp 426 : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>; 427 428 // Vector Conversions Between Floating-point and Fixed-point. 429 def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic; 430 def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic; 431 def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic; 432 def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic; 433 434 // Vector FP->Int Conversions 435 def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic; 436 def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic; 437 def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic; 438 def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic; 439 def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic; 440 def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic; 441 def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic; 442 def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic; 443 def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic; 444 def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic; 445 446 // Vector FP Rounding: only ties to even is unrepresented by a normal 447 // intrinsic. 448 def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic; 449 450 // Scalar FP->Int conversions 451 452 // Vector FP Inexact Narrowing 453 def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic; 454 455 // Scalar FP Inexact Narrowing 456 def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty], 457 [IntrNoMem]>; 458 459 // v8.2-A Dot Product 460 def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic; 461 def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic; 462 463// v8.6-A Matrix Multiply Intrinsics 464 def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic; 465 def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic; 466 def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic; 467 def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic; 468 def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic; 469 def int_aarch64_neon_bfmmla : AdvSIMD_MatMul_Intrinsic; 470 def int_aarch64_neon_bfmlalb : AdvSIMD_FML_Intrinsic; 471 def int_aarch64_neon_bfmlalt : AdvSIMD_FML_Intrinsic; 472 473 474 // v8.6-A Bfloat Intrinsics 475 def int_aarch64_neon_bfcvt 476 : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>; 477 def int_aarch64_neon_bfcvtn 478 : Intrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 479 def int_aarch64_neon_bfcvtn2 480 : Intrinsic<[llvm_v8bf16_ty], 481 [llvm_v8bf16_ty, llvm_v4f32_ty], 482 [IntrNoMem]>; 483 484 // v8.2-A FP16 Fused Multiply-Add Long 485 def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic; 486 def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic; 487 def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic; 488 def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic; 489 490 // v8.3-A Floating-point complex add 491 def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic; 492 def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic; 493} 494 495let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 496 class AdvSIMD_2Vector2Index_Intrinsic 497 : Intrinsic<[llvm_anyvector_ty], 498 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty], 499 [IntrNoMem]>; 500} 501 502// Vector element to element moves 503def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic; 504 505let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 506 class AdvSIMD_1Vec_Load_Intrinsic 507 : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>], 508 [IntrReadMem, IntrArgMemOnly]>; 509 class AdvSIMD_1Vec_Store_Lane_Intrinsic 510 : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], 511 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 512 513 class AdvSIMD_2Vec_Load_Intrinsic 514 : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], 515 [LLVMAnyPointerType<LLVMMatchType<0>>], 516 [IntrReadMem, IntrArgMemOnly]>; 517 class AdvSIMD_2Vec_Load_Lane_Intrinsic 518 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>], 519 [LLVMMatchType<0>, llvm_anyvector_ty, 520 llvm_i64_ty, llvm_anyptr_ty], 521 [IntrReadMem, IntrArgMemOnly]>; 522 class AdvSIMD_2Vec_Store_Intrinsic 523 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 524 LLVMAnyPointerType<LLVMMatchType<0>>], 525 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 526 class AdvSIMD_2Vec_Store_Lane_Intrinsic 527 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 528 llvm_i64_ty, llvm_anyptr_ty], 529 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 530 531 class AdvSIMD_3Vec_Load_Intrinsic 532 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty], 533 [LLVMAnyPointerType<LLVMMatchType<0>>], 534 [IntrReadMem, IntrArgMemOnly]>; 535 class AdvSIMD_3Vec_Load_Lane_Intrinsic 536 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 537 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, 538 llvm_i64_ty, llvm_anyptr_ty], 539 [IntrReadMem, IntrArgMemOnly]>; 540 class AdvSIMD_3Vec_Store_Intrinsic 541 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 542 LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>], 543 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 544 class AdvSIMD_3Vec_Store_Lane_Intrinsic 545 : Intrinsic<[], [llvm_anyvector_ty, 546 LLVMMatchType<0>, LLVMMatchType<0>, 547 llvm_i64_ty, llvm_anyptr_ty], 548 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 549 550 class AdvSIMD_4Vec_Load_Intrinsic 551 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 552 LLVMMatchType<0>, llvm_anyvector_ty], 553 [LLVMAnyPointerType<LLVMMatchType<0>>], 554 [IntrReadMem, IntrArgMemOnly]>; 555 class AdvSIMD_4Vec_Load_Lane_Intrinsic 556 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 557 LLVMMatchType<0>, LLVMMatchType<0>], 558 [LLVMMatchType<0>, LLVMMatchType<0>, 559 LLVMMatchType<0>, llvm_anyvector_ty, 560 llvm_i64_ty, llvm_anyptr_ty], 561 [IntrReadMem, IntrArgMemOnly]>; 562 class AdvSIMD_4Vec_Store_Intrinsic 563 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 564 LLVMMatchType<0>, LLVMMatchType<0>, 565 LLVMAnyPointerType<LLVMMatchType<0>>], 566 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 567 class AdvSIMD_4Vec_Store_Lane_Intrinsic 568 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 569 LLVMMatchType<0>, LLVMMatchType<0>, 570 llvm_i64_ty, llvm_anyptr_ty], 571 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 572} 573 574// Memory ops 575 576def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic; 577def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic; 578def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic; 579 580def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic; 581def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic; 582def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic; 583 584def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic; 585def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic; 586def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic; 587 588def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic; 589def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic; 590def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic; 591 592def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic; 593def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic; 594def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic; 595 596def int_aarch64_neon_st2 : AdvSIMD_2Vec_Store_Intrinsic; 597def int_aarch64_neon_st3 : AdvSIMD_3Vec_Store_Intrinsic; 598def int_aarch64_neon_st4 : AdvSIMD_4Vec_Store_Intrinsic; 599 600def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic; 601def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic; 602def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic; 603 604let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 605 class AdvSIMD_Tbl1_Intrinsic 606 : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>], 607 [IntrNoMem]>; 608 class AdvSIMD_Tbl2_Intrinsic 609 : Intrinsic<[llvm_anyvector_ty], 610 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>; 611 class AdvSIMD_Tbl3_Intrinsic 612 : Intrinsic<[llvm_anyvector_ty], 613 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 614 LLVMMatchType<0>], 615 [IntrNoMem]>; 616 class AdvSIMD_Tbl4_Intrinsic 617 : Intrinsic<[llvm_anyvector_ty], 618 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 619 LLVMMatchType<0>], 620 [IntrNoMem]>; 621 622 class AdvSIMD_Tbx1_Intrinsic 623 : Intrinsic<[llvm_anyvector_ty], 624 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>], 625 [IntrNoMem]>; 626 class AdvSIMD_Tbx2_Intrinsic 627 : Intrinsic<[llvm_anyvector_ty], 628 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 629 LLVMMatchType<0>], 630 [IntrNoMem]>; 631 class AdvSIMD_Tbx3_Intrinsic 632 : Intrinsic<[llvm_anyvector_ty], 633 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 634 llvm_v16i8_ty, LLVMMatchType<0>], 635 [IntrNoMem]>; 636 class AdvSIMD_Tbx4_Intrinsic 637 : Intrinsic<[llvm_anyvector_ty], 638 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 639 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], 640 [IntrNoMem]>; 641} 642def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic; 643def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic; 644def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic; 645def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic; 646 647def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic; 648def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic; 649def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic; 650def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic; 651 652let TargetPrefix = "aarch64" in { 653 class FPCR_Get_Intrinsic 654 : Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>; 655} 656 657// FPCR 658def int_aarch64_get_fpcr : FPCR_Get_Intrinsic; 659 660let TargetPrefix = "aarch64" in { 661 class Crypto_AES_DataKey_Intrinsic 662 : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; 663 664 class Crypto_AES_Data_Intrinsic 665 : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; 666 667 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 668 // (v4i32). 669 class Crypto_SHA_5Hash4Schedule_Intrinsic 670 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], 671 [IntrNoMem]>; 672 673 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 674 // (v4i32). 675 class Crypto_SHA_1Hash_Intrinsic 676 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 677 678 // SHA intrinsic taking 8 words of the schedule 679 class Crypto_SHA_8Schedule_Intrinsic 680 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 681 682 // SHA intrinsic taking 12 words of the schedule 683 class Crypto_SHA_12Schedule_Intrinsic 684 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 685 [IntrNoMem]>; 686 687 // SHA intrinsic taking 8 words of the hash and 4 of the schedule. 688 class Crypto_SHA_8Hash4Schedule_Intrinsic 689 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 690 [IntrNoMem]>; 691} 692 693// AES 694def int_aarch64_crypto_aese : Crypto_AES_DataKey_Intrinsic; 695def int_aarch64_crypto_aesd : Crypto_AES_DataKey_Intrinsic; 696def int_aarch64_crypto_aesmc : Crypto_AES_Data_Intrinsic; 697def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic; 698 699// SHA1 700def int_aarch64_crypto_sha1c : Crypto_SHA_5Hash4Schedule_Intrinsic; 701def int_aarch64_crypto_sha1p : Crypto_SHA_5Hash4Schedule_Intrinsic; 702def int_aarch64_crypto_sha1m : Crypto_SHA_5Hash4Schedule_Intrinsic; 703def int_aarch64_crypto_sha1h : Crypto_SHA_1Hash_Intrinsic; 704 705def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic; 706def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic; 707 708// SHA256 709def int_aarch64_crypto_sha256h : Crypto_SHA_8Hash4Schedule_Intrinsic; 710def int_aarch64_crypto_sha256h2 : Crypto_SHA_8Hash4Schedule_Intrinsic; 711def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic; 712def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic; 713 714//===----------------------------------------------------------------------===// 715// CRC32 716 717let TargetPrefix = "aarch64" in { 718 719def int_aarch64_crc32b : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 720 [IntrNoMem]>; 721def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 722 [IntrNoMem]>; 723def int_aarch64_crc32h : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 724 [IntrNoMem]>; 725def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 726 [IntrNoMem]>; 727def int_aarch64_crc32w : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 728 [IntrNoMem]>; 729def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 730 [IntrNoMem]>; 731def int_aarch64_crc32x : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 732 [IntrNoMem]>; 733def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 734 [IntrNoMem]>; 735} 736 737//===----------------------------------------------------------------------===// 738// Memory Tagging Extensions (MTE) Intrinsics 739let TargetPrefix = "aarch64" in { 740def int_aarch64_irg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 741 [IntrNoMem, IntrHasSideEffects]>; 742def int_aarch64_addg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 743 [IntrNoMem]>; 744def int_aarch64_gmi : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], 745 [IntrNoMem]>; 746def int_aarch64_ldg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], 747 [IntrReadMem]>; 748def int_aarch64_stg : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], 749 [IntrWriteMem]>; 750def int_aarch64_subp : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], 751 [IntrNoMem]>; 752 753// The following are codegen-only intrinsics for stack instrumentation. 754 755// Generate a randomly tagged stack base pointer. 756def int_aarch64_irg_sp : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty], 757 [IntrNoMem, IntrHasSideEffects]>; 758 759// Transfer pointer tag with offset. 760// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where 761// * address is the address in ptr0 762// * tag is a function of (tag in baseptr, tag_offset). 763// Address bits in baseptr and tag bits in ptr0 are ignored. 764// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as 765// ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset 766// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp. 767def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty], 768 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 769 770// Update allocation tags for the memory range to match the tag in the pointer argument. 771def int_aarch64_settag : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 772 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 773 774// Update allocation tags for the memory range to match the tag in the pointer argument, 775// and set memory contents to zero. 776def int_aarch64_settag_zero : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 777 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 778 779// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values. 780def int_aarch64_stgp : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], 781 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 782} 783 784// Transactional Memory Extension (TME) Intrinsics 785let TargetPrefix = "aarch64" in { 786def int_aarch64_tstart : GCCBuiltin<"__builtin_arm_tstart">, 787 Intrinsic<[llvm_i64_ty]>; 788 789def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>; 790 791def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">, 792 Intrinsic<[], [llvm_i64_ty], [ImmArg<ArgIndex<0>>]>; 793 794def int_aarch64_ttest : GCCBuiltin<"__builtin_arm_ttest">, 795 Intrinsic<[llvm_i64_ty], [], 796 [IntrNoMem, IntrHasSideEffects]>; 797} 798 799def llvm_nxv2i1_ty : LLVMType<nxv2i1>; 800def llvm_nxv4i1_ty : LLVMType<nxv4i1>; 801def llvm_nxv8i1_ty : LLVMType<nxv8i1>; 802def llvm_nxv16i1_ty : LLVMType<nxv16i1>; 803def llvm_nxv16i8_ty : LLVMType<nxv16i8>; 804def llvm_nxv4i32_ty : LLVMType<nxv4i32>; 805def llvm_nxv2i64_ty : LLVMType<nxv2i64>; 806def llvm_nxv8f16_ty : LLVMType<nxv8f16>; 807def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>; 808def llvm_nxv4f32_ty : LLVMType<nxv4f32>; 809def llvm_nxv2f64_ty : LLVMType<nxv2f64>; 810 811let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 812 813 class AdvSIMD_SVE_Create_2Vector_Tuple 814 : Intrinsic<[llvm_anyvector_ty], 815 [llvm_anyvector_ty, LLVMMatchType<1>], 816 [IntrReadMem]>; 817 818 class AdvSIMD_SVE_Create_3Vector_Tuple 819 : Intrinsic<[llvm_anyvector_ty], 820 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>], 821 [IntrReadMem]>; 822 823 class AdvSIMD_SVE_Create_4Vector_Tuple 824 : Intrinsic<[llvm_anyvector_ty], 825 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, 826 LLVMMatchType<1>], 827 [IntrReadMem]>; 828 829 class AdvSIMD_SVE_Set_Vector_Tuple 830 : Intrinsic<[llvm_anyvector_ty], 831 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty], 832 [IntrReadMem, ImmArg<ArgIndex<1>>]>; 833 834 class AdvSIMD_SVE_Get_Vector_Tuple 835 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty], 836 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>; 837 838 class AdvSIMD_ManyVec_PredLoad_Intrinsic 839 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>], 840 [IntrReadMem, IntrArgMemOnly]>; 841 842 class AdvSIMD_1Vec_PredLoad_Intrinsic 843 : Intrinsic<[llvm_anyvector_ty], 844 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 845 LLVMPointerToElt<0>], 846 [IntrReadMem, IntrArgMemOnly]>; 847 848 class AdvSIMD_1Vec_PredStore_Intrinsic 849 : Intrinsic<[], 850 [llvm_anyvector_ty, 851 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 852 LLVMPointerToElt<0>], 853 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 854 855 class AdvSIMD_2Vec_PredStore_Intrinsic 856 : Intrinsic<[], 857 [llvm_anyvector_ty, LLVMMatchType<0>, 858 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 859 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 860 861 class AdvSIMD_3Vec_PredStore_Intrinsic 862 : Intrinsic<[], 863 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 864 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 865 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 866 867 class AdvSIMD_4Vec_PredStore_Intrinsic 868 : Intrinsic<[], 869 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 870 LLVMMatchType<0>, 871 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 872 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 873 874 class AdvSIMD_SVE_Index_Intrinsic 875 : Intrinsic<[llvm_anyvector_ty], 876 [LLVMVectorElementType<0>, 877 LLVMVectorElementType<0>], 878 [IntrNoMem]>; 879 880 class AdvSIMD_Merged1VectorArg_Intrinsic 881 : Intrinsic<[llvm_anyvector_ty], 882 [LLVMMatchType<0>, 883 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 884 LLVMMatchType<0>], 885 [IntrNoMem]>; 886 887 class AdvSIMD_2VectorArgIndexed_Intrinsic 888 : Intrinsic<[llvm_anyvector_ty], 889 [LLVMMatchType<0>, 890 LLVMMatchType<0>, 891 llvm_i32_ty], 892 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 893 894 class AdvSIMD_3VectorArgIndexed_Intrinsic 895 : Intrinsic<[llvm_anyvector_ty], 896 [LLVMMatchType<0>, 897 LLVMMatchType<0>, 898 LLVMMatchType<0>, 899 llvm_i32_ty], 900 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 901 902 class AdvSIMD_Pred1VectorArg_Intrinsic 903 : Intrinsic<[llvm_anyvector_ty], 904 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 905 LLVMMatchType<0>], 906 [IntrNoMem]>; 907 908 class AdvSIMD_Pred2VectorArg_Intrinsic 909 : Intrinsic<[llvm_anyvector_ty], 910 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 911 LLVMMatchType<0>, 912 LLVMMatchType<0>], 913 [IntrNoMem]>; 914 915 class AdvSIMD_Pred3VectorArg_Intrinsic 916 : Intrinsic<[llvm_anyvector_ty], 917 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 918 LLVMMatchType<0>, 919 LLVMMatchType<0>, 920 LLVMMatchType<0>], 921 [IntrNoMem]>; 922 923 class AdvSIMD_SVE_Compare_Intrinsic 924 : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 925 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 926 llvm_anyvector_ty, 927 LLVMMatchType<0>], 928 [IntrNoMem]>; 929 930 class AdvSIMD_SVE_CompareWide_Intrinsic 931 : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 932 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 933 llvm_anyvector_ty, 934 llvm_nxv2i64_ty], 935 [IntrNoMem]>; 936 937 class AdvSIMD_SVE_Saturating_Intrinsic 938 : Intrinsic<[llvm_anyvector_ty], 939 [LLVMMatchType<0>, 940 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 941 [IntrNoMem]>; 942 943 class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic 944 : Intrinsic<[llvm_anyvector_ty], 945 [LLVMMatchType<0>, 946 llvm_i32_ty, 947 llvm_i32_ty], 948 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 949 950 class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T> 951 : Intrinsic<[T], 952 [T, llvm_anyvector_ty], 953 [IntrNoMem]>; 954 955 class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T> 956 : Intrinsic<[T], 957 [T, llvm_i32_ty, llvm_i32_ty], 958 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 959 960 class AdvSIMD_SVE_CNT_Intrinsic 961 : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], 962 [LLVMVectorOfBitcastsToInt<0>, 963 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 964 llvm_anyvector_ty], 965 [IntrNoMem]>; 966 967 class AdvSIMD_SVE_ReduceWithInit_Intrinsic 968 : Intrinsic<[LLVMVectorElementType<0>], 969 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 970 LLVMVectorElementType<0>, 971 llvm_anyvector_ty], 972 [IntrNoMem]>; 973 974 class AdvSIMD_SVE_ShiftByImm_Intrinsic 975 : Intrinsic<[llvm_anyvector_ty], 976 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 977 LLVMMatchType<0>, 978 llvm_i32_ty], 979 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 980 981 class AdvSIMD_SVE_ShiftWide_Intrinsic 982 : Intrinsic<[llvm_anyvector_ty], 983 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 984 LLVMMatchType<0>, 985 llvm_nxv2i64_ty], 986 [IntrNoMem]>; 987 988 class AdvSIMD_SVE_Unpack_Intrinsic 989 : Intrinsic<[llvm_anyvector_ty], 990 [LLVMSubdivide2VectorType<0>], 991 [IntrNoMem]>; 992 993 class AdvSIMD_SVE_CADD_Intrinsic 994 : Intrinsic<[llvm_anyvector_ty], 995 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 996 LLVMMatchType<0>, 997 LLVMMatchType<0>, 998 llvm_i32_ty], 999 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1000 1001 class AdvSIMD_SVE_CMLA_Intrinsic 1002 : Intrinsic<[llvm_anyvector_ty], 1003 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1004 LLVMMatchType<0>, 1005 LLVMMatchType<0>, 1006 LLVMMatchType<0>, 1007 llvm_i32_ty], 1008 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 1009 1010 class AdvSIMD_SVE_CMLA_LANE_Intrinsic 1011 : Intrinsic<[llvm_anyvector_ty], 1012 [LLVMMatchType<0>, 1013 LLVMMatchType<0>, 1014 LLVMMatchType<0>, 1015 llvm_i32_ty, 1016 llvm_i32_ty], 1017 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1018 1019 class AdvSIMD_SVE_DUP_Intrinsic 1020 : Intrinsic<[llvm_anyvector_ty], 1021 [LLVMMatchType<0>, 1022 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1023 LLVMVectorElementType<0>], 1024 [IntrNoMem]>; 1025 1026 class AdvSIMD_SVE_DUP_Unpred_Intrinsic 1027 : Intrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>], 1028 [IntrNoMem]>; 1029 1030 class AdvSIMD_SVE_DUPQ_Intrinsic 1031 : Intrinsic<[llvm_anyvector_ty], 1032 [LLVMMatchType<0>, 1033 llvm_i64_ty], 1034 [IntrNoMem]>; 1035 1036 class AdvSIMD_SVE_EXPA_Intrinsic 1037 : Intrinsic<[llvm_anyvector_ty], 1038 [LLVMVectorOfBitcastsToInt<0>], 1039 [IntrNoMem]>; 1040 1041 class AdvSIMD_SVE_FCVT_Intrinsic 1042 : Intrinsic<[llvm_anyvector_ty], 1043 [LLVMMatchType<0>, 1044 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1045 llvm_anyvector_ty], 1046 [IntrNoMem]>; 1047 1048 class AdvSIMD_SVE_FCVTZS_Intrinsic 1049 : Intrinsic<[llvm_anyvector_ty], 1050 [LLVMVectorOfBitcastsToInt<0>, 1051 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1052 llvm_anyvector_ty], 1053 [IntrNoMem]>; 1054 1055 class AdvSIMD_SVE_INSR_Intrinsic 1056 : Intrinsic<[llvm_anyvector_ty], 1057 [LLVMMatchType<0>, 1058 LLVMVectorElementType<0>], 1059 [IntrNoMem]>; 1060 1061 class AdvSIMD_SVE_PTRUE_Intrinsic 1062 : Intrinsic<[llvm_anyvector_ty], 1063 [llvm_i32_ty], 1064 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1065 1066 class AdvSIMD_SVE_PUNPKHI_Intrinsic 1067 : Intrinsic<[LLVMHalfElementsVectorType<0>], 1068 [llvm_anyvector_ty], 1069 [IntrNoMem]>; 1070 1071 class AdvSIMD_SVE_SCALE_Intrinsic 1072 : Intrinsic<[llvm_anyvector_ty], 1073 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1074 LLVMMatchType<0>, 1075 LLVMVectorOfBitcastsToInt<0>], 1076 [IntrNoMem]>; 1077 1078 class AdvSIMD_SVE_SCVTF_Intrinsic 1079 : Intrinsic<[llvm_anyvector_ty], 1080 [LLVMMatchType<0>, 1081 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1082 llvm_anyvector_ty], 1083 [IntrNoMem]>; 1084 1085 class AdvSIMD_SVE_TSMUL_Intrinsic 1086 : Intrinsic<[llvm_anyvector_ty], 1087 [LLVMMatchType<0>, 1088 LLVMVectorOfBitcastsToInt<0>], 1089 [IntrNoMem]>; 1090 1091 class AdvSIMD_SVE_CNTB_Intrinsic 1092 : Intrinsic<[llvm_i64_ty], 1093 [llvm_i32_ty], 1094 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1095 1096 class AdvSIMD_SVE_CNTP_Intrinsic 1097 : Intrinsic<[llvm_i64_ty], 1098 [llvm_anyvector_ty, LLVMMatchType<0>], 1099 [IntrNoMem]>; 1100 1101 class AdvSIMD_SVE_DOT_Intrinsic 1102 : Intrinsic<[llvm_anyvector_ty], 1103 [LLVMMatchType<0>, 1104 LLVMSubdivide4VectorType<0>, 1105 LLVMSubdivide4VectorType<0>], 1106 [IntrNoMem]>; 1107 1108 class AdvSIMD_SVE_DOT_Indexed_Intrinsic 1109 : Intrinsic<[llvm_anyvector_ty], 1110 [LLVMMatchType<0>, 1111 LLVMSubdivide4VectorType<0>, 1112 LLVMSubdivide4VectorType<0>, 1113 llvm_i32_ty], 1114 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1115 1116 class AdvSIMD_SVE_PTEST_Intrinsic 1117 : Intrinsic<[llvm_i1_ty], 1118 [llvm_anyvector_ty, 1119 LLVMMatchType<0>], 1120 [IntrNoMem]>; 1121 1122 class AdvSIMD_SVE_TBL_Intrinsic 1123 : Intrinsic<[llvm_anyvector_ty], 1124 [LLVMMatchType<0>, 1125 LLVMVectorOfBitcastsToInt<0>], 1126 [IntrNoMem]>; 1127 1128 class AdvSIMD_SVE2_TBX_Intrinsic 1129 : Intrinsic<[llvm_anyvector_ty], 1130 [LLVMMatchType<0>, 1131 LLVMMatchType<0>, 1132 LLVMVectorOfBitcastsToInt<0>], 1133 [IntrNoMem]>; 1134 1135 class SVE2_1VectorArg_Long_Intrinsic 1136 : Intrinsic<[llvm_anyvector_ty], 1137 [LLVMSubdivide2VectorType<0>, 1138 llvm_i32_ty], 1139 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1140 1141 class SVE2_2VectorArg_Long_Intrinsic 1142 : Intrinsic<[llvm_anyvector_ty], 1143 [LLVMSubdivide2VectorType<0>, 1144 LLVMSubdivide2VectorType<0>], 1145 [IntrNoMem]>; 1146 1147 class SVE2_2VectorArgIndexed_Long_Intrinsic 1148 : Intrinsic<[llvm_anyvector_ty], 1149 [LLVMSubdivide2VectorType<0>, 1150 LLVMSubdivide2VectorType<0>, 1151 llvm_i32_ty], 1152 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1153 1154 class SVE2_2VectorArg_Wide_Intrinsic 1155 : Intrinsic<[llvm_anyvector_ty], 1156 [LLVMMatchType<0>, 1157 LLVMSubdivide2VectorType<0>], 1158 [IntrNoMem]>; 1159 1160 class SVE2_2VectorArg_Pred_Long_Intrinsic 1161 : Intrinsic<[llvm_anyvector_ty], 1162 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1163 LLVMMatchType<0>, 1164 LLVMSubdivide2VectorType<0>], 1165 [IntrNoMem]>; 1166 1167 class SVE2_3VectorArg_Long_Intrinsic 1168 : Intrinsic<[llvm_anyvector_ty], 1169 [LLVMMatchType<0>, 1170 LLVMSubdivide2VectorType<0>, 1171 LLVMSubdivide2VectorType<0>], 1172 [IntrNoMem]>; 1173 1174 class SVE2_3VectorArgIndexed_Long_Intrinsic 1175 : Intrinsic<[llvm_anyvector_ty], 1176 [LLVMMatchType<0>, 1177 LLVMSubdivide2VectorType<0>, 1178 LLVMSubdivide2VectorType<0>, 1179 llvm_i32_ty], 1180 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1181 1182 class SVE2_1VectorArg_Narrowing_Intrinsic 1183 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1184 [llvm_anyvector_ty], 1185 [IntrNoMem]>; 1186 1187 class SVE2_Merged1VectorArg_Narrowing_Intrinsic 1188 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1189 [LLVMSubdivide2VectorType<0>, 1190 llvm_anyvector_ty], 1191 [IntrNoMem]>; 1192 class SVE2_2VectorArg_Narrowing_Intrinsic 1193 : Intrinsic< 1194 [LLVMSubdivide2VectorType<0>], 1195 [llvm_anyvector_ty, LLVMMatchType<0>], 1196 [IntrNoMem]>; 1197 1198 class SVE2_Merged2VectorArg_Narrowing_Intrinsic 1199 : Intrinsic< 1200 [LLVMSubdivide2VectorType<0>], 1201 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>], 1202 [IntrNoMem]>; 1203 1204 class SVE2_1VectorArg_Imm_Narrowing_Intrinsic 1205 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1206 [llvm_anyvector_ty, llvm_i32_ty], 1207 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1208 1209 class SVE2_2VectorArg_Imm_Narrowing_Intrinsic 1210 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1211 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, 1212 llvm_i32_ty], 1213 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1214 1215 class SVE2_CONFLICT_DETECT_Intrinsic 1216 : Intrinsic<[llvm_anyvector_ty], 1217 [LLVMAnyPointerType<llvm_any_ty>, 1218 LLVMMatchType<1>]>; 1219 1220 class SVE2_3VectorArg_Indexed_Intrinsic 1221 : Intrinsic<[llvm_anyvector_ty], 1222 [LLVMMatchType<0>, 1223 LLVMSubdivide2VectorType<0>, 1224 LLVMSubdivide2VectorType<0>, 1225 llvm_i32_ty], 1226 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1227 1228 class AdvSIMD_SVE_CDOT_LANE_Intrinsic 1229 : Intrinsic<[llvm_anyvector_ty], 1230 [LLVMMatchType<0>, 1231 LLVMSubdivide4VectorType<0>, 1232 LLVMSubdivide4VectorType<0>, 1233 llvm_i32_ty, 1234 llvm_i32_ty], 1235 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1236 1237 // NOTE: There is no relationship between these intrinsics beyond an attempt 1238 // to reuse currently identical class definitions. 1239 class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; 1240 class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic; 1241 class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic; 1242 1243 // This class of intrinsics are not intended to be useful within LLVM IR but 1244 // are instead here to support some of the more regid parts of the ACLE. 1245 class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN> 1246 : Intrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>; 1247} 1248 1249//===----------------------------------------------------------------------===// 1250// SVE 1251 1252let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 1253 1254class AdvSIMD_SVE_Reduce_Intrinsic 1255 : Intrinsic<[LLVMVectorElementType<0>], 1256 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1257 llvm_anyvector_ty], 1258 [IntrNoMem]>; 1259 1260class AdvSIMD_SVE_SADDV_Reduce_Intrinsic 1261 : Intrinsic<[llvm_i64_ty], 1262 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1263 llvm_anyvector_ty], 1264 [IntrNoMem]>; 1265 1266class AdvSIMD_SVE_WHILE_Intrinsic 1267 : Intrinsic<[llvm_anyvector_ty], 1268 [llvm_anyint_ty, LLVMMatchType<1>], 1269 [IntrNoMem]>; 1270 1271class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic 1272 : Intrinsic<[llvm_anyvector_ty], 1273 [ 1274 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1275 LLVMPointerToElt<0>, 1276 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1277 ], 1278 [IntrReadMem, IntrArgMemOnly]>; 1279 1280class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic 1281 : Intrinsic<[llvm_anyvector_ty], 1282 [ 1283 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1284 LLVMPointerToElt<0>, 1285 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1286 ], 1287 [IntrReadMem, IntrArgMemOnly]>; 1288 1289class AdvSIMD_GatherLoad_VS_Intrinsic 1290 : Intrinsic<[llvm_anyvector_ty], 1291 [ 1292 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1293 llvm_anyvector_ty, 1294 llvm_i64_ty 1295 ], 1296 [IntrReadMem, IntrArgMemOnly]>; 1297 1298class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic 1299 : Intrinsic<[], 1300 [ 1301 llvm_anyvector_ty, 1302 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1303 LLVMPointerToElt<0>, 1304 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1305 ], 1306 [IntrWriteMem, IntrArgMemOnly]>; 1307 1308class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic 1309 : Intrinsic<[], 1310 [ 1311 llvm_anyvector_ty, 1312 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1313 LLVMPointerToElt<0>, 1314 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1315 ], 1316 [IntrWriteMem, IntrArgMemOnly]>; 1317 1318class AdvSIMD_ScatterStore_VS_Intrinsic 1319 : Intrinsic<[], 1320 [ 1321 llvm_anyvector_ty, 1322 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1323 llvm_anyvector_ty, llvm_i64_ty 1324 ], 1325 [IntrWriteMem, IntrArgMemOnly]>; 1326 1327 1328class SVE_gather_prf_SV 1329 : Intrinsic<[], 1330 [ 1331 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1332 llvm_ptr_ty, // Base address 1333 llvm_anyvector_ty, // Offsets 1334 llvm_i32_ty // Prfop 1335 ], 1336 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>; 1337 1338class SVE_gather_prf_VS 1339 : Intrinsic<[], 1340 [ 1341 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1342 llvm_anyvector_ty, // Base addresses 1343 llvm_i64_ty, // Scalar offset 1344 llvm_i32_ty // Prfop 1345 ], 1346 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>; 1347 1348class SVE_MatMul_Intrinsic 1349 : Intrinsic<[llvm_anyvector_ty], 1350 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], 1351 [IntrNoMem]>; 1352 1353class SVE_4Vec_BF16 1354 : Intrinsic<[llvm_nxv4f32_ty], 1355 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 1356 [IntrNoMem]>; 1357 1358class SVE_4Vec_BF16_Indexed 1359 : Intrinsic<[llvm_nxv4f32_ty], 1360 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty], 1361 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1362 1363// 1364// Vector tuple creation intrinsics (ACLE) 1365// 1366 1367def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple; 1368def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple; 1369def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple; 1370 1371// 1372// Vector tuple insertion/extraction intrinsics (ACLE) 1373// 1374 1375def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple; 1376def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple; 1377 1378// 1379// Loads 1380// 1381 1382def int_aarch64_sve_ld1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1383 1384def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1385def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1386def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1387 1388def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1389def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1390def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1391 1392def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic; 1393def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic; 1394 1395// 1396// Stores 1397// 1398 1399def int_aarch64_sve_st1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1400def int_aarch64_sve_st2 : AdvSIMD_2Vec_PredStore_Intrinsic; 1401def int_aarch64_sve_st3 : AdvSIMD_3Vec_PredStore_Intrinsic; 1402def int_aarch64_sve_st4 : AdvSIMD_4Vec_PredStore_Intrinsic; 1403 1404def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1405 1406// 1407// Prefetches 1408// 1409 1410def int_aarch64_sve_prf 1411 : Intrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty], 1412 [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>; 1413 1414// Scalar + 32-bit scaled offset vector, zero extend, packed and 1415// unpacked. 1416def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV; 1417def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV; 1418def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV; 1419def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV; 1420 1421// Scalar + 32-bit scaled offset vector, sign extend, packed and 1422// unpacked. 1423def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV; 1424def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV; 1425def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV; 1426def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV; 1427 1428// Scalar + 64-bit scaled offset vector. 1429def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV; 1430def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV; 1431def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV; 1432def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV; 1433 1434// Vector + scalar. 1435def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS; 1436def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS; 1437def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS; 1438def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS; 1439 1440// 1441// Scalar to vector operations 1442// 1443 1444def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic; 1445def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic; 1446 1447 1448def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic; 1449 1450// 1451// Address calculation 1452// 1453 1454def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic; 1455def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic; 1456def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic; 1457def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic; 1458 1459// 1460// Integer arithmetic 1461// 1462 1463def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic; 1464def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; 1465def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic; 1466 1467def int_aarch64_sve_pmul : AdvSIMD_2VectorArg_Intrinsic; 1468 1469def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic; 1470def int_aarch64_sve_mul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1471def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1472def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1473 1474def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1475def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1476def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1477def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1478 1479def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic; 1480def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic; 1481def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic; 1482def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic; 1483def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1484def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1485 1486def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; 1487def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; 1488def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; 1489def int_aarch64_sve_mla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1490def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; 1491def int_aarch64_sve_mls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1492 1493def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1494def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1495 1496def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1497def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1498def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic; 1499def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic; 1500 1501def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic; 1502def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic; 1503def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic; 1504 1505def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; 1506def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; 1507 1508def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic; 1509def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1510 1511def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; 1512def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1513 1514def int_aarch64_sve_sqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1515def int_aarch64_sve_sqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1516def int_aarch64_sve_uqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1517def int_aarch64_sve_uqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1518 1519// Shifts 1520 1521def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic; 1522def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1523def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic; 1524def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic; 1525def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic; 1526def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1527def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic; 1528def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1529 1530// 1531// Integer comparisons 1532// 1533 1534def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1535def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic; 1536def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1537def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic; 1538def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic; 1539def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic; 1540 1541def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1542def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1543def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1544def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1545def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1546def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1547def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1548def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1549def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1550def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1551 1552// 1553// Counting bits 1554// 1555 1556def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic; 1557def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic; 1558def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; 1559 1560// 1561// Counting elements 1562// 1563 1564def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic; 1565def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic; 1566def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic; 1567def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic; 1568 1569def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; 1570 1571// 1572// FFR manipulation 1573// 1574 1575def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, Intrinsic<[llvm_nxv16i1_ty], []>; 1576def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>; 1577def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, Intrinsic<[], []>; 1578def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, Intrinsic<[], [llvm_nxv16i1_ty]>; 1579 1580// 1581// Saturating scalar arithmetic 1582// 1583 1584def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1585def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1586def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1587def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1588 1589def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1590def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1591def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1592def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1593def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1594def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1595def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1596def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1597def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1598def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1599 1600def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1601def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1602def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1603def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1604 1605def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1606def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1607def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1608def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1609def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1610def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1611def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1612def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1613def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1614def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1615 1616def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1617def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1618def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1619def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1620 1621def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1622def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1623def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1624def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1625def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1626def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1627def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1628def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1629def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1630def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1631 1632def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1633def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1634def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1635def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1636 1637def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1638def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1639def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1640def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1641def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1642def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1643def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1644def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1645def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1646def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1647 1648// 1649// Reversal 1650// 1651 1652def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic; 1653def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic; 1654def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic; 1655def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic; 1656 1657// 1658// Permutations and selection 1659// 1660 1661def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic; 1662def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1663def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic; 1664def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1665def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic; 1666def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic; 1667def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic; 1668def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic; 1669def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; 1670def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; 1671def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; 1672def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; 1673def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1674def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1675def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; 1676def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; 1677def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; 1678def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic; 1679def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic; 1680def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1681def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1682def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; 1683def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; 1684def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic; 1685def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic; 1686def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; 1687def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; 1688def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic; 1689def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic; 1690 1691// 1692// Logical operations 1693// 1694 1695def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic; 1696def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic; 1697def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic; 1698def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic; 1699def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic; 1700def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic; 1701 1702// 1703// Conversion 1704// 1705 1706def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1707def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1708def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1709def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1710def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1711def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1712 1713// 1714// While comparisons 1715// 1716 1717def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic; 1718def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic; 1719def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic; 1720def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic; 1721def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic; 1722def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic; 1723def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic; 1724def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic; 1725 1726// 1727// Floating-point arithmetic 1728// 1729 1730def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1731def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic; 1732def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic; 1733def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic; 1734def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic; 1735def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 1736def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1737def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1738def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic; 1739def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic; 1740def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic; 1741def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic; 1742def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic; 1743def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic; 1744def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic; 1745def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1746def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic; 1747def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1748def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 1749def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic; 1750def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic; 1751def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic; 1752def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1753def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic; 1754def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic; 1755def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic; 1756def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 1757def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic; 1758def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic; 1759def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic; 1760def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic; 1761def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic; 1762def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic; 1763def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic; 1764def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic; 1765def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic; 1766def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic; 1767def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic; 1768def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic; 1769def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic; 1770def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic; 1771def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic; 1772def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 1773def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic; 1774def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic; 1775def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic; 1776 1777// 1778// Floating-point reductions 1779// 1780 1781def int_aarch64_sve_fadda : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1782def int_aarch64_sve_faddv : AdvSIMD_SVE_Reduce_Intrinsic; 1783def int_aarch64_sve_fmaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1784def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic; 1785def int_aarch64_sve_fminv : AdvSIMD_SVE_Reduce_Intrinsic; 1786def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic; 1787 1788// 1789// Floating-point conversions 1790// 1791 1792def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic; 1793def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic; 1794def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic; 1795def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 1796def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 1797 1798// 1799// Floating-point comparisons 1800// 1801 1802def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic; 1803def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic; 1804 1805def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1806def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic; 1807def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1808def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic; 1809def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic; 1810 1811def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1812def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1813def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 1814def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1815 1816def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<"svcvt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 1817def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 1818 1819def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1820def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1821def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 1822def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1823 1824def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 1825def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1826def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1827 1828def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1829def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 1830def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1831 1832def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1833def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1834def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 1835def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1836 1837def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1838def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1839 1840def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 1841def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1842def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1843def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 1844 1845def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 1846def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1847def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1848def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 1849 1850// 1851// Predicate creation 1852// 1853 1854def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic; 1855 1856// 1857// Predicate operations 1858// 1859 1860def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1861def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1862def int_aarch64_sve_brka : AdvSIMD_Merged1VectorArg_Intrinsic; 1863def int_aarch64_sve_brka_z : AdvSIMD_Pred1VectorArg_Intrinsic; 1864def int_aarch64_sve_brkb : AdvSIMD_Merged1VectorArg_Intrinsic; 1865def int_aarch64_sve_brkb_z : AdvSIMD_Pred1VectorArg_Intrinsic; 1866def int_aarch64_sve_brkn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1867def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1868def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1869def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1870def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1871def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1872def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1873def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1874def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic; 1875def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic; 1876def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; 1877def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; 1878 1879// 1880// Testing predicates 1881// 1882 1883def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic; 1884def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic; 1885def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic; 1886 1887// 1888// Reinterpreting data 1889// 1890 1891def int_aarch64_sve_convert_from_svbool : Intrinsic<[llvm_anyvector_ty], 1892 [llvm_nxv16i1_ty], 1893 [IntrNoMem]>; 1894 1895def int_aarch64_sve_convert_to_svbool : Intrinsic<[llvm_nxv16i1_ty], 1896 [llvm_anyvector_ty], 1897 [IntrNoMem]>; 1898 1899// 1900// Gather loads: scalar base + vector offsets 1901// 1902 1903// 64 bit unscaled offsets 1904def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1905 1906// 64 bit scaled offsets 1907def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1908 1909// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1910def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1911def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1912 1913// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1914def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1915def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1916 1917// 1918// Gather loads: vector base + scalar offset 1919// 1920 1921def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 1922 1923 1924// 1925// First-faulting gather loads: scalar base + vector offsets 1926// 1927 1928// 64 bit unscaled offsets 1929def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1930 1931// 64 bit scaled offsets 1932def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1933 1934// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 1935def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1936def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1937 1938// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 1939def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1940def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1941 1942// 1943// First-faulting gather loads: vector base + scalar offset 1944// 1945 1946def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 1947 1948 1949// 1950// Non-temporal gather loads: scalar base + vector offsets 1951// 1952 1953// 64 bit unscaled offsets 1954def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1955 1956// 64 bit indices 1957def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1958 1959// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 1960def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1961 1962// 1963// Non-temporal gather loads: vector base + scalar offset 1964// 1965 1966def int_aarch64_sve_ldnt1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 1967 1968// 1969// Scatter stores: scalar base + vector offsets 1970// 1971 1972// 64 bit unscaled offsets 1973def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 1974 1975// 64 bit scaled offsets 1976def int_aarch64_sve_st1_scatter_index 1977 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 1978 1979// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1980def int_aarch64_sve_st1_scatter_sxtw 1981 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 1982 1983def int_aarch64_sve_st1_scatter_uxtw 1984 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 1985 1986// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1987def int_aarch64_sve_st1_scatter_sxtw_index 1988 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 1989 1990def int_aarch64_sve_st1_scatter_uxtw_index 1991 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 1992 1993// 1994// Scatter stores: vector base + scalar offset 1995// 1996 1997def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 1998 1999// 2000// Non-temporal scatter stores: scalar base + vector offsets 2001// 2002 2003// 64 bit unscaled offsets 2004def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2005 2006// 64 bit indices 2007def int_aarch64_sve_stnt1_scatter_index 2008 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2009 2010// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2011def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2012 2013// 2014// Non-temporal scatter stores: vector base + scalar offset 2015// 2016 2017def int_aarch64_sve_stnt1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2018 2019// 2020// SVE2 - Uniform DSP operations 2021// 2022 2023def int_aarch64_sve_saba : AdvSIMD_3VectorArg_Intrinsic; 2024def int_aarch64_sve_shadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2025def int_aarch64_sve_shsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2026def int_aarch64_sve_shsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2027def int_aarch64_sve_sli : AdvSIMD_2VectorArgIndexed_Intrinsic; 2028def int_aarch64_sve_sqabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2029def int_aarch64_sve_sqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2030def int_aarch64_sve_sqdmulh : AdvSIMD_2VectorArg_Intrinsic; 2031def int_aarch64_sve_sqdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2032def int_aarch64_sve_sqneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2033def int_aarch64_sve_sqrdmlah : AdvSIMD_3VectorArg_Intrinsic; 2034def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2035def int_aarch64_sve_sqrdmlsh : AdvSIMD_3VectorArg_Intrinsic; 2036def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2037def int_aarch64_sve_sqrdmulh : AdvSIMD_2VectorArg_Intrinsic; 2038def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2039def int_aarch64_sve_sqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2040def int_aarch64_sve_sqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2041def int_aarch64_sve_sqshlu : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2042def int_aarch64_sve_sqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2043def int_aarch64_sve_sqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2044def int_aarch64_sve_srhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2045def int_aarch64_sve_sri : AdvSIMD_2VectorArgIndexed_Intrinsic; 2046def int_aarch64_sve_srshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2047def int_aarch64_sve_srshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2048def int_aarch64_sve_srsra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2049def int_aarch64_sve_ssra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2050def int_aarch64_sve_suqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2051def int_aarch64_sve_uaba : AdvSIMD_3VectorArg_Intrinsic; 2052def int_aarch64_sve_uhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2053def int_aarch64_sve_uhsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2054def int_aarch64_sve_uhsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2055def int_aarch64_sve_uqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2056def int_aarch64_sve_uqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2057def int_aarch64_sve_uqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2058def int_aarch64_sve_uqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2059def int_aarch64_sve_uqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2060def int_aarch64_sve_urecpe : AdvSIMD_Merged1VectorArg_Intrinsic; 2061def int_aarch64_sve_urhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2062def int_aarch64_sve_urshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2063def int_aarch64_sve_urshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2064def int_aarch64_sve_ursqrte : AdvSIMD_Merged1VectorArg_Intrinsic; 2065def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2066def int_aarch64_sve_usqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2067def int_aarch64_sve_usra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2068 2069// 2070// SVE2 - Widening DSP operations 2071// 2072 2073def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic; 2074def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic; 2075def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic; 2076def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic; 2077def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic; 2078def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic; 2079def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic; 2080def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic; 2081def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic; 2082def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic; 2083def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic; 2084def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic; 2085def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic; 2086def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic; 2087def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic; 2088def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic; 2089def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic; 2090def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic; 2091def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic; 2092def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic; 2093def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic; 2094def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic; 2095def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic; 2096def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic; 2097def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic; 2098def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic; 2099def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic; 2100def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic; 2101 2102// 2103// SVE2 - Non-widening pairwise arithmetic 2104// 2105 2106def int_aarch64_sve_addp : AdvSIMD_Pred2VectorArg_Intrinsic; 2107def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic; 2108def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2109def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2110def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2111def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2112def int_aarch64_sve_smaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2113def int_aarch64_sve_sminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2114def int_aarch64_sve_umaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2115def int_aarch64_sve_uminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2116 2117// 2118// SVE2 - Widening pairwise arithmetic 2119// 2120 2121def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2122def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2123 2124// 2125// SVE2 - Uniform complex integer arithmetic 2126// 2127 2128def int_aarch64_sve_cadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2129def int_aarch64_sve_sqcadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2130def int_aarch64_sve_cmla_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2131def int_aarch64_sve_cmla_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2132def int_aarch64_sve_sqrdcmlah_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2133def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2134 2135// 2136// SVE2 - Widening complex integer arithmetic 2137// 2138 2139def int_aarch64_sve_saddlbt : SVE2_2VectorArg_Long_Intrinsic; 2140def int_aarch64_sve_ssublbt : SVE2_2VectorArg_Long_Intrinsic; 2141def int_aarch64_sve_ssubltb : SVE2_2VectorArg_Long_Intrinsic; 2142 2143// 2144// SVE2 - Widening complex integer dot product 2145// 2146 2147def int_aarch64_sve_cdot : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2148def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic; 2149 2150// 2151// SVE2 - Floating-point widening multiply-accumulate 2152// 2153 2154def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic; 2155def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2156def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic; 2157def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2158def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic; 2159def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2160def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic; 2161def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2162 2163// 2164// SVE2 - Floating-point integer binary logarithm 2165// 2166 2167def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic; 2168 2169// 2170// SVE2 - Vector histogram count 2171// 2172 2173def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic; 2174def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic; 2175 2176// 2177// SVE2 - Character match 2178// 2179 2180def int_aarch64_sve_match : AdvSIMD_SVE_Compare_Intrinsic; 2181def int_aarch64_sve_nmatch : AdvSIMD_SVE_Compare_Intrinsic; 2182 2183// 2184// SVE2 - Unary narrowing operations 2185// 2186 2187def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2188def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2189def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic; 2190def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2191def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2192def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2193 2194// 2195// SVE2 - Binary narrowing DSP operations 2196// 2197def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2198def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2199 2200def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2201def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2202 2203def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2204def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2205 2206def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2207def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2208 2209// Narrowing shift right 2210def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2211def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2212 2213def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2214def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2215 2216// Saturating shift right - signed input/output 2217def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2218def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2219 2220def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2221def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2222 2223// Saturating shift right - unsigned input/output 2224def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2225def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2226 2227def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2228def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2229 2230// Saturating shift right - signed input, unsigned output 2231def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2232def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2233 2234def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2235def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2236 2237// SVE2 MLA LANE. 2238def int_aarch64_sve_smlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2239def int_aarch64_sve_smlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2240def int_aarch64_sve_umlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2241def int_aarch64_sve_umlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2242def int_aarch64_sve_smlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2243def int_aarch64_sve_smlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2244def int_aarch64_sve_umlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2245def int_aarch64_sve_umlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2246def int_aarch64_sve_smullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2247def int_aarch64_sve_smullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2248def int_aarch64_sve_umullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2249def int_aarch64_sve_umullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2250def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2251def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2252def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2253def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2254def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2255def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2256 2257// SVE2 MLA Unpredicated. 2258def int_aarch64_sve_smlalb : SVE2_3VectorArg_Long_Intrinsic; 2259def int_aarch64_sve_smlalt : SVE2_3VectorArg_Long_Intrinsic; 2260def int_aarch64_sve_umlalb : SVE2_3VectorArg_Long_Intrinsic; 2261def int_aarch64_sve_umlalt : SVE2_3VectorArg_Long_Intrinsic; 2262def int_aarch64_sve_smlslb : SVE2_3VectorArg_Long_Intrinsic; 2263def int_aarch64_sve_smlslt : SVE2_3VectorArg_Long_Intrinsic; 2264def int_aarch64_sve_umlslb : SVE2_3VectorArg_Long_Intrinsic; 2265def int_aarch64_sve_umlslt : SVE2_3VectorArg_Long_Intrinsic; 2266def int_aarch64_sve_smullb : SVE2_2VectorArg_Long_Intrinsic; 2267def int_aarch64_sve_smullt : SVE2_2VectorArg_Long_Intrinsic; 2268def int_aarch64_sve_umullb : SVE2_2VectorArg_Long_Intrinsic; 2269def int_aarch64_sve_umullt : SVE2_2VectorArg_Long_Intrinsic; 2270 2271def int_aarch64_sve_sqdmlalb : SVE2_3VectorArg_Long_Intrinsic; 2272def int_aarch64_sve_sqdmlalt : SVE2_3VectorArg_Long_Intrinsic; 2273def int_aarch64_sve_sqdmlslb : SVE2_3VectorArg_Long_Intrinsic; 2274def int_aarch64_sve_sqdmlslt : SVE2_3VectorArg_Long_Intrinsic; 2275def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic; 2276def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic; 2277def int_aarch64_sve_sqdmlalbt : SVE2_3VectorArg_Long_Intrinsic; 2278def int_aarch64_sve_sqdmlslbt : SVE2_3VectorArg_Long_Intrinsic; 2279 2280// SVE2 ADDSUB Long Unpredicated. 2281def int_aarch64_sve_adclb : AdvSIMD_3VectorArg_Intrinsic; 2282def int_aarch64_sve_adclt : AdvSIMD_3VectorArg_Intrinsic; 2283def int_aarch64_sve_sbclb : AdvSIMD_3VectorArg_Intrinsic; 2284def int_aarch64_sve_sbclt : AdvSIMD_3VectorArg_Intrinsic; 2285 2286// 2287// SVE2 - Polynomial arithmetic 2288// 2289def int_aarch64_sve_eorbt : AdvSIMD_3VectorArg_Intrinsic; 2290def int_aarch64_sve_eortb : AdvSIMD_3VectorArg_Intrinsic; 2291def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic; 2292def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic; 2293 2294// 2295// SVE2 bitwise ternary operations. 2296// 2297def int_aarch64_sve_eor3 : AdvSIMD_3VectorArg_Intrinsic; 2298def int_aarch64_sve_bcax : AdvSIMD_3VectorArg_Intrinsic; 2299def int_aarch64_sve_bsl : AdvSIMD_3VectorArg_Intrinsic; 2300def int_aarch64_sve_bsl1n : AdvSIMD_3VectorArg_Intrinsic; 2301def int_aarch64_sve_bsl2n : AdvSIMD_3VectorArg_Intrinsic; 2302def int_aarch64_sve_nbsl : AdvSIMD_3VectorArg_Intrinsic; 2303def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic; 2304 2305// 2306// SVE2 - Optional AES, SHA-3 and SM4 2307// 2308 2309def int_aarch64_sve_aesd : GCCBuiltin<"__builtin_sve_svaesd_u8">, 2310 Intrinsic<[llvm_nxv16i8_ty], 2311 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2312 [IntrNoMem]>; 2313def int_aarch64_sve_aesimc : GCCBuiltin<"__builtin_sve_svaesimc_u8">, 2314 Intrinsic<[llvm_nxv16i8_ty], 2315 [llvm_nxv16i8_ty], 2316 [IntrNoMem]>; 2317def int_aarch64_sve_aese : GCCBuiltin<"__builtin_sve_svaese_u8">, 2318 Intrinsic<[llvm_nxv16i8_ty], 2319 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2320 [IntrNoMem]>; 2321def int_aarch64_sve_aesmc : GCCBuiltin<"__builtin_sve_svaesmc_u8">, 2322 Intrinsic<[llvm_nxv16i8_ty], 2323 [llvm_nxv16i8_ty], 2324 [IntrNoMem]>; 2325def int_aarch64_sve_rax1 : GCCBuiltin<"__builtin_sve_svrax1_u64">, 2326 Intrinsic<[llvm_nxv2i64_ty], 2327 [llvm_nxv2i64_ty, llvm_nxv2i64_ty], 2328 [IntrNoMem]>; 2329def int_aarch64_sve_sm4e : GCCBuiltin<"__builtin_sve_svsm4e_u32">, 2330 Intrinsic<[llvm_nxv4i32_ty], 2331 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2332 [IntrNoMem]>; 2333def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">, 2334 Intrinsic<[llvm_nxv4i32_ty], 2335 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2336 [IntrNoMem]>; 2337// 2338// SVE2 - Extended table lookup/permute 2339// 2340 2341def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic; 2342def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic; 2343 2344// 2345// SVE2 - Optional bit permutation 2346// 2347 2348def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic; 2349def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic; 2350def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic; 2351 2352 2353// 2354// SVE ACLE: 7.3. INT8 matrix multiply extensions 2355// 2356def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic; 2357def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic; 2358def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic; 2359 2360def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic; 2361def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2362def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2363 2364// 2365// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions 2366// 2367def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; 2368 2369// 2370// SVE ACLE: 7.2. BFloat16 extensions 2371// 2372 2373def int_aarch64_sve_bfdot : SVE_4Vec_BF16; 2374def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16; 2375def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16; 2376 2377def int_aarch64_sve_bfmmla : SVE_4Vec_BF16; 2378 2379def int_aarch64_sve_bfdot_lane : SVE_4Vec_BF16_Indexed; 2380def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed; 2381def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed; 2382} 2383 2384// 2385// SVE2 - Contiguous conflict detection 2386// 2387 2388def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic; 2389def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic; 2390def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic; 2391def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic; 2392def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic; 2393def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic; 2394def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic; 2395def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic; 2396