1//===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the set of ACLE-specified source-level intrinsic 10// functions wrapping the MVE vector instruction set and scalar shift 11// operations. 12// 13// Refer to comments in arm_mve_defs.td for the infrastructure used in 14// here, and to MveEmitter.cpp for how those are used in turn to 15// generate code. 16// 17//===----------------------------------------------------------------------===// 18 19include "arm_mve_defs.td" 20 21let params = T.Usual in 22foreach n = [ 2, 4 ] in { 23 def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>), 24 (CustomCodegen<"VST24"> n:$NumVectors, 25 "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>; 26 def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>), 27 (CustomCodegen<"VLD24"> n:$NumVectors, 28 "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>; 29} 30 31multiclass bit_op_fp<IRBuilder bitop> { 32def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), 33 (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>; 34} 35 36multiclass bit_op_fp_with_inv<IRBuilder bitop> { 37def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), 38 (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>; 39} 40 41let params = T.Signed in { 42def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 43 (IRIntBase<"sadd_sat", [Vector]> $a, $b)>; 44def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 45 (IRIntBase<"ssub_sat", [Vector]> $a, $b)>; 46} 47let params = T.Unsigned in { 48def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b), 49 (IRIntBase<"uadd_sat", [Vector]> $a, $b)>, 50 NameOverride<"vqaddq">; 51def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b), 52 (IRIntBase<"usub_sat", [Vector]> $a, $b)>, 53 NameOverride<"vqsubq">; 54} 55 56// Some intrinsics below are implemented not as IR fragments, but as 57// special-purpose IR intrinsics. This is because such a general form 58// (such as NEON uses) required a variable-width vector size, and we are 59// restricted to 128 bit. Although we can possibly get clever with lane 60// operations, the consequent IR representation would be very hard to 61// write sensibly. In particular, doubling a vector's width would be a 62// mess. Other intrinsics just don't translate nicely into IR. 63let params = T.Int in { 64def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>; 65def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 66 (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>; 67def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 68 (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>; 69def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>; 70def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>; 71def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>; 72def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>; 73def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>; 74def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>; 75def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 76 (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>; 77def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>; 78def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 79 (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>; 80def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 81 (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>; 82def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 83 (IRInt<"vmull", [DblVector, Vector]> 84 $a, $b, (unsignedflag Scalar), 0)>; 85def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 86 (IRInt<"vmull", [DblVector, Vector]> 87 $a, $b, (unsignedflag Scalar), 1)>; 88} 89let params = T.Signed in { 90def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 91 (IRInt<"vqdmulh", [Vector]> $a, $b)>; 92def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 93 (IRInt<"vqrdmulh", [Vector]> $a, $b)>; 94} 95 96let params = T.Poly, overrideKindLetter = "p" in { 97def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 98 (IRInt<"vmull_poly", [DblVector, Vector]> 99 $a, $b, 0)>; 100def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 101 (IRInt<"vmull_poly", [DblVector, Vector]> 102 $a, $b, 1)>; 103} 104 105let params = T.Float in { 106def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>, 107 NameOverride<"vaddq">; 108defm vandqf: bit_op_fp<and>, NameOverride<"vandq">; 109defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">; 110defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">; 111defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">; 112defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">; 113def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>, 114 NameOverride<"vsubq">; 115def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>, 116 NameOverride<"vmulq">; 117} 118 119// The bitcasting below is not overcomplicating the IR because while 120// Vector and UVector may be different vector types at the C level i.e. 121// vectors of same size signed/unsigned ints. Once they're lowered 122// to IR, they are just bit vectors with no sign at all, so the 123// bitcasts will be automatically elided by IRBuilder. 124multiclass predicated_bit_op_fp<string int_op> { 125def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, 126 Predicate:$pred), 127 (bitcast (IRInt<int_op, [UVector, Predicate]> 128 (bitcast $a, UVector), 129 (bitcast $b, UVector), 130 $pred, 131 (bitcast $inactive, UVector)), Vector)>; 132} 133 134// Plain intrinsics 135let params = T.Usual in { 136def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 137 (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>; 138} 139 140multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?), 141 int wantXVariant = 1> { 142 defm "" : IntrinsicMX< 143 Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 144 !con((IRInt<operation, [Vector, Predicate]> $a, $b), 145 extraArgs, (? $pred, $inactive)), wantXVariant>; 146} 147 148multiclass VectorVectorArithmeticBitcast<string operation> { 149 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, 150 Predicate:$pred), 151 (bitcast (IRInt<operation, [UVector, Predicate]> 152 (bitcast $a, UVector), 153 (bitcast $b, UVector), 154 $pred, 155 (bitcast $inactive, UVector)), Vector)>; 156} 157 158// Predicated intrinsics 159let params = T.Usual in { 160 defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>; 161 defm vaddq : VectorVectorArithmetic<"add_predicated">; 162 defm vsubq : VectorVectorArithmetic<"sub_predicated">; 163 defm vmulq : VectorVectorArithmetic<"mul_predicated">; 164 defm vandq : VectorVectorArithmeticBitcast<"and_predicated">; 165 defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">; 166 defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">; 167 defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">; 168 defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">; 169} 170 171multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?)> { 172 defm "" : IntrinsicMX< 173 DblVector, (args Vector:$a, Vector:$b, Predicate:$pred), 174 !con((IRInt<operation, [DblVector, Vector, Predicate]> $a, $b), 175 extraArgs, (? $pred, $inactive))>; 176} 177 178// Predicated intrinsics - Int types only 179let params = T.Int in { 180 defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>; 181 defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>; 182 defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>; 183 defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>; 184 defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>; 185 defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>; 186 defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>; 187 defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>; 188 defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>; 189 defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>; 190 defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>; 191} 192let params = T.Signed in { 193 defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>; 194 defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>; 195} 196 197let params = T.Poly, overrideKindLetter = "p" in { 198 defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>; 199 defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>; 200} 201 202// Predicated intrinsics - Float types only 203let params = T.Float in { 204 defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>; 205 defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>; 206} 207 208let params = T.Int in { 209def vminvq: Intrinsic<Scalar, (args Scalar:$prev, Vector:$vec), 210 (Scalar (IRInt<"minv", [Vector], 1> $prev, $vec))>; 211def vmaxvq: Intrinsic<Scalar, (args Scalar:$prev, Vector:$vec), 212 (Scalar (IRInt<"maxv", [Vector], 1> $prev, $vec))>; 213} 214 215foreach half = [ "b", "t" ] in { 216 defvar halfconst = !if(!eq(half, "b"), 0, 1); 217 218 let params = [f32], pnt = PNT_None in { 219 def vcvt#half#q_f16: Intrinsic< 220 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a), 221 (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>; 222 def vcvt#half#q_m_f16: Intrinsic< 223 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred), 224 (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>; 225 } // params = [f32], pnt = PNT_None 226} // loop over half = "b", "t" 227 228multiclass compare_with_pred<string condname, dag arguments, 229 dag cmp, string suffix> { 230 // Make the predicated and unpredicated versions of a single comparison. 231 def: Intrinsic<Predicate, arguments, 232 (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>, 233 NameOverride<"vcmp" # condname # "q" # suffix>; 234 def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)), 235 (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>, 236 NameOverride<"vcmp" # condname # "q_m" # suffix>; 237} 238 239multiclass compare<string condname, IRBuilder cmpop> { 240 // Make all four variants of a comparison: the vector/vector and 241 // vector/scalar forms, each using compare_with_pred to make a 242 // predicated and unpredicated version. 243 defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb), 244 (cmpop $va, $vb), "">; 245 let pnt = PNT_NType in { 246 defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb), 247 (cmpop $va, (splat $sb)), "_n">; 248 } 249} 250let params = T.Int in { 251 defm: compare<"eq", icmp_eq>; 252 defm: compare<"ne", icmp_ne>; 253} 254let params = T.Signed in { 255 defm: compare<"gt", icmp_sgt>; 256 defm: compare<"ge", icmp_sge>; 257 defm: compare<"lt", icmp_slt>; 258 defm: compare<"le", icmp_sle>; 259} 260let params = T.Unsigned in { 261 defm: compare<"hi", icmp_ugt>; 262 defm: compare<"cs", icmp_uge>; 263} 264let params = T.Float in { 265 defm: compare<"eq", fcmp_eq>; 266 defm: compare<"ne", fcmp_ne>; 267 defm: compare<"gt", fcmp_gt>; 268 defm: compare<"ge", fcmp_ge>; 269 defm: compare<"lt", fcmp_lt>; 270 defm: compare<"le", fcmp_le>; 271} 272 273let params = T.Signed in { 274 def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 275 (select (icmp_sle $a, $b), $a, $b)>; 276 def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 277 (select (icmp_sge $a, $b), $a, $b)>; 278} 279let params = T.Unsigned in { 280 def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b), 281 (select (icmp_ule $a, $b), $a, $b)>, 282 NameOverride<"vminq">; 283 def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b), 284 (select (icmp_uge $a, $b), $a, $b)>, 285 NameOverride<"vmaxq">; 286} 287let params = T.Float in { 288 def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 289 (IRIntBase<"minnum", [Vector]> $a, $b)>; 290 def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 291 (IRIntBase<"maxnum", [Vector]> $a, $b)>; 292} 293 294def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred), 295 (select $pred, $t, $f)> { let params = T.Usual; } 296def vpselq_64: Intrinsic< 297 Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred), 298 (bitcast (select $pred, (bitcast $t, VecOf<u32>), 299 (bitcast $f, VecOf<u32>)), Vector)>, 300 NameOverride<"vpselq"> { let params = T.All64; } 301 302let params = [Void], pnt = PNT_None in { 303 304 multiclass vctp<Type pred, string intname> { 305 def "": Intrinsic<pred, (args u32:$val), 306 (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>; 307 def _m: Intrinsic<pred, (args u32:$val, pred:$inpred), 308 (u16 (IRInt<"pred_v2i", [pred]> (and $inpred, 309 (IRIntBase<intname> $val))))>; 310 } 311 defm vctp8q: vctp<PredOf<u8>, "arm_mve_vctp8">; 312 defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">; 313 defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">; 314 defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">; 315 316 def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred), 317 (xor $pred, (u16 65535))>; 318 319} 320 321multiclass contiguous_load<string mnemonic, PrimitiveType memtype, 322 list<Type> same_size, list<Type> wider> { 323 // Intrinsics named with explicit memory and element sizes that match: 324 // vldrbq_?8, vldrhq_?16, vldrwq_?32. 325 let params = same_size, pnt = PNT_None in { 326 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), 327 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>, 328 NameOverride<mnemonic>; 329 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, 330 Predicate:$pred), 331 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]> 332 (CPtr<Vector> $addr), !srl(memtype.size,3), 333 $pred, (zeroinit Vector))>, 334 NameOverride<mnemonic # "_z">; 335 } 336 337 // Synonyms for the above, with the generic name vld1q that just means 338 // 'memory and element sizes match', and allows convenient polymorphism with 339 // the memory and element types covariant. 340 let params = same_size in { 341 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), 342 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>, 343 NameOverride<"vld1q">; 344 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, 345 Predicate:$pred), 346 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]> 347 (CPtr<Vector> $addr), !srl(memtype.size,3), 348 $pred, (zeroinit Vector))>, 349 NameOverride<"vld1q_z">; 350 } 351 352 // Intrinsics with the memory size narrower than the vector element, so that 353 // they load less than 128 bits of memory and sign/zero extend each loaded 354 // value into a wider vector lane. 355 let params = wider, pnt = PNT_None in { 356 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), 357 (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>> 358 $addr), !srl(memtype.size,3))), 359 Vector, (unsignedflag Scalar))>, 360 NameOverride<mnemonic>; 361 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, 362 Predicate:$pred), 363 (extend (IRIntBase<"masked_load", 364 [NarrowedVecOf<memtype,Vector>, 365 CPtr<NarrowedVecOf<memtype,Vector>>]> 366 (CPtr<NarrowedVecOf<memtype,Vector>> $addr), 367 !srl(memtype.size,3), $pred, 368 (zeroinit NarrowedVecOf<memtype,Vector>)), 369 Vector, (unsignedflag Scalar))>, 370 NameOverride<mnemonic # "_z">; 371 } 372} 373 374defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>; 375defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>; 376defm: contiguous_load<"vldrwq", u32, T.All32, []>; 377 378multiclass contiguous_store<string mnemonic, PrimitiveType memtype, 379 list<Type> same_size, list<Type> wider> { 380 // Intrinsics named with explicit memory and element sizes that match: 381 // vstrbq_?8, vstrhq_?16, vstrwq_?32. 382 let params = same_size in { 383 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 384 Vector:$value), 385 (store $value, 386 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>, 387 NameOverride<mnemonic>; 388 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 389 Vector:$value, Predicate:$pred), 390 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]> 391 $value, (Ptr<Vector> $addr), 392 !srl(memtype.size,3), $pred)>, 393 NameOverride<mnemonic # "_p">; 394 } 395 396 // Synonyms for the above, with the generic name vst1q that just means 397 // 'memory and element sizes match', and allows convenient polymorphism with 398 // the memory and element types covariant. 399 let params = same_size in { 400 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 401 Vector:$value), 402 (store $value, 403 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>, 404 NameOverride<"vst1q">; 405 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 406 Vector:$value, Predicate:$pred), 407 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]> 408 $value, (Ptr<Vector> $addr), 409 !srl(memtype.size,3), $pred)>, 410 NameOverride<"vst1q_p">; 411 } 412 413 // Intrinsics with the memory size narrower than the vector element, so that 414 // they store less than 128 bits of memory, truncating each vector lane into 415 // a narrower value to store. 416 let params = wider in { 417 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 418 Vector:$value), 419 (store (trunc $value, NarrowedVecOf<memtype,Vector>), 420 (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr), 421 !srl(memtype.size,3)))>, 422 NameOverride<mnemonic>; 423 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 424 Vector:$value, Predicate:$pred), 425 (IRIntBase<"masked_store", 426 [NarrowedVecOf<memtype,Vector>, 427 Ptr<NarrowedVecOf<memtype,Vector>>]> 428 (trunc $value, NarrowedVecOf<memtype,Vector>), 429 (Ptr<NarrowedVecOf<memtype,Vector>> $addr), 430 !srl(memtype.size,3), $pred)>, 431 NameOverride<mnemonic # "_p">; 432 } 433} 434 435defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>; 436defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>; 437defm: contiguous_store<"vstrwq", u32, T.All32, []>; 438 439multiclass gather_base<list<Type> types, int size> { 440 let params = types, pnt = PNT_None in { 441 def _gather_base: Intrinsic< 442 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset), 443 (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>; 444 445 def _gather_base_z: Intrinsic< 446 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred), 447 (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]> 448 $addr, $offset, $pred)>; 449 450 def _gather_base_wb: Intrinsic< 451 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset), 452 (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]> 453 (load $addr), $offset):$pair, 454 (store (xval $pair, 1), $addr), 455 (xval $pair, 0))>; 456 457 def _gather_base_wb_z: Intrinsic< 458 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, 459 Predicate:$pred), 460 (seq (IRInt<"vldr_gather_base_wb_predicated", 461 [Vector, UVector, Predicate]> 462 (load $addr), $offset, $pred):$pair, 463 (store (xval $pair, 1), $addr), 464 (xval $pair, 0))>; 465 } 466} 467 468defm vldrwq: gather_base<T.All32, 4>; 469defm vldrdq: gather_base<T.All64, 8>; 470 471multiclass scatter_base<list<Type> types, int size> { 472 let params = types in { 473 def _scatter_base: Intrinsic< 474 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data), 475 (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>; 476 477 def _scatter_base_p: Intrinsic< 478 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data, 479 Predicate:$pred), 480 (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]> 481 $addr, $offset, $data, $pred)>; 482 483 def _scatter_base_wb: Intrinsic< 484 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data), 485 (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]> 486 (load $addr), $offset, $data):$wbaddr, 487 (store $wbaddr, $addr))>; 488 489 def _scatter_base_wb_p: Intrinsic< 490 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, 491 Vector:$data, Predicate:$pred), 492 (seq (IRInt<"vstr_scatter_base_wb_predicated", 493 [UVector, Vector, Predicate]> 494 (load $addr), $offset, $data, $pred):$wbaddr, 495 (store $wbaddr, $addr))>; 496 } 497} 498 499defm vstrwq: scatter_base<T.All32, 4>; 500defm vstrdq: scatter_base<T.All64, 8>; 501 502multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> { 503 let params = types in { 504 def _gather_offset: Intrinsic< 505 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets), 506 (IRInt<"vldr_gather_offset", 507 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]> 508 $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>; 509 def _gather_offset_z: Intrinsic< 510 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 511 Predicate:$pred), 512 (IRInt<"vldr_gather_offset_predicated", 513 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]> 514 $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>; 515 } 516} 517 518multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype, 519 int shift> { 520 let params = types in { 521 def _gather_shifted_offset: Intrinsic< 522 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets), 523 (IRInt<"vldr_gather_offset", 524 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]> 525 $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>; 526 def _gather_shifted_offset_z: Intrinsic< 527 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 528 Predicate:$pred), 529 (IRInt<"vldr_gather_offset_predicated", 530 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]> 531 $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>; 532 } 533} 534 535multiclass gather_offset_both<list<Type> types, PrimitiveType memtype, 536 int shift> { 537 defm "": gather_offset_unshifted<types, memtype>; 538 defm "": gather_offset_shifted<types, memtype, shift>; 539} 540 541defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>; 542defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>; 543defm vldrwq: gather_offset_both<T.All32, u32, 2>; 544defm vldrdq: gather_offset_both<T.Int64, u64, 3>; 545 546multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> { 547 let params = types in { 548 def _scatter_offset: Intrinsic< 549 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 550 Vector:$data), 551 (IRInt<"vstr_scatter_offset", 552 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]> 553 $base, $offsets, $data, memtype.size, 0)>; 554 def _scatter_offset_p: Intrinsic< 555 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 556 Vector:$data, Predicate:$pred), 557 (IRInt<"vstr_scatter_offset_predicated", 558 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]> 559 $base, $offsets, $data, memtype.size, 0, $pred)>; 560 } 561} 562 563multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype, 564 int shift> { 565 let params = types in { 566 def _scatter_shifted_offset: Intrinsic< 567 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 568 Vector:$data), 569 (IRInt<"vstr_scatter_offset", 570 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]> 571 $base, $offsets, $data, memtype.size, shift)>; 572 def _scatter_shifted_offset_p: Intrinsic< 573 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 574 Vector:$data, Predicate:$pred), 575 (IRInt<"vstr_scatter_offset_predicated", 576 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]> 577 $base, $offsets, $data, memtype.size, shift, $pred)>; 578 } 579} 580 581multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype, 582 int shift> { 583 defm "": scatter_offset_unshifted<types, memtype>; 584 defm "": scatter_offset_shifted<types, memtype, shift>; 585} 586 587defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>; 588defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>; 589defm vstrwq: scatter_offset_both<T.All32, u32, 2>; 590defm vstrdq: scatter_offset_both<T.Int64, u64, 3>; 591 592let params = T.Int in { 593 def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh), 594 (shl $v, (splat (Scalar $sh)))>; 595 defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh, 596 Predicate:$pred), 597 (IRInt<"shl_imm_predicated", [Vector, Predicate]> 598 $v, $sh, $pred, $inactive), 1, "_n">; 599 600 let pnt = PNT_NType in { 601 def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh), 602 (immshr $v, $sh, (unsignedflag Scalar))>; 603 defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh, 604 Predicate:$pred), 605 (IRInt<"shr_imm_predicated", [Vector, Predicate]> 606 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">; 607 } 608} 609 610let params = T.Int in { 611 def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh), 612 (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>; 613 def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v, 614 imm_0toNm1:$sh, Predicate:$pred), 615 (IRInt<"vqshl_imm_predicated", [Vector, Predicate]> 616 $v, $sh, (unsignedflag Scalar), $pred, $inactive)>; 617 618 let pnt = PNT_NType in { 619 def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh), 620 (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>; 621 defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh, 622 Predicate:$pred), 623 (IRInt<"vrshr_imm_predicated", [Vector, Predicate]> 624 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">; 625 } 626} 627 628let params = T.Signed, pnt = PNT_NType in { 629 def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh), 630 (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>; 631 def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v, 632 imm_0toNm1:$sh, Predicate:$pred), 633 (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]> 634 $v, $sh, $pred, $inactive)>; 635} 636 637multiclass vshll_imm<int top> { 638 let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in { 639 def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh), 640 (IRInt<"vshll_imm", [DblVector, Vector]> 641 $v, $sh, (unsignedflag Scalar), top)>; 642 defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh, 643 Predicate:$pred), 644 (IRInt<"vshll_imm_predicated", [DblVector, Vector, Predicate]> 645 $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">; 646 } 647} 648defm vshllbq : vshll_imm<0>; 649defm vshlltq : vshll_imm<1>; 650 651multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME, 652 dag extraargs = (?)> { 653 defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)), 654 [Vector], [outtype, Vector]); 655 656 def q_n: Intrinsic< 657 outtype, (args outtype:$a, Vector:$b, imm:$sh), 658 !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>; 659 660 def q_m_n: Intrinsic< 661 outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred), 662 !con((IRInt<intname # "_predicated", intparams # [Predicate]> 663 $a, $b, $sh), extraargs, (? $pred))>; 664} 665 666multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> { 667 defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>; 668 defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>; 669} 670 671let params = [s16, s32, u16, u32], pnt = PNT_NType in { 672 defvar U = (unsignedflag Scalar); 673 defm vshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>; 674 defm vqshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>; 675 defm vrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>; 676 defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>; 677} 678let params = [s16, s32], pnt = PNT_NType in { 679 defm vqshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>; 680 defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>; 681} 682let params = T.Int, pnt = PNT_NType in { 683 defm vsli : DyadicImmShift<Vector, imm_0toNm1>; 684 defm vsri : DyadicImmShift<Vector, imm_1toN>; 685} 686 687multiclass VSHL_non_imm<string scalarSuffix, int q, int r, 688 PolymorphicNameType pnt_scalar_unpred = PNT_Type> { 689 let pnt = pnt_scalar_unpred in { 690 def scalarSuffix: Intrinsic< 691 Vector, (args Vector:$in, s32:$sh), 692 (IRInt<"vshl_scalar", [Vector]> $in, $sh, 693 q, r, (unsignedflag Scalar))>; 694 } 695 def "_m" # scalarSuffix: Intrinsic< 696 Vector, (args Vector:$in, s32:$sh, Predicate:$pred), 697 (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh, 698 q, r, (unsignedflag Scalar), $pred)>; 699 700 def "": Intrinsic< 701 Vector, (args Vector:$in, SVector:$sh), 702 (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh, 703 q, r, (unsignedflag Scalar))>; 704 defm "": IntrinsicMX< 705 Vector, (args Vector:$in, SVector:$sh, Predicate:$pred), 706 (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh, 707 q, r, (unsignedflag Scalar), $pred, $inactive), 708 // The saturating shift intrinsics don't have an x variant, so we 709 // set wantXVariant to 1 iff q == 0 710 !eq(q, 0)>; 711} 712 713let params = T.Int in { 714 defm vshlq : VSHL_non_imm<"_r", 0, 0>; 715 defm vqshlq : VSHL_non_imm<"_r", 1, 0>; 716 defm vrshlq : VSHL_non_imm<"_n", 0, 1, PNT_NType>; 717 defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>; 718} 719 720// Base class for the scalar shift intrinsics. 721class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>: 722 Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> { 723 let params = [Void]; 724 let pnt = PNT_None; 725} 726 727// Subclass that includes the machinery to take a 64-bit input apart 728// into halves, retrieve the two halves of a shifted output as a pair, 729// and glue the pieces of the pair back into an i64 for output. 730class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>: 731 ScalarShift<argtype, shiftCountArg, 732 (seq (u32 (lshr $value, (argtype 32))):$hi, 733 (u32 $value):$lo, 734 shiftCodeGen:$pair, 735 (or (shl (u64 (xval $pair, 1)), (u64 32)), 736 (u64 (xval $pair, 0))))>; 737 738// The family of saturating/rounding scalar shifts that take an 739// immediate shift count. They come in matched 32- and 64-bit pairs. 740multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> { 741 def "": ScalarShift<arg32, (args imm_1to32:$sh), 742 (IRInt<NAME> $value, $sh)>; 743 def l: LongScalarShift<arg64, (args imm_1to32:$sh), 744 (IRInt<NAME # "l"> $lo, $hi, $sh)>; 745} 746defm uqshl: ScalarSaturatingShiftImm<u32, u64>; 747defm urshr: ScalarSaturatingShiftImm<u32, u64>; 748defm sqshl: ScalarSaturatingShiftImm<s32, s64>; 749defm srshr: ScalarSaturatingShiftImm<s32, s64>; 750 751// The family of saturating/rounding scalar shifts that take a 752// register shift count. They also have 32- and 64-bit forms, but the 753// 64-bit form also has a version that saturates to 48 bits, so the IR 754// intrinsic takes an extra saturation-type operand. 755multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> { 756 def "": ScalarShift<arg32, (args s32:$sh), 757 (IRInt<NAME> $value, $sh)>; 758 def l: LongScalarShift<arg64, (args s32:$sh), 759 (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>; 760 def l_sat48: LongScalarShift<arg64, (args s32:$sh), 761 (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>; 762} 763defm uqrshl: ScalarSaturatingShiftReg<u32, u64>; 764defm sqrshr: ScalarSaturatingShiftReg<s32, s64>; 765 766// The intrinsics for LSLL and ASRL come in 64-bit versions only, with 767// no saturation count. 768def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>; 769def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>; 770 771let params = T.Int32 in { 772def vadcq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry), 773 (seq (IRInt<"vadc", [Vector]> $a, $b, (shl (load $carry), 29)):$pair, 774 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 775 (xval $pair, 0))>; 776def vadciq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry), 777 (seq (IRInt<"vadc", [Vector]> $a, $b, 0):$pair, 778 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 779 (xval $pair, 0))>; 780def vadcq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, 781 Ptr<uint>:$carry, Predicate:$pred), 782 (seq (IRInt<"vadc_predicated", [Vector, Predicate]> $inactive, $a, $b, 783 (shl (load $carry), 29), $pred):$pair, 784 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 785 (xval $pair, 0))>; 786def vadciq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, 787 Ptr<uint>:$carry, Predicate:$pred), 788 (seq (IRInt<"vadc_predicated", [Vector, Predicate]> $inactive, $a, $b, 789 0, $pred):$pair, 790 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 791 (xval $pair, 0))>; 792} 793 794multiclass VectorComplexAddPred<dag not_halving, dag angle> { 795 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b), 796 (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>; 797 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 798 (IRInt<"vcaddq_predicated", [Vector, Predicate]> 799 not_halving, angle, $inactive, $a, $b, $pred)>; 800} 801 802multiclass VectorComplexMulPred<dag angle> { 803 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b), 804 (IRInt<"vcmulq", [Vector]> angle, $a, $b)>; 805 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 806 (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b, 807 $pred)>; 808} 809 810multiclass VectorComplexMLAPred<dag angle> { 811 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c), 812 (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>; 813 def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c, 814 Predicate:$pred), 815 (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>; 816} 817 818multiclass VectorComplexAddAngle<dag not_halving> { 819 defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>; 820 defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>; 821} 822 823multiclass VectorComplexMulAngle { 824 defm "" : VectorComplexMulPred<(u32 0)>; 825 defm _rot90 : VectorComplexMulPred<(u32 1)>; 826 defm _rot180 : VectorComplexMulPred<(u32 2)>; 827 defm _rot270 : VectorComplexMulPred<(u32 3)>; 828} 829 830multiclass VectorComplexMLAAngle { 831 defm "" : VectorComplexMLAPred<(u32 0)>; 832 defm _rot90 : VectorComplexMLAPred<(u32 1)>; 833 defm _rot180 : VectorComplexMLAPred<(u32 2)>; 834 defm _rot270 : VectorComplexMLAPred<(u32 3)>; 835} 836 837let params = T.Usual in 838defm vcaddq : VectorComplexAddAngle<(u32 1)>; 839 840let params = T.Signed in 841defm vhcaddq : VectorComplexAddAngle<(u32 0)>; 842 843let params = T.Float in { 844defm vcmulq : VectorComplexMulAngle; 845defm vcmlaq : VectorComplexMLAAngle; 846} 847 848multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> { 849 def xsuffix#"q" 850 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b), 851 (IRInt<"vmldava", [Vector]> 852 (unsignedflag Scalar), subtract, exchange, 853 (zeroinit Scalar32), $a, $b)>; 854 def xsuffix#"q_p" 855 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred), 856 (IRInt<"vmldava_predicated", [Vector, Predicate]> 857 (unsignedflag Scalar), subtract, exchange, 858 (zeroinit Scalar32), $a, $b, $pred)>; 859 860 def "a"#xsuffix#"q" 861 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c), 862 (IRInt<"vmldava", [Vector]> 863 (unsignedflag Scalar), subtract, exchange, 864 $a, $b, $c)>; 865 def "a"#xsuffix#"q_p" 866 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c, 867 Predicate:$pred), 868 (IRInt<"vmldava_predicated", [Vector, Predicate]> 869 (unsignedflag Scalar), subtract, exchange, 870 $a, $b, $c, $pred)>; 871} 872 873class IntrSplit64<Type resty, dag args, dag codegen> 874 : Intrinsic<resty, args, 875 (seq (u32 (lshr $a, (u64 32))):$hi, 876 (u32 $a):$lo, 877 codegen:$pair, 878 (or (shl (u64 (xval $pair, 1)), (u64 32)), 879 (u64 (xval $pair, 0))))>; 880 881class IntrSplit64ZeroInit<Type resty, dag args, dag codegen> 882 : Intrinsic<resty, args, 883 (seq (zeroinit u32):$hi, 884 (zeroinit u32):$lo, 885 codegen:$pair, 886 (or (shl (u64 (xval $pair, 1)), (u64 32)), 887 (u64 (xval $pair, 0))))>; 888 889multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange, 890 string xsuffix, string irname> { 891 def xsuffix#"q" 892 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b), 893 (IRInt<irname, [Vector]> 894 (unsignedflag Scalar), subtract, exchange, 895 $lo, $hi, $a, $b)>; 896 def xsuffix#"q_p" 897 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b, 898 Predicate:$pred), 899 (IRInt<irname#"_predicated", [Vector, Predicate]> 900 (unsignedflag Scalar), subtract, exchange, 901 $lo, $hi, $a, $b, $pred)>; 902 903 def "a"#xsuffix#"q" 904 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c), 905 (IRInt<irname, [Vector]> 906 (unsignedflag Scalar), subtract, exchange, 907 $lo, $hi, $b, $c)>; 908 def "a"#xsuffix#"q_p" 909 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c, 910 Predicate:$pred), 911 (IRInt<irname#"_predicated", [Vector, Predicate]> 912 (unsignedflag Scalar), subtract, exchange, 913 $lo, $hi, $b, $c, $pred)>; 914} 915 916multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> { 917 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">; 918} 919 920multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> { 921 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, 922 "vrmlldavha">; 923} 924 925let params = T.Int in { 926def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c), 927 (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>; 928def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c, 929 Predicate:$pred), 930 (IRInt<"vabav_predicated", [Vector, Predicate]> 931 (unsignedflag Scalar), $a, $b, $c, $pred)>; 932 933defm vmladav : MVEBinaryVectorHoriz32<V.False, V.False, "">; 934} 935 936let params = T.Signed in { 937defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">; 938defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">; 939defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">; 940} 941 942let params = [u16, s16, u32, s32] in 943defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">; 944 945let params = [s16, s32] in { 946defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">; 947defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">; 948defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">; 949} 950 951let params = T.Int32 in 952defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">; 953 954let params = [s32] in { 955defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">; 956defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">; 957defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">; 958} 959 960foreach desttype = T.All in { 961 // We want a vreinterpretq between every pair of supported vector types 962 // _except_ that there shouldn't be one from a type to itself. 963 // 964 // So this foldl expression implements what you'd write in Python as 965 // [srctype for srctype in T.All if srctype != desttype] 966 let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist, 967 !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype]))) 968 in { 969 def "vreinterpretq_" # desttype: Intrinsic< 970 VecOf<desttype>, (args Vector:$x), (bitcast $x, VecOf<desttype>)>; 971 } 972} 973 974let params = T.All in { 975 let pnt = PNT_None in { 976 def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b), 977 (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0), 978 $b, 1), Vector)>; 979 def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>; 980 } 981 982 // This is the polymorphic form of vuninitializedq, which takes no type 983 // suffix, but takes an _unevaluated_ vector parameter and returns an 984 // uninitialized vector of the same vector type. 985 // 986 // This intrinsic has no _non_-polymorphic form exposed to the user. But each 987 // separately typed version of it still has to have its own clang builtin id, 988 // which can't be called vuninitializedq_u32 or similar because that would 989 // collide with the explicit nullary versions above. So I'm calling them 990 // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that 991 // full name never appears in the header file due to the polymorphicOnly 992 // flag, and the _polymorphic suffix is omitted from the shortened name by 993 // the custom PolymorphicNameType here. 994 let polymorphicOnly = 1, nonEvaluating = 1, 995 pnt = PolymorphicNameType<1, "polymorphic"> in { 996 def vuninitializedq_polymorphic: Intrinsic< 997 Vector, (args Vector), (undef Vector)>; 998 } 999 1000 def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane), 1001 (xelt_var $v, $lane)>; 1002 def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane), 1003 (ielt_var $v, $e, $lane)>; 1004} 1005