1//===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the set of ACLE-specified source-level intrinsic 10// functions wrapping the MVE vector instruction set and scalar shift 11// operations. 12// 13// Refer to comments in arm_mve_defs.td for the infrastructure used in 14// here, and to MveEmitter.cpp for how those are used in turn to 15// generate code. 16// 17//===----------------------------------------------------------------------===// 18 19include "arm_mve_defs.td" 20 21let params = T.Usual in 22foreach n = [ 2, 4 ] in { 23 def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>), 24 (CustomCodegen<"VST24"> n:$NumVectors, 25 "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>; 26 def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>), 27 (CustomCodegen<"VLD24"> n:$NumVectors, 28 "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>; 29} 30 31multiclass bit_op_fp<IRBuilder bitop> { 32def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), 33 (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>; 34} 35 36multiclass bit_op_fp_with_inv<IRBuilder bitop> { 37def "": Intrinsic<Vector, (args Vector:$a, Vector:$b), 38 (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>; 39} 40 41let params = T.Signed in { 42def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 43 (IRIntBase<"sadd_sat", [Vector]> $a, $b)>; 44def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 45 (IRIntBase<"ssub_sat", [Vector]> $a, $b)>; 46let pnt = PNT_NType in { 47 def vqaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 48 (IRIntBase<"sadd_sat", [Vector]> $a, (splat $b))>; 49 def vqsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 50 (IRIntBase<"ssub_sat", [Vector]> $a, (splat $b))>; 51} 52} 53let params = T.Unsigned in { 54def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b), 55 (IRIntBase<"uadd_sat", [Vector]> $a, $b)>, 56 NameOverride<"vqaddq">; 57def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b), 58 (IRIntBase<"usub_sat", [Vector]> $a, $b)>, 59 NameOverride<"vqsubq">; 60let pnt = PNT_NType in { 61 def vqaddq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 62 (IRIntBase<"uadd_sat", [Vector]> $a, (splat $b))>, 63 NameOverride<"vqaddq_n">; 64 def vqsubq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 65 (IRIntBase<"usub_sat", [Vector]> $a, (splat $b))>, 66 NameOverride<"vqsubq_n">; 67} 68} 69 70// Some intrinsics below are implemented not as IR fragments, but as 71// special-purpose IR intrinsics. This is because such a general form 72// (such as NEON uses) required a variable-width vector size, and we are 73// restricted to 128 bit. Although we can possibly get clever with lane 74// operations, the consequent IR representation would be very hard to 75// write sensibly. In particular, doubling a vector's width would be a 76// mess. Other intrinsics just don't translate nicely into IR. 77let params = T.Int in { 78def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>; 79def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 80 (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>; 81def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 82 (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>; 83def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>; 84def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>; 85def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>; 86def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>; 87def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>; 88def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>; 89def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 90 (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>; 91def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>; 92def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 93 (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>; 94def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 95 (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>; 96def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 97 (IRInt<"vmull", [DblVector, Vector]> 98 $a, $b, (unsignedflag Scalar), 0)>; 99def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 100 (IRInt<"vmull", [DblVector, Vector]> 101 $a, $b, (unsignedflag Scalar), 1)>; 102let pnt = PNT_NType in { 103 def vaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 104 (add $a, (splat $b))>; 105 def vsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 106 (sub $a, (splat $b))>; 107 def vmulq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 108 (mul $a, (splat $b))>; 109 def vhaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 110 (IRInt<"vhadd", [Vector]> $a, (splat $b), 111 (unsignedflag Scalar))>; 112 def vhsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 113 (IRInt<"vhsub", [Vector]> $a, (splat $b), 114 (unsignedflag Scalar))>; 115} 116} 117let params = T.Signed in { 118def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 119 (IRInt<"vqdmulh", [Vector]> $a, $b)>; 120def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 121 (IRInt<"vqrdmulh", [Vector]> $a, $b)>; 122let pnt = PNT_NType in { 123 def vqdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 124 (IRInt<"vqdmulh", [Vector]> $a, (splat $b))>; 125 def vqrdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 126 (IRInt<"vqrdmulh", [Vector]> $a, (splat $b))>; 127} 128} 129 130let params = T.Poly, overrideKindLetter = "p" in { 131def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 132 (IRInt<"vmull_poly", [DblVector, Vector]> 133 $a, $b, 0)>; 134def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 135 (IRInt<"vmull_poly", [DblVector, Vector]> 136 $a, $b, 1)>; 137} 138 139let params = T.Float in { 140def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>, 141 NameOverride<"vaddq">; 142defm vandqf: bit_op_fp<and>, NameOverride<"vandq">; 143defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">; 144defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">; 145defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">; 146defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">; 147def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>, 148 NameOverride<"vsubq">; 149def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>, 150 NameOverride<"vmulq">; 151 152let pnt = PNT_NType in { 153 def vaddqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 154 (fadd $a, (splat $b))>, 155 NameOverride<"vaddq_n">; 156 def vsubqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 157 (fsub $a, (splat $b))>, 158 NameOverride<"vsubq_n">; 159 def vmulqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b), 160 (fmul $a, (splat $b))>, 161 NameOverride<"vmulq_n">; 162} 163} 164 165multiclass FMA<bit add> { 166 // FMS instructions are defined in the ArmARM as if they negate the 167 // second multiply input. 168 defvar m2_cg = !if(add, (id $m2), (fneg $m2)); 169 170 defvar unpred_cg = (IRIntBase<"fma", [Vector]> $m1, m2_cg, $addend); 171 defvar pred_cg = (IRInt<"fma_predicated", [Vector, Predicate]> 172 $m1, m2_cg, $addend, $pred); 173 174 def q: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2), 175 unpred_cg>; 176 177 def q_m: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2, 178 Predicate:$pred), pred_cg>; 179 180 // Only FMA has the vector/scalar variants, not FMS 181 if add then let pnt = PNT_NType in { 182 183 def q_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, 184 unpromoted<Scalar>:$m2_s), 185 (seq (splat $m2_s):$m2, unpred_cg)>; 186 def sq_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2, 187 unpromoted<Scalar>:$addend_s), 188 (seq (splat $addend_s):$addend, unpred_cg)>; 189 def q_m_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, 190 unpromoted<Scalar>:$m2_s, 191 Predicate:$pred), 192 (seq (splat $m2_s):$m2, pred_cg)>; 193 def sq_m_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2, 194 unpromoted<Scalar>:$addend_s, 195 Predicate:$pred), 196 (seq (splat $addend_s):$addend, pred_cg)>; 197 } 198} 199 200let params = T.Float in { 201 defm vfma: FMA<1>; 202 defm vfms: FMA<0>; 203} 204 205let params = T.Int, pnt = PNT_NType in { 206 def vmlaq_n: Intrinsic< 207 Vector, (args Vector:$addend, Vector:$m1, unpromoted<Scalar>:$m2_s), 208 (add (mul $m1, (splat $m2_s)), $addend)>; 209 def vmlasq_n: Intrinsic< 210 Vector, (args Vector:$m1, Vector:$m2, unpromoted<Scalar>:$addend_s), 211 (add (mul $m1, $m2), (splat $addend_s))>; 212 213 def vmlaq_m_n: Intrinsic< 214 Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred), 215 (IRInt<"vmla_n_predicated", [Vector, Predicate]> 216 $addend, $m1, $m2_s, $pred)>; 217 def vmlasq_m_n: Intrinsic< 218 Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred), 219 (IRInt<"vmlas_n_predicated", [Vector, Predicate]> 220 $m1, $m2, $addend_s, $pred)>; 221} 222 223multiclass VQDMLA { 224 def hq_n: Intrinsic< 225 Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s), 226 (IRInt<NAME # "h", [Vector]> $addend, $m1, $m2_s)>; 227 def shq_n: Intrinsic< 228 Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s), 229 (IRInt<NAME # "sh", [Vector]> $m1, $m2, $addend_s)>; 230 231 def hq_m_n: Intrinsic< 232 Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred), 233 (IRInt<NAME # "h_predicated", [Vector, Predicate]> 234 $addend, $m1, $m2_s, $pred)>; 235 def shq_m_n: Intrinsic< 236 Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred), 237 (IRInt<NAME # "sh_predicated", [Vector, Predicate]> 238 $m1, $m2, $addend_s, $pred)>; 239} 240 241let params = T.Signed, pnt = PNT_NType in { 242 defm vqdmla: VQDMLA; 243 defm vqrdmla: VQDMLA; 244} 245 246multiclass VQDMLAD<int exchange, int round, int subtract> { 247 def "": Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c), 248 (IRInt<"vqdmlad", [Vector]> $a, $b, $c, 249 (u32 exchange), (u32 round), (u32 subtract))>; 250 def _m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c, 251 Predicate:$pred), 252 (IRInt<"vqdmlad_predicated", [Vector, Predicate]> $a, $b, $c, 253 (u32 exchange), (u32 round), (u32 subtract), $pred)>; 254} 255let params = T.Signed in { 256 defm vqdmladhq: VQDMLAD<0, 0, 0>; 257 defm vqdmladhxq: VQDMLAD<1, 0, 0>; 258 defm vqdmlsdhq: VQDMLAD<0, 0, 1>; 259 defm vqdmlsdhxq: VQDMLAD<1, 0, 1>; 260 defm vqrdmladhq: VQDMLAD<0, 1, 0>; 261 defm vqrdmladhxq: VQDMLAD<1, 1, 0>; 262 defm vqrdmlsdhq: VQDMLAD<0, 1, 1>; 263 defm vqrdmlsdhxq: VQDMLAD<1, 1, 1>; 264} 265 266let params = !listconcat(T.Int16, T.Int32) in { 267 let pnt = PNT_None in { 268 def vmvnq_n: Intrinsic<Vector, (args imm_simd_vmvn:$imm), 269 (not (splat (Scalar $imm)))>; 270 } 271 defm vmvnq: IntrinsicMX<Vector, (args imm_simd_vmvn:$imm, Predicate:$pred), 272 (select $pred, (not (splat (Scalar $imm))), $inactive), 273 1, "_n", PNT_NType, PNT_None>; 274 let pnt = PNT_NType in { 275 def vbicq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm), 276 (and $v, (not (splat (Scalar $imm))))>; 277 def vorrq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm), 278 (or $v, (splat (Scalar $imm)))>; 279 } 280 def vbicq_m_n: Intrinsic< 281 Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred), 282 (select $pred, (and $v, (not (splat (Scalar $imm)))), $v)>; 283 def vorrq_m_n: Intrinsic< 284 Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred), 285 (select $pred, (or $v, (splat (Scalar $imm))), $v)>; 286} 287 288let params = T.Usual in { 289 let pnt = PNT_None in 290 def vdupq_n: Intrinsic<Vector, (args unpromoted<Scalar>:$s), (splat $s)>; 291 292 defm vdupq: IntrinsicMX< 293 Vector, (args unpromoted<Scalar>:$s, Predicate:$pred), 294 (select $pred, (splat $s), $inactive), 1, "_n", PNT_NType, PNT_None>; 295} 296 297multiclass vxdup_mc<dag paramsIn, dag paramsOut> { 298 defvar UnpredInt = IRInt<NAME, [Vector]>; 299 defvar PredInt = IRInt<NAME # "_predicated", [Vector, Predicate]>; 300 defvar UnpredIntCall = !con((UnpredInt $base), paramsOut); 301 defvar PredIntCall = !con((PredInt $inactive, $base), paramsOut, (? $pred)); 302 303 // Straightforward case with neither writeback nor predication 304 let pnt = PNT_N in 305 def q_n: Intrinsic<Vector, !con((args u32:$base), paramsIn), 306 (xval UnpredIntCall, 0)>; 307 308 // Predicated form without writeback 309 defm q: IntrinsicMX< 310 Vector, !con((args u32:$base), paramsIn, (? Predicate:$pred)), 311 (xval PredIntCall, 0), 1, "_n", PNT_NType, PNT_N>; 312 313 // Writeback without predication 314 let pnt = PNT_WB in 315 def q_wb: Intrinsic< 316 Vector, !con((args Ptr<u32>:$baseaddr), paramsIn), 317 (seq (load $baseaddr):$base, 318 UnpredIntCall:$pair, 319 (store (xval $pair, 1), $baseaddr), 320 (xval $pair, 0))>; 321 322 // Both writeback and predicated 323 defm q: IntrinsicMX< 324 Vector, !con((args Ptr<u32>:$baseaddr), paramsIn, (? Predicate:$pred)), 325 (seq (load $baseaddr):$base, 326 PredIntCall:$pair, 327 (store (xval $pair, 1), $baseaddr), 328 (xval $pair, 0)), 1, "_wb", PNT_WBType, PNT_WB>; 329} 330 331let params = T.Unsigned in { 332 defm vidup: vxdup_mc<(? imm_1248:$step), (? $step)>; 333 defm vddup: vxdup_mc<(? imm_1248:$step), (? $step)>; 334 defm viwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>; 335 defm vdwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>; 336} 337 338let params = T.Int in { 339 def vmvnq: Intrinsic<Vector, (args Vector:$a), 340 (xor $a, (uint_max Vector))>; 341 defm vmvnq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 342 (IRInt<"mvn_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; 343 def vclzq: Intrinsic<Vector, (args Vector:$a), 344 (IRIntBase<"ctlz", [Vector]> $a, (i1 0))>; 345 defm vclzq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 346 (IRInt<"clz_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; 347} 348let params = T.Signed in { 349 def vclsq: Intrinsic<Vector, (args Vector:$a), (IRInt<"vcls", [Vector]> $a)>; 350 defm vclsq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 351 (IRInt<"cls_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; 352 353 def vnegq: Intrinsic<Vector, (args Vector:$a), 354 (sub (zeroinit Vector), $a)>; 355 def vabsq: Intrinsic<Vector, (args Vector:$a), 356 (select (icmp_slt $a, (zeroinit Vector)), 357 (sub (zeroinit Vector), $a), $a)>; 358 def vqnegq: Intrinsic<Vector, (args Vector:$a), 359 (select (icmp_eq $a, (int_min Vector)), 360 (int_max Vector), 361 (sub (zeroinit Vector), $a))>; 362 def vqabsq: Intrinsic<Vector, (args Vector:$a), 363 (select (icmp_sgt $a, (zeroinit Vector)), $a, 364 (select (icmp_eq $a, (int_min Vector)), 365 (int_max Vector), 366 (sub (zeroinit Vector), $a)))>; 367 368 foreach name = ["qneg", "qabs"] in { 369 defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 370 (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive), 371 0 /* no _x variant for saturating intrinsics */>; 372 } 373} 374let params = !listconcat(T.Signed, T.Float) in { 375 foreach name = ["neg", "abs"] in { 376 defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 377 (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive)>; 378 } 379} 380let params = T.Float in { 381 def vnegq_f: Intrinsic<Vector, (args Vector:$a), (fneg $a)>, 382 NameOverride<"vnegq">; 383 def vabsq_f: Intrinsic<Vector, (args Vector:$a), 384 (IRIntBase<"fabs", [Vector]> $a)>, NameOverride<"vabsq">; 385} 386 387// The bitcasting below is not overcomplicating the IR because while 388// Vector and UVector may be different vector types at the C level i.e. 389// vectors of same size signed/unsigned ints. Once they're lowered 390// to IR, they are just bit vectors with no sign at all, so the 391// bitcasts will be automatically elided by IRBuilder. 392multiclass predicated_bit_op_fp<string int_op> { 393def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, 394 Predicate:$pred), 395 (bitcast (IRInt<int_op, [UVector, Predicate]> 396 (bitcast $a, UVector), 397 (bitcast $b, UVector), 398 $pred, 399 (bitcast $inactive, UVector)), Vector)>; 400} 401 402// Plain intrinsics 403let params = T.Usual in { 404def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 405 (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>; 406} 407 408multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?), 409 int wantXVariant = 1> { 410 defm "" : IntrinsicMX< 411 Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 412 !con((IRInt<operation, [Vector, Predicate]> $a, $b), 413 extraArgs, (? $pred, $inactive)), wantXVariant>; 414} 415 416multiclass VectorScalarArithmetic<string operation, string basename, 417 dag extraArgs = (?), 418 int wantXVariant = 1> { 419 defm "" : IntrinsicMXNameOverride< 420 Vector, (args Vector:$a, unpromoted<Scalar>:$b, Predicate:$pred), 421 !con((IRInt<operation, [Vector, Predicate]> $a, (splat $b)), 422 extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n", 423 PNT_NType, PNT_NType>; 424} 425 426multiclass VectorVectorArithmeticBitcast<string operation> { 427 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, 428 Predicate:$pred), 429 (bitcast (IRInt<operation, [UVector, Predicate]> 430 (bitcast $a, UVector), 431 (bitcast $b, UVector), 432 $pred, 433 (bitcast $inactive, UVector)), Vector)>; 434} 435 436// Predicated intrinsics 437let params = T.Usual in { 438 defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>; 439 defm vaddq : VectorVectorArithmetic<"add_predicated">; 440 defm vsubq : VectorVectorArithmetic<"sub_predicated">; 441 defm vmulq : VectorVectorArithmetic<"mul_predicated">; 442 defm vandq : VectorVectorArithmeticBitcast<"and_predicated">; 443 defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">; 444 defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">; 445 defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">; 446 defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">; 447 448 defm : VectorScalarArithmetic<"add_predicated", "vaddq">; 449 defm : VectorScalarArithmetic<"sub_predicated", "vsubq">; 450 defm : VectorScalarArithmetic<"mul_predicated", "vmulq">; 451} 452 453multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?), 454 int wantXVariant = 1> { 455 defm "" : IntrinsicMX< 456 DblVector, (args Vector:$a, Vector:$b, DblPredicate:$pred), 457 !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, $b), 458 extraArgs, (? $pred, $inactive)), wantXVariant>; 459} 460 461multiclass DblVectorScalarArithmetic<string operation, string basename, 462 dag extraArgs = (?), 463 int wantXVariant = 1> { 464 defm "" : IntrinsicMXNameOverride< 465 DblVector, (args Vector:$a, unpromoted<Scalar>:$b, DblPredicate:$pred), 466 !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, (splat $b)), 467 extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n", 468 PNT_NType, PNT_NType>; 469} 470 471// Predicated intrinsics - Int types only 472let params = T.Int in { 473 defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>; 474 defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>; 475 defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>; 476 defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>; 477 defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>; 478 defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>; 479 defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>; 480 defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>; 481 defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>; 482 defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>; 483 defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>; 484 485 defm : VectorScalarArithmetic<"qadd_predicated", "vqaddq", (? (unsignedflag Scalar)), 0>; 486 defm : VectorScalarArithmetic<"hadd_predicated", "vhaddq", (? (unsignedflag Scalar))>; 487 defm : VectorScalarArithmetic<"qsub_predicated", "vqsubq", (? (unsignedflag Scalar)), 0>; 488 defm : VectorScalarArithmetic<"hsub_predicated", "vhsubq", (? (unsignedflag Scalar))>; 489} 490let params = T.Signed in { 491 defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>; 492 defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>; 493 def vminaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred), 494 (IRInt<"vmina_predicated", [UVector,Predicate]> $a, $b, $pred)>; 495 def vmaxaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred), 496 (IRInt<"vmaxa_predicated", [UVector,Predicate]> $a, $b, $pred)>; 497 498 defm : VectorScalarArithmetic<"qdmulh_predicated", "vqdmulhq", (?), 0>; 499 defm : VectorScalarArithmetic<"qrdmulh_predicated", "vqrdmulhq", (?), 0>; 500} 501 502let params = T.Poly, overrideKindLetter = "p" in { 503 defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>; 504 defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>; 505} 506 507let params = [s16, s32] in { 508 def vqdmullbq: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 509 (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 0)>; 510 def vqdmulltq: Intrinsic<DblVector, (args Vector:$a, Vector:$b), 511 (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 1)>; 512 defm vqdmullbq: DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 0)), 0>; 513 defm vqdmulltq: DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 1)), 0>; 514 515 let pnt = PNT_NType in { 516 def vqdmullbq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b), 517 (IRInt<"vqdmull", [DblVector, Vector]> 518 $a, (splat $b), 0)>; 519 def vqdmulltq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b), 520 (IRInt<"vqdmull", [DblVector, Vector]> 521 $a, (splat $b), 1)>; 522 } 523 defm vqdmullbq_n: DblVectorScalarArithmetic<"vqdmull_predicated", 524 "vqdmullbq", (? (u32 0)), 0>; 525 defm vqdmulltq_n: DblVectorScalarArithmetic<"vqdmull_predicated", 526 "vqdmulltq", (? (u32 1)), 0>; 527} 528 529// Predicated intrinsics - Float types only 530let params = T.Float in { 531 defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>; 532 defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>; 533 def vminnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 534 (IRInt<"vminnma_predicated", [Vector,Predicate]> $a, $b, $pred)>; 535 def vmaxnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 536 (IRInt<"vmaxnma_predicated", [Vector,Predicate]> $a, $b, $pred)>; 537} 538 539multiclass Reduction<Type Accumulator, string basename, list<Type> basetypes, 540 bit needSign = 0, 541 dag postCG = (seq (id $ret)), 542 dag accArg = (args Accumulator:$prev), 543 dag preCG = (seq)> { 544 defvar intArgsBase = (? $prev, $vec); 545 defvar intArgsUnpred = !con(intArgsBase, 546 !if(needSign, (? (unsignedflag Scalar)), (?))); 547 defvar intArgsPred = !con(intArgsUnpred, (? $pred)); 548 defvar intUnpred = !setop(intArgsUnpred, IRInt<basename, basetypes>); 549 defvar intPred = !setop(intArgsPred, IRInt< 550 basename#"_predicated", !listconcat(basetypes, [Predicate])>); 551 552 def "": Intrinsic< 553 Accumulator, !con(accArg, (args Vector:$vec)), 554 !con(preCG, (seq intUnpred:$ret), postCG)>; 555 def _p: Intrinsic< 556 Accumulator, !con(accArg, (args Vector:$vec, Predicate:$pred)), 557 !con(preCG, (seq intPred:$ret), postCG)>; 558} 559 560let params = T.Int in { 561defm vminvq: Reduction<Scalar, "minv", [Vector], 1, (seq (Scalar $ret))>; 562defm vmaxvq: Reduction<Scalar, "maxv", [Vector], 1, (seq (Scalar $ret))>; 563} 564 565let params = T.Signed in { 566defm vminavq: Reduction<UScalar, "minav", [Vector], 0, (seq (UScalar $ret))>; 567defm vmaxavq: Reduction<UScalar, "maxav", [Vector], 0, (seq (UScalar $ret))>; 568} 569 570let params = T.Float in { 571defm vminnmvq: Reduction<Scalar, "minnmv", [Scalar, Vector]>; 572defm vmaxnmvq: Reduction<Scalar, "maxnmv", [Scalar, Vector]>; 573defm vminnmavq: Reduction<Scalar, "minnmav", [Scalar, Vector]>; 574defm vmaxnmavq: Reduction<Scalar, "maxnmav", [Scalar, Vector]>; 575} 576 577foreach half = [ "b", "t" ] in { 578 defvar halfconst = !if(!eq(half, "b"), 0, 1); 579 580 let params = [f32], pnt = PNT_None in { 581 def vcvt#half#q_f16: Intrinsic< 582 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a), 583 (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>; 584 def vcvt#half#q_m_f16: Intrinsic< 585 VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred), 586 (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>; 587 } // params = [f32], pnt = PNT_None 588 589 let params = [f16], pnt = PNT_None in { 590 def vcvt#half#q_f32: Intrinsic<VecOf<f32>, (args Vector:$a), 591 (IRInt<"vcvt_widen"> $a, halfconst)>; 592 defm vcvt#half#q: IntrinsicMX< 593 VecOf<f32>, (args Vector:$a, PredOf<f32>:$pred), 594 (IRInt<"vcvt_widen_predicated"> $inactive, $a, halfconst, $pred), 595 1, "_f32">; 596 } // params = [f16], pnt = PNT_None 597} // loop over half = "b", "t" 598 599multiclass float_int_conversions<Type FScalar, Type IScalar, IRBuilderBase ftoi, IRBuilderBase itof> { 600 defvar FVector = VecOf<FScalar>; 601 defvar IVector = VecOf<IScalar>; 602 603 let params = [IScalar] in { 604 let pnt = PNT_2Type in { 605 def : Intrinsic<FVector, (args IVector:$a), (itof $a, FVector)>, 606 NameOverride<"vcvtq_" # FScalar>; 607 } 608 defm vcvtq: IntrinsicMX<FVector, (args IVector:$a, Predicate:$pred), 609 (IRInt<"vcvt_fp_int_predicated", [FVector, IVector, Predicate]> 610 $a, (unsignedflag IScalar), $pred, $inactive), 611 1, "_" # FScalar, PNT_2Type, PNT_2Type>; 612 } 613 let params = [FScalar] in { 614 let pnt = PNT_None in { 615 def : Intrinsic<IVector, (args FVector:$a), (ftoi $a, IVector)>, 616 NameOverride<"vcvtq_" # IScalar>; 617 618 foreach suffix = ["a","n","p","m"] in 619 def : Intrinsic<IVector, (args FVector:$a), 620 (IRInt<"vcvt"#suffix, [IVector, FVector]> 621 (unsignedflag IScalar), $a)>, 622 NameOverride<"vcvt"#suffix#"q_" # IScalar>; 623 } 624 defm vcvtq: IntrinsicMX<IVector, (args FVector:$a, Predicate:$pred), 625 (IRInt<"vcvt_fp_int_predicated", [IVector, FVector, Predicate]> 626 $a, (unsignedflag IScalar), $pred, $inactive), 627 1, "_" # IScalar, PNT_2Type, PNT_None>; 628 629 foreach suffix = ["a","n","p","m"] in { 630 defm "vcvt"#suffix#"q" : IntrinsicMX< 631 IVector, (args FVector:$a, Predicate:$pred), 632 (IRInt<"vcvt"#suffix#"_predicated", [IVector, FVector, Predicate]> 633 (unsignedflag IScalar), $inactive, $a, $pred), 634 1, "_" # IScalar, PNT_2Type, PNT_None>; 635 } 636 } 637} 638 639defm "" : float_int_conversions<f32, u32, fptoui, uitofp>; 640defm "" : float_int_conversions<f16, u16, fptoui, uitofp>; 641defm "" : float_int_conversions<f32, s32, fptosi, sitofp>; 642defm "" : float_int_conversions<f16, s16, fptosi, sitofp>; 643 644multiclass vmovl<bit top> { 645 let params = [s8, u8, s16, u16] in { 646 def "": Intrinsic<DblVector, (args Vector:$a), 647 (extend (unzip $a, top), DblVector, (unsignedflag Scalar))>; 648 defm "": IntrinsicMX<DblVector, (args Vector:$a, DblPredicate:$pred), 649 (IRInt<"vmovl_predicated", [DblVector, Vector, DblPredicate]> 650 $a, (unsignedflag Scalar), top, $pred, $inactive)>; 651 } 652} 653 654defm vmovlbq: vmovl<0>; 655defm vmovltq: vmovl<1>; 656 657multiclass vmovn<bit top, dag wide_result> { 658 let params = [s16, u16, s32, u32] in { 659 def "": Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a), 660 (trunc wide_result, HalfVector)>; 661 def _m: Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a, 662 Predicate:$pred), 663 (IRInt<"vmovn_predicated", [HalfVector, Vector, Predicate]> 664 $inactive, $a, top, $pred)>; 665 } 666} 667 668defm vmovntq: vmovn<1, (zip (vreinterpret $inactive, Vector), $a)>; 669defm vmovnbq: vmovn<0, 670 (zip $a, (vreinterpret (vrev $inactive, (bitsize Scalar)), Vector))>; 671 672multiclass vqmovn<bit top, Type RetScalar> { 673 defvar RetVector = VecOf<RetScalar>; 674 675 let params = [s16, u16, s32, u32] in { 676 def : Intrinsic< 677 RetVector, (args RetVector:$inactive, Vector:$a), 678 (IRInt<"vqmovn", [RetVector, Vector]> 679 $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), top)>, 680 NameOverride<NAME>; 681 def: Intrinsic< 682 RetVector, (args RetVector:$inactive, Vector:$a, Predicate:$pred), 683 (IRInt<"vqmovn_predicated", [RetVector, Vector, Predicate]> 684 $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), 685 top, $pred)>, 686 NameOverride<NAME # "_m">; 687 } 688} 689 690let params = [s16, s32, u16, u32] in { 691 defm vqmovntq: vqmovn<1, HalfScalar>; 692 defm vqmovnbq: vqmovn<0, HalfScalar>; 693} 694let params = [s16, s32] in { 695 defm vqmovuntq: vqmovn<1, UHalfScalar>; 696 defm vqmovunbq: vqmovn<0, UHalfScalar>; 697} 698 699multiclass vrnd<IRIntBase ir_int, string suffix> { 700 let params = T.Float in { 701 def "": Intrinsic<Vector, (args Vector:$a), (ir_int $a)>; 702 defm "": IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 703 (IRInt<"vrint"#suffix#"_predicated", [Vector, Predicate]> 704 $a, $pred, $inactive)>; 705 } 706} 707 708defm vrndq: vrnd<IRIntBase<"trunc", [Vector]>, "z">; 709defm vrndmq: vrnd<IRIntBase<"floor", [Vector]>, "m">; 710defm vrndpq: vrnd<IRIntBase<"ceil", [Vector]>, "p">; 711defm vrndaq: vrnd<IRIntBase<"round", [Vector]>, "a">; 712defm vrndxq: vrnd<IRIntBase<"rint", [Vector]>, "x">; 713defm vrndnq: vrnd<IRInt<"vrintn", [Vector]>, "n">; 714 715multiclass compare_with_pred<string condname, dag arguments, 716 dag cmp, string suffix> { 717 // Make the predicated and unpredicated versions of a single comparison. 718 def: Intrinsic<Predicate, arguments, 719 (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>, 720 NameOverride<"vcmp" # condname # "q" # suffix>; 721 def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)), 722 (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>, 723 NameOverride<"vcmp" # condname # "q_m" # suffix>; 724} 725 726multiclass compare<string condname, IRBuilder cmpop> { 727 // Make all four variants of a comparison: the vector/vector and 728 // vector/scalar forms, each using compare_with_pred to make a 729 // predicated and unpredicated version. 730 defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb), 731 (cmpop $va, $vb), "">; 732 let pnt = PNT_NType in { 733 defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb), 734 (cmpop $va, (splat $sb)), "_n">; 735 } 736} 737let params = T.Int in { 738 defm: compare<"eq", icmp_eq>; 739 defm: compare<"ne", icmp_ne>; 740} 741let params = T.Signed in { 742 defm: compare<"gt", icmp_sgt>; 743 defm: compare<"ge", icmp_sge>; 744 defm: compare<"lt", icmp_slt>; 745 defm: compare<"le", icmp_sle>; 746} 747let params = T.Unsigned in { 748 defm: compare<"hi", icmp_ugt>; 749 defm: compare<"cs", icmp_uge>; 750} 751let params = T.Float in { 752 defm: compare<"eq", fcmp_eq>; 753 defm: compare<"ne", fcmp_ne>; 754 defm: compare<"gt", fcmp_gt>; 755 defm: compare<"ge", fcmp_ge>; 756 defm: compare<"lt", fcmp_lt>; 757 defm: compare<"le", fcmp_le>; 758} 759 760let params = T.Signed in { 761 def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 762 (select (icmp_sle $a, $b), $a, $b)>; 763 def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 764 (select (icmp_sge $a, $b), $a, $b)>; 765 def vminaq: Intrinsic<UVector, (args UVector:$a, Vector:$b), 766 (seq (select (icmp_slt $b, (zeroinit Vector)), 767 (sub (zeroinit Vector), $b), $b):$absb, 768 (select (icmp_ule $a, $absb), $a, $absb))>; 769 def vmaxaq: Intrinsic<UVector, (args UVector:$a, Vector:$b), 770 (seq (select (icmp_slt $b, (zeroinit Vector)), 771 (sub (zeroinit Vector), $b), $b):$absb, 772 (select (icmp_uge $a, $absb), $a, $absb))>; 773} 774let params = T.Unsigned in { 775 def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b), 776 (select (icmp_ule $a, $b), $a, $b)>, 777 NameOverride<"vminq">; 778 def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b), 779 (select (icmp_uge $a, $b), $a, $b)>, 780 NameOverride<"vmaxq">; 781} 782let params = T.Float in { 783 def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 784 (IRIntBase<"minnum", [Vector]> $a, $b)>; 785 def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 786 (IRIntBase<"maxnum", [Vector]> $a, $b)>; 787 def vminnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 788 (IRIntBase<"minnum", [Vector]> 789 (IRIntBase<"fabs", [Vector]> $a), 790 (IRIntBase<"fabs", [Vector]> $b))>; 791 def vmaxnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b), 792 (IRIntBase<"maxnum", [Vector]> 793 (IRIntBase<"fabs", [Vector]> $a), 794 (IRIntBase<"fabs", [Vector]> $b))>; 795} 796 797def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred), 798 (select $pred, $t, $f)> { let params = T.Usual; } 799def vpselq_64: Intrinsic< 800 Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred), 801 (bitcast (select $pred, (bitcast $t, VecOf<u32>), 802 (bitcast $f, VecOf<u32>)), Vector)>, 803 NameOverride<"vpselq"> { let params = T.All64; } 804 805let params = [Void], pnt = PNT_None in { 806 807 multiclass vctp<Type pred, string intname> { 808 def "": Intrinsic<pred, (args u32:$val), 809 (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>; 810 def _m: Intrinsic<pred, (args u32:$val, pred:$inpred), 811 (u16 (IRInt<"pred_v2i", [pred]> (and $inpred, 812 (IRIntBase<intname> $val))))>; 813 } 814 defm vctp8q: vctp<PredOf<u8>, "arm_mve_vctp8">; 815 defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">; 816 defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">; 817 defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">; 818 819 def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred), 820 (xor $pred, (u16 65535))>; 821 822} 823 824multiclass contiguous_load<string mnemonic, PrimitiveType memtype, 825 list<Type> same_size, list<Type> wider> { 826 // Intrinsics named with explicit memory and element sizes that match: 827 // vldrbq_?8, vldrhq_?16, vldrwq_?32. 828 let params = same_size, pnt = PNT_None in { 829 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), 830 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>, 831 NameOverride<mnemonic>; 832 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, 833 Predicate:$pred), 834 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]> 835 (CPtr<Vector> $addr), !srl(memtype.size,3), 836 $pred, (zeroinit Vector))>, 837 NameOverride<mnemonic # "_z">; 838 } 839 840 // Synonyms for the above, with the generic name vld1q that just means 841 // 'memory and element sizes match', and allows convenient polymorphism with 842 // the memory and element types covariant. 843 let params = same_size in { 844 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), 845 (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>, 846 NameOverride<"vld1q">; 847 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, 848 Predicate:$pred), 849 (IRIntBase<"masked_load", [Vector, CPtr<Vector>]> 850 (CPtr<Vector> $addr), !srl(memtype.size,3), 851 $pred, (zeroinit Vector))>, 852 NameOverride<"vld1q_z">; 853 } 854 855 // Intrinsics with the memory size narrower than the vector element, so that 856 // they load less than 128 bits of memory and sign/zero extend each loaded 857 // value into a wider vector lane. 858 let params = wider, pnt = PNT_None in { 859 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr), 860 (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>> 861 $addr), !srl(memtype.size,3))), 862 Vector, (unsignedflag Scalar))>, 863 NameOverride<mnemonic>; 864 def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr, 865 Predicate:$pred), 866 (extend (IRIntBase<"masked_load", 867 [NarrowedVecOf<memtype,Vector>, 868 CPtr<NarrowedVecOf<memtype,Vector>>]> 869 (CPtr<NarrowedVecOf<memtype,Vector>> $addr), 870 !srl(memtype.size,3), $pred, 871 (zeroinit NarrowedVecOf<memtype,Vector>)), 872 Vector, (unsignedflag Scalar))>, 873 NameOverride<mnemonic # "_z">; 874 } 875} 876 877defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>; 878defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>; 879defm: contiguous_load<"vldrwq", u32, T.All32, []>; 880 881multiclass contiguous_store<string mnemonic, PrimitiveType memtype, 882 list<Type> same_size, list<Type> wider> { 883 // Intrinsics named with explicit memory and element sizes that match: 884 // vstrbq_?8, vstrhq_?16, vstrwq_?32. 885 let params = same_size in { 886 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 887 Vector:$value), 888 (store $value, 889 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>, 890 NameOverride<mnemonic>; 891 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 892 Vector:$value, Predicate:$pred), 893 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]> 894 $value, (Ptr<Vector> $addr), 895 !srl(memtype.size,3), $pred)>, 896 NameOverride<mnemonic # "_p">; 897 } 898 899 // Synonyms for the above, with the generic name vst1q that just means 900 // 'memory and element sizes match', and allows convenient polymorphism with 901 // the memory and element types covariant. 902 let params = same_size in { 903 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 904 Vector:$value), 905 (store $value, 906 (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>, 907 NameOverride<"vst1q">; 908 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 909 Vector:$value, Predicate:$pred), 910 (IRIntBase<"masked_store", [Vector, Ptr<Vector>]> 911 $value, (Ptr<Vector> $addr), 912 !srl(memtype.size,3), $pred)>, 913 NameOverride<"vst1q_p">; 914 } 915 916 // Intrinsics with the memory size narrower than the vector element, so that 917 // they store less than 128 bits of memory, truncating each vector lane into 918 // a narrower value to store. 919 let params = wider in { 920 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 921 Vector:$value), 922 (store (trunc $value, NarrowedVecOf<memtype,Vector>), 923 (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr), 924 !srl(memtype.size,3)))>, 925 NameOverride<mnemonic>; 926 def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr, 927 Vector:$value, Predicate:$pred), 928 (IRIntBase<"masked_store", 929 [NarrowedVecOf<memtype,Vector>, 930 Ptr<NarrowedVecOf<memtype,Vector>>]> 931 (trunc $value, NarrowedVecOf<memtype,Vector>), 932 (Ptr<NarrowedVecOf<memtype,Vector>> $addr), 933 !srl(memtype.size,3), $pred)>, 934 NameOverride<mnemonic # "_p">; 935 } 936} 937 938defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>; 939defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>; 940defm: contiguous_store<"vstrwq", u32, T.All32, []>; 941 942multiclass gather_base<list<Type> types, int size> { 943 let params = types, pnt = PNT_None in { 944 def _gather_base: Intrinsic< 945 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset), 946 (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>; 947 948 def _gather_base_z: Intrinsic< 949 Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred), 950 (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]> 951 $addr, $offset, $pred)>; 952 953 def _gather_base_wb: Intrinsic< 954 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset), 955 (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]> 956 (load $addr), $offset):$pair, 957 (store (xval $pair, 1), $addr), 958 (xval $pair, 0))>; 959 960 def _gather_base_wb_z: Intrinsic< 961 Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, 962 Predicate:$pred), 963 (seq (IRInt<"vldr_gather_base_wb_predicated", 964 [Vector, UVector, Predicate]> 965 (load $addr), $offset, $pred):$pair, 966 (store (xval $pair, 1), $addr), 967 (xval $pair, 0))>; 968 } 969} 970 971defm vldrwq: gather_base<T.All32, 4>; 972defm vldrdq: gather_base<T.All64, 8>; 973 974multiclass scatter_base<list<Type> types, int size> { 975 let params = types in { 976 def _scatter_base: Intrinsic< 977 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data), 978 (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>; 979 980 def _scatter_base_p: Intrinsic< 981 Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data, 982 Predicate:$pred), 983 (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]> 984 $addr, $offset, $data, $pred)>; 985 986 def _scatter_base_wb: Intrinsic< 987 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data), 988 (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]> 989 (load $addr), $offset, $data):$wbaddr, 990 (store $wbaddr, $addr))>; 991 992 def _scatter_base_wb_p: Intrinsic< 993 Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, 994 Vector:$data, Predicate:$pred), 995 (seq (IRInt<"vstr_scatter_base_wb_predicated", 996 [UVector, Vector, Predicate]> 997 (load $addr), $offset, $data, $pred):$wbaddr, 998 (store $wbaddr, $addr))>; 999 } 1000} 1001 1002defm vstrwq: scatter_base<T.All32, 4>; 1003defm vstrdq: scatter_base<T.All64, 8>; 1004 1005multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> { 1006 let params = types in { 1007 def _gather_offset: Intrinsic< 1008 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets), 1009 (IRInt<"vldr_gather_offset", 1010 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]> 1011 $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>; 1012 def _gather_offset_z: Intrinsic< 1013 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 1014 Predicate:$pred), 1015 (IRInt<"vldr_gather_offset_predicated", 1016 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]> 1017 $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>; 1018 } 1019} 1020 1021multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype, 1022 int shift> { 1023 let params = types in { 1024 def _gather_shifted_offset: Intrinsic< 1025 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets), 1026 (IRInt<"vldr_gather_offset", 1027 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]> 1028 $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>; 1029 def _gather_shifted_offset_z: Intrinsic< 1030 Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 1031 Predicate:$pred), 1032 (IRInt<"vldr_gather_offset_predicated", 1033 [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]> 1034 $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>; 1035 } 1036} 1037 1038multiclass gather_offset_both<list<Type> types, PrimitiveType memtype, 1039 int shift> { 1040 defm "": gather_offset_unshifted<types, memtype>; 1041 defm "": gather_offset_shifted<types, memtype, shift>; 1042} 1043 1044defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>; 1045defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>; 1046defm vldrwq: gather_offset_both<T.All32, u32, 2>; 1047defm vldrdq: gather_offset_both<T.Int64, u64, 3>; 1048 1049multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> { 1050 let params = types in { 1051 def _scatter_offset: Intrinsic< 1052 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 1053 Vector:$data), 1054 (IRInt<"vstr_scatter_offset", 1055 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]> 1056 $base, $offsets, $data, memtype.size, 0)>; 1057 def _scatter_offset_p: Intrinsic< 1058 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 1059 Vector:$data, Predicate:$pred), 1060 (IRInt<"vstr_scatter_offset_predicated", 1061 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]> 1062 $base, $offsets, $data, memtype.size, 0, $pred)>; 1063 } 1064} 1065 1066multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype, 1067 int shift> { 1068 let params = types in { 1069 def _scatter_shifted_offset: Intrinsic< 1070 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 1071 Vector:$data), 1072 (IRInt<"vstr_scatter_offset", 1073 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]> 1074 $base, $offsets, $data, memtype.size, shift)>; 1075 def _scatter_shifted_offset_p: Intrinsic< 1076 Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets, 1077 Vector:$data, Predicate:$pred), 1078 (IRInt<"vstr_scatter_offset_predicated", 1079 [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]> 1080 $base, $offsets, $data, memtype.size, shift, $pred)>; 1081 } 1082} 1083 1084multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype, 1085 int shift> { 1086 defm "": scatter_offset_unshifted<types, memtype>; 1087 defm "": scatter_offset_shifted<types, memtype, shift>; 1088} 1089 1090defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>; 1091defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>; 1092defm vstrwq: scatter_offset_both<T.All32, u32, 2>; 1093defm vstrdq: scatter_offset_both<T.Int64, u64, 3>; 1094 1095let params = T.Int in { 1096 def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh), 1097 (shl $v, (splat (Scalar $sh)))>; 1098 defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh, 1099 Predicate:$pred), 1100 (IRInt<"shl_imm_predicated", [Vector, Predicate]> 1101 $v, $sh, $pred, $inactive), 1, "_n">; 1102 1103 let pnt = PNT_NType in { 1104 def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh), 1105 (immshr $v, $sh, (unsignedflag Scalar))>; 1106 defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh, 1107 Predicate:$pred), 1108 (IRInt<"shr_imm_predicated", [Vector, Predicate]> 1109 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">; 1110 } 1111} 1112 1113let params = T.Int in { 1114 def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh), 1115 (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>; 1116 def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v, 1117 imm_0toNm1:$sh, Predicate:$pred), 1118 (IRInt<"vqshl_imm_predicated", [Vector, Predicate]> 1119 $v, $sh, (unsignedflag Scalar), $pred, $inactive)>; 1120 1121 let pnt = PNT_NType in { 1122 def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh), 1123 (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>; 1124 defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh, 1125 Predicate:$pred), 1126 (IRInt<"vrshr_imm_predicated", [Vector, Predicate]> 1127 $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">; 1128 } 1129} 1130 1131let params = T.Signed, pnt = PNT_NType in { 1132 def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh), 1133 (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>; 1134 def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v, 1135 imm_0toNm1:$sh, Predicate:$pred), 1136 (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]> 1137 $v, $sh, $pred, $inactive)>; 1138} 1139 1140multiclass vshll_imm<int top> { 1141 let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in { 1142 def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh), 1143 (IRInt<"vshll_imm", [DblVector, Vector]> 1144 $v, $sh, (unsignedflag Scalar), top)>; 1145 defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh, 1146 DblPredicate:$pred), 1147 (IRInt<"vshll_imm_predicated", [DblVector, Vector, DblPredicate]> 1148 $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">; 1149 } 1150} 1151defm vshllbq : vshll_imm<0>; 1152defm vshlltq : vshll_imm<1>; 1153 1154multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME, 1155 dag extraargs = (?)> { 1156 defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)), 1157 [Vector], [outtype, Vector]); 1158 1159 def q_n: Intrinsic< 1160 outtype, (args outtype:$a, Vector:$b, imm:$sh), 1161 !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>; 1162 1163 def q_m_n: Intrinsic< 1164 outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred), 1165 !con((IRInt<intname # "_predicated", intparams # [Predicate]> 1166 $a, $b, $sh), extraargs, (? $pred))>; 1167} 1168 1169multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> { 1170 defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>; 1171 defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>; 1172} 1173 1174let params = [s16, s32, u16, u32], pnt = PNT_NType in { 1175 defvar U = (unsignedflag Scalar); 1176 defm vshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>; 1177 defm vqshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>; 1178 defm vrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>; 1179 defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>; 1180} 1181let params = [s16, s32], pnt = PNT_NType in { 1182 defm vqshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>; 1183 defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>; 1184} 1185let params = T.Int, pnt = PNT_NType in { 1186 defm vsli : DyadicImmShift<Vector, imm_0toNm1>; 1187 defm vsri : DyadicImmShift<Vector, imm_1toN>; 1188} 1189 1190multiclass VSHL_non_imm<string scalarSuffix, int q, int r, 1191 PolymorphicNameType pnt_scalar_unpred = PNT_Type> { 1192 let pnt = pnt_scalar_unpred in { 1193 def scalarSuffix: Intrinsic< 1194 Vector, (args Vector:$in, s32:$sh), 1195 (IRInt<"vshl_scalar", [Vector]> $in, $sh, 1196 q, r, (unsignedflag Scalar))>; 1197 } 1198 def "_m" # scalarSuffix: Intrinsic< 1199 Vector, (args Vector:$in, s32:$sh, Predicate:$pred), 1200 (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh, 1201 q, r, (unsignedflag Scalar), $pred)>; 1202 1203 def "": Intrinsic< 1204 Vector, (args Vector:$in, SVector:$sh), 1205 (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh, 1206 q, r, (unsignedflag Scalar))>; 1207 defm "": IntrinsicMX< 1208 Vector, (args Vector:$in, SVector:$sh, Predicate:$pred), 1209 (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh, 1210 q, r, (unsignedflag Scalar), $pred, $inactive), 1211 // The saturating shift intrinsics don't have an x variant, so we 1212 // set wantXVariant to 1 iff q == 0 1213 !eq(q, 0)>; 1214} 1215 1216let params = T.Int in { 1217 defm vshlq : VSHL_non_imm<"_r", 0, 0>; 1218 defm vqshlq : VSHL_non_imm<"_r", 1, 0>; 1219 defm vrshlq : VSHL_non_imm<"_n", 0, 1, PNT_NType>; 1220 defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>; 1221} 1222 1223// Base class for the scalar shift intrinsics. 1224class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>: 1225 Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> { 1226 let params = [Void]; 1227 let pnt = PNT_None; 1228} 1229 1230// Subclass that includes the machinery to take a 64-bit input apart 1231// into halves, retrieve the two halves of a shifted output as a pair, 1232// and glue the pieces of the pair back into an i64 for output. 1233class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>: 1234 ScalarShift<argtype, shiftCountArg, 1235 (seq (u32 (lshr $value, (argtype 32))):$hi, 1236 (u32 $value):$lo, 1237 shiftCodeGen:$pair, 1238 (or (shl (u64 (xval $pair, 1)), (u64 32)), 1239 (u64 (xval $pair, 0))))>; 1240 1241// The family of saturating/rounding scalar shifts that take an 1242// immediate shift count. They come in matched 32- and 64-bit pairs. 1243multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> { 1244 def "": ScalarShift<arg32, (args imm_1to32:$sh), 1245 (IRInt<NAME> $value, $sh)>; 1246 def l: LongScalarShift<arg64, (args imm_1to32:$sh), 1247 (IRInt<NAME # "l"> $lo, $hi, $sh)>; 1248} 1249defm uqshl: ScalarSaturatingShiftImm<u32, u64>; 1250defm urshr: ScalarSaturatingShiftImm<u32, u64>; 1251defm sqshl: ScalarSaturatingShiftImm<s32, s64>; 1252defm srshr: ScalarSaturatingShiftImm<s32, s64>; 1253 1254// The family of saturating/rounding scalar shifts that take a 1255// register shift count. They also have 32- and 64-bit forms, but the 1256// 64-bit form also has a version that saturates to 48 bits, so the IR 1257// intrinsic takes an extra saturation-type operand. 1258multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> { 1259 def "": ScalarShift<arg32, (args s32:$sh), 1260 (IRInt<NAME> $value, $sh)>; 1261 def l: LongScalarShift<arg64, (args s32:$sh), 1262 (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>; 1263 def l_sat48: LongScalarShift<arg64, (args s32:$sh), 1264 (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>; 1265} 1266defm uqrshl: ScalarSaturatingShiftReg<u32, u64>; 1267defm sqrshr: ScalarSaturatingShiftReg<s32, s64>; 1268 1269// The intrinsics for LSLL and ASRL come in 64-bit versions only, with 1270// no saturation count. 1271def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>; 1272def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>; 1273 1274multiclass vadcsbc { 1275 def q: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry), 1276 (seq (IRInt<NAME, [Vector]> $a, $b, (shl (load $carry), 29)):$pair, 1277 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 1278 (xval $pair, 0))>; 1279 def iq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry), 1280 (seq (IRInt<NAME, [Vector]> $a, $b, 0):$pair, 1281 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 1282 (xval $pair, 0))>; 1283 def q_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, 1284 Ptr<uint>:$carry, Predicate:$pred), 1285 (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b, 1286 (shl (load $carry), 29), $pred):$pair, 1287 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 1288 (xval $pair, 0))>; 1289 def iq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b, 1290 Ptr<uint>:$carry, Predicate:$pred), 1291 (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b, 1292 0, $pred):$pair, 1293 (store (and 1, (lshr (xval $pair, 1), 29)), $carry), 1294 (xval $pair, 0))>; 1295} 1296let params = T.Int32 in { 1297 defm vadc: vadcsbc; 1298 defm vsbc: vadcsbc; 1299} 1300 1301let params = T.Int in { 1302 def vshlcq: Intrinsic< 1303 Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm), 1304 (seq (load $ps):$s, 1305 (IRInt<"vshlc", [Vector]> $v, $s, $imm):$pair, 1306 (store (xval $pair, 0), $ps), 1307 (xval $pair, 1))>; 1308 def vshlcq_m: Intrinsic< 1309 Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm, Predicate:$pred), 1310 (seq (load $ps):$s, 1311 (IRInt<"vshlc_predicated", [Vector, Predicate]> 1312 $v, $s, $imm, $pred):$pair, 1313 (store (xval $pair, 0), $ps), 1314 (xval $pair, 1))>; 1315} 1316 1317multiclass VectorComplexAddPred<dag not_halving, dag angle> { 1318 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b), 1319 (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>; 1320 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 1321 (IRInt<"vcaddq_predicated", [Vector, Predicate]> 1322 not_halving, angle, $inactive, $a, $b, $pred)>; 1323} 1324 1325multiclass VectorComplexMulPred<dag angle> { 1326 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b), 1327 (IRInt<"vcmulq", [Vector]> angle, $a, $b)>; 1328 defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred), 1329 (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b, 1330 $pred)>; 1331} 1332 1333multiclass VectorComplexMLAPred<dag angle> { 1334 def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c), 1335 (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>; 1336 def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c, 1337 Predicate:$pred), 1338 (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>; 1339} 1340 1341multiclass VectorComplexAddAngle<dag not_halving> { 1342 defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>; 1343 defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>; 1344} 1345 1346multiclass VectorComplexMulAngle { 1347 defm "" : VectorComplexMulPred<(u32 0)>; 1348 defm _rot90 : VectorComplexMulPred<(u32 1)>; 1349 defm _rot180 : VectorComplexMulPred<(u32 2)>; 1350 defm _rot270 : VectorComplexMulPred<(u32 3)>; 1351} 1352 1353multiclass VectorComplexMLAAngle { 1354 defm "" : VectorComplexMLAPred<(u32 0)>; 1355 defm _rot90 : VectorComplexMLAPred<(u32 1)>; 1356 defm _rot180 : VectorComplexMLAPred<(u32 2)>; 1357 defm _rot270 : VectorComplexMLAPred<(u32 3)>; 1358} 1359 1360let params = T.Usual in 1361defm vcaddq : VectorComplexAddAngle<(u32 1)>; 1362 1363let params = T.Signed in 1364defm vhcaddq : VectorComplexAddAngle<(u32 0)>; 1365 1366let params = T.Float in { 1367defm vcmulq : VectorComplexMulAngle; 1368defm vcmlaq : VectorComplexMLAAngle; 1369} 1370 1371multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> { 1372 def xsuffix#"q" 1373 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b), 1374 (IRInt<"vmldava", [Vector]> 1375 (unsignedflag Scalar), subtract, exchange, 1376 (zeroinit Scalar32), $a, $b)>; 1377 def xsuffix#"q_p" 1378 : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred), 1379 (IRInt<"vmldava_predicated", [Vector, Predicate]> 1380 (unsignedflag Scalar), subtract, exchange, 1381 (zeroinit Scalar32), $a, $b, $pred)>; 1382 1383 def "a"#xsuffix#"q" 1384 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c), 1385 (IRInt<"vmldava", [Vector]> 1386 (unsignedflag Scalar), subtract, exchange, 1387 $a, $b, $c)>; 1388 def "a"#xsuffix#"q_p" 1389 : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c, 1390 Predicate:$pred), 1391 (IRInt<"vmldava_predicated", [Vector, Predicate]> 1392 (unsignedflag Scalar), subtract, exchange, 1393 $a, $b, $c, $pred)>; 1394} 1395 1396class IntrSplit64<Type resty, dag args, dag codegen> 1397 : Intrinsic<resty, args, 1398 (seq (u32 (lshr $a, (u64 32))):$hi, 1399 (u32 $a):$lo, 1400 codegen:$pair, 1401 (or (shl (u64 (xval $pair, 1)), (u64 32)), 1402 (u64 (xval $pair, 0))))>; 1403 1404class IntrSplit64ZeroInit<Type resty, dag args, dag codegen> 1405 : Intrinsic<resty, args, 1406 (seq (zeroinit u32):$hi, 1407 (zeroinit u32):$lo, 1408 codegen:$pair, 1409 (or (shl (u64 (xval $pair, 1)), (u64 32)), 1410 (u64 (xval $pair, 0))))>; 1411 1412multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange, 1413 string xsuffix, string irname> { 1414 def xsuffix#"q" 1415 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b), 1416 (IRInt<irname, [Vector]> 1417 (unsignedflag Scalar), subtract, exchange, 1418 $lo, $hi, $a, $b)>; 1419 def xsuffix#"q_p" 1420 : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b, 1421 Predicate:$pred), 1422 (IRInt<irname#"_predicated", [Vector, Predicate]> 1423 (unsignedflag Scalar), subtract, exchange, 1424 $lo, $hi, $a, $b, $pred)>; 1425 1426 def "a"#xsuffix#"q" 1427 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c), 1428 (IRInt<irname, [Vector]> 1429 (unsignedflag Scalar), subtract, exchange, 1430 $lo, $hi, $b, $c)>; 1431 def "a"#xsuffix#"q_p" 1432 : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c, 1433 Predicate:$pred), 1434 (IRInt<irname#"_predicated", [Vector, Predicate]> 1435 (unsignedflag Scalar), subtract, exchange, 1436 $lo, $hi, $b, $c, $pred)>; 1437} 1438 1439multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> { 1440 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">; 1441} 1442 1443multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> { 1444 defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, 1445 "vrmlldavha">; 1446} 1447 1448multiclass VADDV<bit acc, bit pred, string intbase, Type Scalar> { 1449 defvar accArg = !if(acc, (args Scalar:$acc), (args)); 1450 defvar predArg = !if(pred, (args Predicate:$pred), (args)); 1451 defvar intrinsic = !if(pred, 1452 IRInt<intbase # "_predicated", [Vector, Predicate]>, 1453 IRInt<intbase, [Vector]>); 1454 defvar intCG = !con((intrinsic $v, (unsignedflag Scalar)), 1455 !if(pred, (? $pred), (?))); 1456 defvar accCG = !if(acc, (add intCG, $acc), intCG); 1457 1458 def "": Intrinsic<Scalar, !con(accArg, (args Vector:$v), predArg), accCG>; 1459} 1460 1461let params = T.Int in { 1462defm vaddvq : VADDV<0, 0, "addv", Scalar32>; 1463defm vaddvaq : VADDV<1, 0, "addv", Scalar32>; 1464defm vaddvq_p : VADDV<0, 1, "addv", Scalar32>; 1465defm vaddvaq_p : VADDV<1, 1, "addv", Scalar32>; 1466} 1467 1468let params = [s32, u32] in { 1469defm vaddlvq : VADDV<0, 0, "addlv", Scalar64>; 1470defm vaddlvaq : VADDV<1, 0, "addlv", Scalar64>; 1471defm vaddlvq_p : VADDV<0, 1, "addlv", Scalar64>; 1472defm vaddlvaq_p : VADDV<1, 1, "addlv", Scalar64>; 1473} 1474 1475let params = T.Int in { 1476def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c), 1477 (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>; 1478def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c, 1479 Predicate:$pred), 1480 (IRInt<"vabav_predicated", [Vector, Predicate]> 1481 (unsignedflag Scalar), $a, $b, $c, $pred)>; 1482 1483defm vmladav : MVEBinaryVectorHoriz32<V.False, V.False, "">; 1484} 1485 1486let params = T.Signed in { 1487defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">; 1488defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">; 1489defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">; 1490} 1491 1492let params = [u16, s16, u32, s32] in 1493defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">; 1494 1495let params = [s16, s32] in { 1496defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">; 1497defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">; 1498defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">; 1499} 1500 1501let params = T.Int32 in 1502defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">; 1503 1504let params = [s32] in { 1505defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">; 1506defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">; 1507defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">; 1508} 1509 1510multiclass vrev_predicated<int revsize> { 1511 defm "" : IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred), 1512 (IRInt<"vrev_predicated", [Vector, Predicate]> 1513 $a, revsize, $pred, $inactive)>; 1514} 1515 1516let params = T.All8 in { 1517 def vrev16q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 16)>; 1518 defm vrev16q: vrev_predicated<16>; 1519} 1520let params = !listconcat(T.All8, T.All16) in { 1521 def vrev32q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 32)>; 1522 defm vrev32q: vrev_predicated<32>; 1523} 1524let params = T.Usual in { 1525 def vrev64q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 64)>; 1526 defm vrev64q: vrev_predicated<64>; 1527} 1528 1529foreach desttype = T.All in { 1530 // We want a vreinterpretq between every pair of supported vector types 1531 // _except_ that there shouldn't be one from a type to itself. 1532 // 1533 // So this foldl expression implements what you'd write in Python as 1534 // [srctype for srctype in T.All if srctype != desttype] 1535 let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist, 1536 !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype]))) 1537 in { 1538 def "vreinterpretq_" # desttype: Intrinsic< 1539 VecOf<desttype>, (args Vector:$x), (vreinterpret $x, VecOf<desttype>)>; 1540 } 1541} 1542 1543let params = T.All in { 1544 let pnt = PNT_None in { 1545 def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b), 1546 (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0), 1547 $b, 1), Vector)>; 1548 def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>; 1549 } 1550 1551 // This is the polymorphic form of vuninitializedq, which takes no type 1552 // suffix, but takes an _unevaluated_ vector parameter and returns an 1553 // uninitialized vector of the same vector type. 1554 // 1555 // This intrinsic has no _non_-polymorphic form exposed to the user. But each 1556 // separately typed version of it still has to have its own clang builtin id, 1557 // which can't be called vuninitializedq_u32 or similar because that would 1558 // collide with the explicit nullary versions above. So I'm calling them 1559 // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that 1560 // full name never appears in the header file due to the polymorphicOnly 1561 // flag, and the _polymorphic suffix is omitted from the shortened name by 1562 // the custom PolymorphicNameType here. 1563 let polymorphicOnly = 1, nonEvaluating = 1, 1564 pnt = PolymorphicNameType<1, "polymorphic"> in { 1565 def vuninitializedq_polymorphic: Intrinsic< 1566 Vector, (args Vector), (undef Vector)>; 1567 } 1568 1569 def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane), 1570 (xelt_var $v, $lane)>; 1571 def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane), 1572 (ielt_var $v, $e, $lane)>; 1573} 1574 1575foreach desttype = !listconcat(T.Int16, T.Int32, T.Float) in { 1576 defvar is_dest_float = !eq(desttype.kind, "f"); 1577 defvar is_dest_unsigned = !eq(desttype.kind, "u"); 1578 // First immediate operand of the LLVM intrinsic 1579 defvar unsigned_flag = !if(is_dest_float, (unsignedflag Scalar), 1580 !if(is_dest_unsigned, V.True, V.False)); 1581 // For float->int conversions _n and _x_n intrinsics are not polymorphic 1582 // because the signedness of the destination type cannot be inferred. 1583 defvar pnt_nx = !if(is_dest_float, PNT_2Type, PNT_None); 1584 1585 let params = !if(is_dest_float, 1586 !if(!eq(desttype.size, 16), T.Int16, T.Int32), 1587 !if(!eq(desttype.size, 16), [f16], [f32])) in { 1588 let pnt = pnt_nx in 1589 def "vcvtq_n_"#desttype : Intrinsic<VecOf<desttype>, 1590 (args Vector:$a, imm_1toN:$b), 1591 (IRInt<"vcvt_fix", [VecOf<desttype>, Vector]> unsigned_flag, $a, $b)>; 1592 1593 defm "vcvtq" : IntrinsicMX<VecOf<desttype>, 1594 (args Vector:$a, imm_1toN:$b, Predicate:$p), 1595 (IRInt<"vcvt_fix_predicated", [VecOf<desttype>, Vector, Predicate]> 1596 unsigned_flag, $inactive, $a, $b, $p), 1597 1, "_n_"#desttype, PNT_2Type, pnt_nx>; 1598 } 1599} 1600 1601let params = T.Usual in { 1602let pnt = PNT_NType in 1603def vbrsrq_n: Intrinsic<Vector, (args Vector:$a, s32:$b), 1604 (IRInt<"vbrsr", [Vector]> $a, $b)>; 1605defm vbrsrq : IntrinsicMX<Vector, (args Vector:$a, s32:$b, Predicate:$pred), 1606 (IRInt<"vbrsr_predicated", [Vector, Predicate]> 1607 $inactive, $a, $b, $pred), 1, "_n", 1608 PNT_NType, PNT_NType>; 1609} 1610