1//===- arm_mve.td - ACLE intrinsic functions for MVE architecture ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the set of ACLE-specified source-level intrinsic
10// functions wrapping the MVE vector instruction set and scalar shift
11// operations.
12//
13// Refer to comments in arm_mve_defs.td for the infrastructure used in
14// here, and to MveEmitter.cpp for how those are used in turn to
15// generate code.
16//
17//===----------------------------------------------------------------------===//
18
19include "arm_mve_defs.td"
20
21let params = T.Usual in
22foreach n = [ 2, 4 ] in {
23  def "vst"#n#"q": Intrinsic<Void, (args Ptr<Scalar>, MultiVector<n>),
24                             (CustomCodegen<"VST24"> n:$NumVectors,
25                              "Intrinsic::arm_mve_vst"#n#"q":$IRIntr)>;
26  def "vld"#n#"q": Intrinsic<MultiVector<n>, (args CPtr<Scalar>),
27                             (CustomCodegen<"VLD24"> n:$NumVectors,
28                              "Intrinsic::arm_mve_vld"#n#"q":$IRIntr)>;
29}
30
31multiclass bit_op_fp<IRBuilder bitop> {
32def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
33   (bitcast (bitop (bitcast $a, UVector), (bitcast $b, UVector)), Vector)>;
34}
35
36multiclass bit_op_fp_with_inv<IRBuilder bitop> {
37def "": Intrinsic<Vector, (args Vector:$a, Vector:$b),
38   (bitcast (bitop (bitcast $a, UVector), (not (bitcast $b, UVector))), Vector)>;
39}
40
41let params = T.Signed in {
42def vqaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
43                              (IRIntBase<"sadd_sat", [Vector]> $a, $b)>;
44def vqsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
45                              (IRIntBase<"ssub_sat", [Vector]> $a, $b)>;
46let pnt = PNT_NType in {
47  def vqaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
48                                  (IRIntBase<"sadd_sat", [Vector]> $a, (splat $b))>;
49  def vqsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
50                                  (IRIntBase<"ssub_sat", [Vector]> $a, (splat $b))>;
51}
52}
53let params = T.Unsigned in {
54def vqaddq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
55                                (IRIntBase<"uadd_sat", [Vector]> $a, $b)>,
56              NameOverride<"vqaddq">;
57def vqsubq_u: Intrinsic<Vector, (args Vector:$a, Vector:$b),
58                                (IRIntBase<"usub_sat", [Vector]> $a, $b)>,
59              NameOverride<"vqsubq">;
60let pnt = PNT_NType in {
61  def vqaddq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
62                                    (IRIntBase<"uadd_sat", [Vector]> $a, (splat $b))>,
63                  NameOverride<"vqaddq_n">;
64  def vqsubq_u_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
65                                    (IRIntBase<"usub_sat", [Vector]> $a, (splat $b))>,
66                  NameOverride<"vqsubq_n">;
67}
68}
69
70// Some intrinsics below are implemented not as IR fragments, but as
71// special-purpose IR intrinsics. This is because such a general form
72// (such as NEON uses) required a variable-width vector size, and we are
73// restricted to 128 bit. Although we can possibly get clever with lane
74// operations, the consequent IR representation would be very hard to
75// write sensibly. In particular, doubling a vector's width would be a
76// mess. Other intrinsics just don't translate nicely into IR.
77let params = T.Int in {
78def vaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (add $a, $b)>;
79def vhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
80                              (IRInt<"vhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
81def vrhaddq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
82                               (IRInt<"vrhadd", [Vector]> $a, $b, (unsignedflag Scalar))>;
83def vandq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, $b)>;
84def vbicq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (and $a, (not $b))>;
85def veorq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (xor $a, $b)>;
86def vornq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, (not $b))>;
87def vorrq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (or $a, $b)>;
88def vsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (sub $a, $b)>;
89def vhsubq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
90                              (IRInt<"vhsub", [Vector]> $a, $b, (unsignedflag Scalar))>;
91def vmulq: Intrinsic<Vector, (args Vector:$a, Vector:$b), (mul $a, $b)>;
92def vmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
93                              (IRInt<"vmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
94def vrmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
95                               (IRInt<"vrmulh", [Vector]> $a, $b, (unsignedflag Scalar))>;
96def vmullbq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
97                                      (IRInt<"vmull", [DblVector, Vector]>
98                                       $a, $b, (unsignedflag Scalar), 0)>;
99def vmulltq_int: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
100                                      (IRInt<"vmull", [DblVector, Vector]>
101                                       $a, $b, (unsignedflag Scalar), 1)>;
102let pnt = PNT_NType in {
103  def vaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
104                                 (add $a, (splat $b))>;
105  def vsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
106                                 (sub $a, (splat $b))>;
107  def vmulq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
108                                 (mul $a, (splat $b))>;
109  def vhaddq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
110                                  (IRInt<"vhadd", [Vector]> $a, (splat $b),
111                                  (unsignedflag Scalar))>;
112  def vhsubq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
113                                  (IRInt<"vhsub", [Vector]> $a, (splat $b),
114                                  (unsignedflag Scalar))>;
115}
116}
117let params = T.Signed in {
118def vqdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
119                                (IRInt<"vqdmulh", [Vector]> $a, $b)>;
120def vqrdmulhq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
121                                 (IRInt<"vqrdmulh", [Vector]> $a, $b)>;
122let pnt = PNT_NType in {
123  def vqdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
124                                    (IRInt<"vqdmulh", [Vector]> $a, (splat $b))>;
125  def vqrdmulhq_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
126                                    (IRInt<"vqrdmulh", [Vector]> $a, (splat $b))>;
127}
128}
129
130let params = T.Poly, overrideKindLetter = "p" in {
131def vmullbq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
132                                       (IRInt<"vmull_poly", [DblVector, Vector]>
133                                        $a, $b, 0)>;
134def vmulltq_poly: Intrinsic<DblVector, (args Vector:$a, Vector:$b),
135                                       (IRInt<"vmull_poly", [DblVector, Vector]>
136                                        $a, $b, 1)>;
137}
138
139let params = T.Float in {
140def vaddqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fadd $a, $b)>,
141            NameOverride<"vaddq">;
142defm vandqf: bit_op_fp<and>, NameOverride<"vandq">;
143defm vbicqf: bit_op_fp_with_inv<and>, NameOverride<"vbicq">;
144defm veorqf: bit_op_fp<xor>, NameOverride<"veorq">;
145defm vornqf: bit_op_fp_with_inv<or>, NameOverride<"vornq">;
146defm vorrqf: bit_op_fp<or>, NameOverride<"vorrq">;
147def vsubqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fsub $a, $b)>,
148            NameOverride<"vsubq">;
149def vmulqf: Intrinsic<Vector, (args Vector:$a, Vector:$b), (fmul $a, $b)>,
150            NameOverride<"vmulq">;
151
152let pnt = PNT_NType in {
153  def vaddqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
154                                  (fadd $a, (splat $b))>,
155                NameOverride<"vaddq_n">;
156  def vsubqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
157                                  (fsub $a, (splat $b))>,
158                NameOverride<"vsubq_n">;
159  def vmulqf_n: Intrinsic<Vector, (args Vector:$a, unpromoted<Scalar>:$b),
160                                  (fmul $a, (splat $b))>,
161                NameOverride<"vmulq_n">;
162}
163}
164
165multiclass FMA<bit add> {
166  // FMS instructions are defined in the ArmARM as if they negate the
167  // second multiply input.
168  defvar m2_cg = !if(add, (id $m2), (fneg $m2));
169
170  defvar unpred_cg = (IRIntBase<"fma", [Vector]> $m1, m2_cg, $addend);
171  defvar pred_cg   = (IRInt<"fma_predicated", [Vector, Predicate]>
172                          $m1, m2_cg, $addend, $pred);
173
174  def q: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2),
175                   unpred_cg>;
176
177  def q_m: Intrinsic<Vector, (args Vector:$addend, Vector:$m1, Vector:$m2,
178                                   Predicate:$pred), pred_cg>;
179
180  // Only FMA has the vector/scalar variants, not FMS
181  if add then let pnt = PNT_NType in {
182
183    def q_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1,
184                                     unpromoted<Scalar>:$m2_s),
185                     (seq (splat $m2_s):$m2, unpred_cg)>;
186    def sq_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2,
187                                      unpromoted<Scalar>:$addend_s),
188                        (seq (splat $addend_s):$addend, unpred_cg)>;
189    def q_m_n: Intrinsic<Vector, (args Vector:$addend, Vector:$m1,
190                                       unpromoted<Scalar>:$m2_s,
191                                       Predicate:$pred),
192                         (seq (splat $m2_s):$m2, pred_cg)>;
193    def sq_m_n: Intrinsic<Vector, (args Vector:$m1, Vector:$m2,
194                                        unpromoted<Scalar>:$addend_s,
195                                        Predicate:$pred),
196                          (seq (splat $addend_s):$addend, pred_cg)>;
197  }
198}
199
200let params = T.Float in {
201  defm vfma: FMA<1>;
202  defm vfms: FMA<0>;
203}
204
205let params = T.Int, pnt = PNT_NType in {
206  def vmlaq_n: Intrinsic<
207    Vector, (args Vector:$addend, Vector:$m1, unpromoted<Scalar>:$m2_s),
208    (add (mul $m1, (splat $m2_s)), $addend)>;
209  def vmlasq_n: Intrinsic<
210    Vector, (args Vector:$m1, Vector:$m2, unpromoted<Scalar>:$addend_s),
211    (add (mul $m1, $m2), (splat $addend_s))>;
212
213  def vmlaq_m_n: Intrinsic<
214    Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred),
215    (IRInt<"vmla_n_predicated", [Vector, Predicate]>
216        $addend, $m1, $m2_s, $pred)>;
217  def vmlasq_m_n: Intrinsic<
218    Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred),
219    (IRInt<"vmlas_n_predicated", [Vector, Predicate]>
220        $m1, $m2, $addend_s, $pred)>;
221}
222
223multiclass VQDMLA {
224  def hq_n: Intrinsic<
225    Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s),
226    (IRInt<NAME # "h", [Vector]> $addend, $m1, $m2_s)>;
227  def shq_n: Intrinsic<
228    Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s),
229    (IRInt<NAME # "sh", [Vector]> $m1, $m2, $addend_s)>;
230
231  def hq_m_n: Intrinsic<
232    Vector, (args Vector:$addend, Vector:$m1, Scalar:$m2_s, Predicate:$pred),
233    (IRInt<NAME # "h_predicated", [Vector, Predicate]>
234         $addend, $m1, $m2_s, $pred)>;
235  def shq_m_n: Intrinsic<
236    Vector, (args Vector:$m1, Vector:$m2, Scalar:$addend_s, Predicate:$pred),
237    (IRInt<NAME # "sh_predicated", [Vector, Predicate]>
238         $m1, $m2, $addend_s, $pred)>;
239}
240
241let params = T.Signed, pnt = PNT_NType in {
242  defm vqdmla: VQDMLA;
243  defm vqrdmla: VQDMLA;
244}
245
246multiclass VQDMLAD<int exchange, int round, int subtract> {
247  def "": Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
248       (IRInt<"vqdmlad", [Vector]> $a, $b, $c,
249            (u32 exchange), (u32 round), (u32 subtract))>;
250  def _m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
251                                  Predicate:$pred),
252       (IRInt<"vqdmlad_predicated", [Vector, Predicate]> $a, $b, $c,
253            (u32 exchange), (u32 round), (u32 subtract), $pred)>;
254}
255let params = T.Signed in {
256  defm vqdmladhq:   VQDMLAD<0, 0, 0>;
257  defm vqdmladhxq:  VQDMLAD<1, 0, 0>;
258  defm vqdmlsdhq:   VQDMLAD<0, 0, 1>;
259  defm vqdmlsdhxq:  VQDMLAD<1, 0, 1>;
260  defm vqrdmladhq:  VQDMLAD<0, 1, 0>;
261  defm vqrdmladhxq: VQDMLAD<1, 1, 0>;
262  defm vqrdmlsdhq:  VQDMLAD<0, 1, 1>;
263  defm vqrdmlsdhxq: VQDMLAD<1, 1, 1>;
264}
265
266let params = !listconcat(T.Int16, T.Int32) in {
267  let pnt = PNT_None in {
268    def vmvnq_n: Intrinsic<Vector, (args imm_simd_vmvn:$imm),
269                           (not (splat (Scalar $imm)))>;
270  }
271  defm vmvnq: IntrinsicMX<Vector, (args imm_simd_vmvn:$imm, Predicate:$pred),
272                     (select $pred, (not (splat (Scalar $imm))), $inactive),
273                     1, "_n", PNT_NType, PNT_None>;
274  let pnt = PNT_NType in {
275    def vbicq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm),
276                           (and $v, (not (splat (Scalar $imm))))>;
277    def vorrq_n: Intrinsic<Vector, (args Vector:$v, imm_simd_restrictive:$imm),
278                           (or $v, (splat (Scalar $imm)))>;
279  }
280  def vbicq_m_n: Intrinsic<
281    Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred),
282    (select $pred, (and $v, (not (splat (Scalar $imm)))), $v)>;
283  def vorrq_m_n: Intrinsic<
284    Vector, (args Vector:$v, imm_simd_restrictive:$imm, Predicate:$pred),
285    (select $pred, (or $v, (splat (Scalar $imm))), $v)>;
286}
287
288let params = T.Usual in {
289  let pnt = PNT_None in
290    def vdupq_n: Intrinsic<Vector, (args unpromoted<Scalar>:$s), (splat $s)>;
291
292  defm vdupq: IntrinsicMX<
293      Vector, (args unpromoted<Scalar>:$s, Predicate:$pred),
294      (select $pred, (splat $s), $inactive), 1, "_n", PNT_NType, PNT_None>;
295}
296
297multiclass vxdup_mc<dag paramsIn, dag paramsOut> {
298  defvar UnpredInt     = IRInt<NAME, [Vector]>;
299  defvar PredInt       = IRInt<NAME # "_predicated", [Vector, Predicate]>;
300  defvar UnpredIntCall = !con((UnpredInt $base), paramsOut);
301  defvar PredIntCall   = !con((PredInt $inactive, $base), paramsOut, (? $pred));
302
303  // Straightforward case with neither writeback nor predication
304  let pnt = PNT_N in
305    def q_n: Intrinsic<Vector, !con((args u32:$base), paramsIn),
306                       (xval UnpredIntCall, 0)>;
307
308  // Predicated form without writeback
309  defm q: IntrinsicMX<
310      Vector, !con((args u32:$base), paramsIn, (? Predicate:$pred)),
311      (xval PredIntCall, 0), 1, "_n", PNT_NType, PNT_N>;
312
313  // Writeback without predication
314  let pnt = PNT_WB in
315    def q_wb: Intrinsic<
316      Vector, !con((args Ptr<u32>:$baseaddr), paramsIn),
317      (seq (load $baseaddr):$base,
318           UnpredIntCall:$pair,
319           (store (xval $pair, 1), $baseaddr),
320           (xval $pair, 0))>;
321
322  // Both writeback and predicated
323  defm q: IntrinsicMX<
324      Vector, !con((args Ptr<u32>:$baseaddr), paramsIn, (? Predicate:$pred)),
325      (seq (load $baseaddr):$base,
326           PredIntCall:$pair,
327           (store (xval $pair, 1), $baseaddr),
328           (xval $pair, 0)), 1, "_wb", PNT_WBType, PNT_WB>;
329}
330
331let params = T.Unsigned in {
332  defm vidup:  vxdup_mc<(?             imm_1248:$step), (?         $step)>;
333  defm vddup:  vxdup_mc<(?             imm_1248:$step), (?         $step)>;
334  defm viwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
335  defm vdwdup: vxdup_mc<(? u32:$limit, imm_1248:$step), (? $limit, $step)>;
336}
337
338let params = T.Int in {
339  def vmvnq: Intrinsic<Vector, (args Vector:$a),
340                       (xor $a, (uint_max Vector))>;
341  defm vmvnq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
342    (IRInt<"mvn_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
343  def vclzq: Intrinsic<Vector, (args Vector:$a),
344                       (IRIntBase<"ctlz", [Vector]> $a, (i1 0))>;
345  defm vclzq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
346    (IRInt<"clz_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
347}
348let params = T.Signed in {
349  def vclsq: Intrinsic<Vector, (args Vector:$a), (IRInt<"vcls", [Vector]> $a)>;
350  defm vclsq: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
351    (IRInt<"cls_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
352
353  def vnegq: Intrinsic<Vector, (args Vector:$a),
354                       (sub (zeroinit Vector), $a)>;
355  def vabsq: Intrinsic<Vector, (args Vector:$a),
356                       (select (icmp_slt $a, (zeroinit Vector)),
357                               (sub (zeroinit Vector), $a), $a)>;
358  def vqnegq: Intrinsic<Vector, (args Vector:$a),
359                        (select (icmp_eq $a, (int_min Vector)),
360                                (int_max Vector),
361                                (sub (zeroinit Vector), $a))>;
362  def vqabsq: Intrinsic<Vector, (args Vector:$a),
363                        (select (icmp_sgt $a, (zeroinit Vector)), $a,
364                                (select (icmp_eq $a, (int_min Vector)),
365                                        (int_max Vector),
366                                        (sub (zeroinit Vector), $a)))>;
367
368  foreach name = ["qneg", "qabs"] in {
369    defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
370      (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive),
371      0 /* no _x variant for saturating intrinsics */>;
372  }
373}
374let params = !listconcat(T.Signed, T.Float) in {
375  foreach name = ["neg", "abs"] in {
376    defm v#name#q: IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
377      (IRInt<name#"_predicated", [Vector, Predicate]> $a, $pred, $inactive)>;
378  }
379}
380let params = T.Float in {
381  def vnegq_f: Intrinsic<Vector, (args Vector:$a), (fneg $a)>,
382               NameOverride<"vnegq">;
383  def vabsq_f: Intrinsic<Vector, (args Vector:$a),
384               (IRIntBase<"fabs", [Vector]> $a)>, NameOverride<"vabsq">;
385}
386
387// The bitcasting below is not overcomplicating the IR because while
388// Vector and UVector may be different vector types at the C level i.e.
389// vectors of same size signed/unsigned ints. Once they're lowered
390// to IR, they are just bit vectors with no sign at all, so the
391// bitcasts will be automatically elided by IRBuilder.
392multiclass predicated_bit_op_fp<string int_op> {
393def "": Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
394                                Predicate:$pred),
395    (bitcast (IRInt<int_op, [UVector, Predicate]>
396                    (bitcast $a, UVector),
397                    (bitcast $b, UVector),
398                    $pred,
399                    (bitcast $inactive, UVector)), Vector)>;
400}
401
402// Plain intrinsics
403let params = T.Usual in {
404def vabdq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
405                     (IRInt<"vabd", [Vector]> $a, $b, (unsignedflag Scalar))>;
406}
407
408multiclass VectorVectorArithmetic<string operation, dag extraArgs = (?),
409                                  bit wantXVariant = 1> {
410  defm "" : IntrinsicMX<
411      Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
412      !con((IRInt<operation, [Vector, Predicate]> $a, $b),
413           extraArgs, (? $pred, $inactive)), wantXVariant>;
414}
415
416multiclass VectorScalarArithmetic<string operation, string basename,
417                                  dag extraArgs = (?),
418                                  bit wantXVariant = 1> {
419  defm "" : IntrinsicMXNameOverride<
420      Vector, (args Vector:$a, unpromoted<Scalar>:$b, Predicate:$pred),
421      !con((IRInt<operation, [Vector, Predicate]> $a, (splat $b)),
422           extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n",
423           PNT_NType, PNT_NType>;
424}
425
426multiclass VectorVectorArithmeticBitcast<string operation> {
427  defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b,
428                                 Predicate:$pred),
429                                (bitcast (IRInt<operation, [UVector, Predicate]>
430                                          (bitcast $a, UVector),
431                                          (bitcast $b, UVector),
432                                          $pred,
433                                          (bitcast $inactive, UVector)), Vector)>;
434}
435
436// Predicated intrinsics
437let params = T.Usual in {
438  defm vabdq : VectorVectorArithmetic<"abd_predicated", (? (unsignedflag Scalar))>;
439  defm vaddq : VectorVectorArithmetic<"add_predicated">;
440  defm vsubq : VectorVectorArithmetic<"sub_predicated">;
441  defm vmulq : VectorVectorArithmetic<"mul_predicated">;
442  defm vandq : VectorVectorArithmeticBitcast<"and_predicated">;
443  defm vbicq : VectorVectorArithmeticBitcast<"bic_predicated">;
444  defm veorq : VectorVectorArithmeticBitcast<"eor_predicated">;
445  defm vornq : VectorVectorArithmeticBitcast<"orn_predicated">;
446  defm vorrq : VectorVectorArithmeticBitcast<"orr_predicated">;
447
448  defm : VectorScalarArithmetic<"add_predicated", "vaddq">;
449  defm : VectorScalarArithmetic<"sub_predicated", "vsubq">;
450  defm : VectorScalarArithmetic<"mul_predicated", "vmulq">;
451}
452
453multiclass DblVectorVectorArithmetic<string operation, dag extraArgs = (?),
454                                     bit wantXVariant = 1> {
455  defm "" : IntrinsicMX<
456      DblVector, (args Vector:$a, Vector:$b, DblPredicate:$pred),
457      !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, $b),
458           extraArgs, (? $pred, $inactive)), wantXVariant>;
459}
460
461multiclass DblVectorScalarArithmetic<string operation, string basename,
462                                     dag extraArgs = (?),
463                                     bit wantXVariant = 1> {
464  defm "" : IntrinsicMXNameOverride<
465      DblVector, (args Vector:$a, unpromoted<Scalar>:$b, DblPredicate:$pred),
466      !con((IRInt<operation, [DblVector, Vector, DblPredicate]> $a, (splat $b)),
467           extraArgs, (? $pred, $inactive)), basename, wantXVariant, "_n",
468           PNT_NType, PNT_NType>;
469}
470
471// Predicated intrinsics - Int types only
472let params = T.Int in {
473  defm vminq : VectorVectorArithmetic<"min_predicated", (? (unsignedflag Scalar))>;
474  defm vmaxq : VectorVectorArithmetic<"max_predicated", (? (unsignedflag Scalar))>;
475  defm vmulhq : VectorVectorArithmetic<"mulh_predicated", (? (unsignedflag Scalar))>;
476  defm vrmulhq : VectorVectorArithmetic<"rmulh_predicated", (? (unsignedflag Scalar))>;
477  defm vqaddq : VectorVectorArithmetic<"qadd_predicated", (? (unsignedflag Scalar)), 0>;
478  defm vhaddq : VectorVectorArithmetic<"hadd_predicated", (? (unsignedflag Scalar))>;
479  defm vrhaddq : VectorVectorArithmetic<"rhadd_predicated", (? (unsignedflag Scalar))>;
480  defm vqsubq : VectorVectorArithmetic<"qsub_predicated", (? (unsignedflag Scalar)), 0>;
481  defm vhsubq : VectorVectorArithmetic<"hsub_predicated", (? (unsignedflag Scalar))>;
482  defm vmullbq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 0))>;
483  defm vmulltq_int : DblVectorVectorArithmetic<"mull_int_predicated", (? (unsignedflag Scalar), (u32 1))>;
484
485  defm : VectorScalarArithmetic<"qadd_predicated", "vqaddq", (? (unsignedflag Scalar)), 0>;
486  defm : VectorScalarArithmetic<"hadd_predicated", "vhaddq", (? (unsignedflag Scalar))>;
487  defm : VectorScalarArithmetic<"qsub_predicated", "vqsubq", (? (unsignedflag Scalar)), 0>;
488  defm : VectorScalarArithmetic<"hsub_predicated", "vhsubq", (? (unsignedflag Scalar))>;
489}
490let params = T.Signed in {
491  defm vqdmulhq : VectorVectorArithmetic<"qdmulh_predicated", (?), 0>;
492  defm vqrdmulhq : VectorVectorArithmetic<"qrdmulh_predicated", (?), 0>;
493  def vminaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred),
494                                   (IRInt<"vmina_predicated", [UVector,Predicate]> $a, $b, $pred)>;
495  def vmaxaq_m: Intrinsic<UVector, (args UVector:$a, Vector:$b, Predicate:$pred),
496                                   (IRInt<"vmaxa_predicated", [UVector,Predicate]> $a, $b, $pred)>;
497
498  defm : VectorScalarArithmetic<"qdmulh_predicated", "vqdmulhq", (?), 0>;
499  defm : VectorScalarArithmetic<"qrdmulh_predicated", "vqrdmulhq", (?), 0>;
500}
501
502let params = T.Poly, overrideKindLetter = "p" in {
503  defm vmullbq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 0))>;
504  defm vmulltq_poly : DblVectorVectorArithmetic<"mull_poly_predicated", (? (u32 1))>;
505}
506
507let params = [s16, s32] in {
508  def  vqdmullbq:  Intrinsic<DblVector, (args Vector:$a, Vector:$b),
509                             (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 0)>;
510  def  vqdmulltq:  Intrinsic<DblVector, (args Vector:$a, Vector:$b),
511                             (IRInt<"vqdmull", [DblVector, Vector]> $a, $b, 1)>;
512  defm vqdmullbq:  DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 0)), 0>;
513  defm vqdmulltq:  DblVectorVectorArithmetic<"vqdmull_predicated", (? (u32 1)), 0>;
514
515  let pnt = PNT_NType in {
516    def vqdmullbq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b),
517                               (IRInt<"vqdmull", [DblVector, Vector]>
518                                $a, (splat $b), 0)>;
519    def vqdmulltq_n: Intrinsic<DblVector, (args Vector:$a, unpromoted<Scalar>:$b),
520                               (IRInt<"vqdmull", [DblVector, Vector]>
521                                $a, (splat $b), 1)>;
522  }
523  defm vqdmullbq_n: DblVectorScalarArithmetic<"vqdmull_predicated",
524                                              "vqdmullbq", (? (u32 0)), 0>;
525  defm vqdmulltq_n: DblVectorScalarArithmetic<"vqdmull_predicated",
526                                              "vqdmulltq", (? (u32 1)), 0>;
527}
528
529// Predicated intrinsics - Float types only
530let params = T.Float in {
531  defm vminnmq : VectorVectorArithmetic<"min_predicated", (? (u32 0))>;
532  defm vmaxnmq : VectorVectorArithmetic<"max_predicated", (? (u32 0))>;
533  def vminnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
534                                    (IRInt<"vminnma_predicated", [Vector,Predicate]> $a, $b, $pred)>;
535  def vmaxnmaq_m: Intrinsic<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
536                                    (IRInt<"vmaxnma_predicated", [Vector,Predicate]> $a, $b, $pred)>;
537}
538
539multiclass Reduction<Type Accumulator, string basename, list<Type> basetypes,
540                     bit needSign = 0,
541                     dag postCG = (seq (id $ret)),
542                     dag accArg = (args Accumulator:$prev),
543                     dag preCG = (seq)> {
544  defvar intArgsBase   = (? $prev, $vec);
545  defvar intArgsUnpred = !con(intArgsBase,
546                              !if(needSign, (? (unsignedflag Scalar)), (?)));
547  defvar intArgsPred   = !con(intArgsUnpred, (? $pred));
548  defvar intUnpred     = !setdagop(intArgsUnpred, IRInt<basename, basetypes>);
549  defvar intPred       = !setdagop(intArgsPred, IRInt<
550            basename#"_predicated", !listconcat(basetypes, [Predicate])>);
551
552  def "": Intrinsic<
553    Accumulator, !con(accArg, (args Vector:$vec)),
554    !con(preCG, (seq intUnpred:$ret), postCG)>;
555  def _p: Intrinsic<
556    Accumulator, !con(accArg, (args Vector:$vec, Predicate:$pred)),
557    !con(preCG, (seq intPred:$ret), postCG)>;
558}
559
560let params = T.Int in {
561defm vminvq: Reduction<Scalar, "minv", [Vector], 1, (seq (Scalar $ret))>;
562defm vmaxvq: Reduction<Scalar, "maxv", [Vector], 1, (seq (Scalar $ret))>;
563}
564
565let params = T.Signed in {
566defm vminavq: Reduction<UScalar, "minav", [Vector], 0, (seq (UScalar $ret))>;
567defm vmaxavq: Reduction<UScalar, "maxav", [Vector], 0, (seq (UScalar $ret))>;
568}
569
570let params = T.Float in {
571defm vminnmvq: Reduction<Scalar, "minnmv", [Scalar, Vector]>;
572defm vmaxnmvq: Reduction<Scalar, "maxnmv", [Scalar, Vector]>;
573defm vminnmavq: Reduction<Scalar, "minnmav", [Scalar, Vector]>;
574defm vmaxnmavq: Reduction<Scalar, "maxnmav", [Scalar, Vector]>;
575}
576
577foreach half = [ "b", "t" ] in {
578  defvar halfconst = !if(!eq(half, "b"), 0, 1);
579
580  let params = [f32], pnt = PNT_None in {
581    def vcvt#half#q_f16: Intrinsic<
582      VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a),
583      (IRInt<"vcvt_narrow"> $inactive, $a, halfconst)>;
584    def vcvt#half#q_m_f16: Intrinsic<
585      VecOf<f16>, (args VecOf<f16>:$inactive, Vector:$a, PredOf<f32>:$pred),
586      (IRInt<"vcvt_narrow_predicated"> $inactive, $a, halfconst, $pred)>;
587  } // params = [f32], pnt = PNT_None
588
589  let params = [f16], pnt = PNT_None in {
590    def vcvt#half#q_f32: Intrinsic<VecOf<f32>, (args Vector:$a),
591      (IRInt<"vcvt_widen"> $a, halfconst)>;
592    defm vcvt#half#q: IntrinsicMX<
593      VecOf<f32>, (args Vector:$a, PredOf<f32>:$pred),
594      (IRInt<"vcvt_widen_predicated"> $inactive, $a, halfconst, $pred),
595      1, "_f32">;
596  } // params = [f16], pnt = PNT_None
597} // loop over half = "b", "t"
598
599multiclass float_int_conversions<Type FScalar, Type IScalar, IRBuilderBase ftoi, IRBuilderBase itof> {
600  defvar FVector = VecOf<FScalar>;
601  defvar IVector = VecOf<IScalar>;
602
603  let params = [IScalar] in {
604    let pnt = PNT_2Type in {
605      def : Intrinsic<FVector, (args IVector:$a), (itof $a, FVector)>,
606            NameOverride<"vcvtq_" # FScalar>;
607    }
608    defm vcvtq: IntrinsicMX<FVector, (args IVector:$a, Predicate:$pred),
609        (IRInt<"vcvt_fp_int_predicated", [FVector, IVector, Predicate]>
610            $a, (unsignedflag IScalar), $pred, $inactive),
611        1, "_" # FScalar, PNT_2Type, PNT_2Type>;
612  }
613  let params = [FScalar] in {
614    let pnt = PNT_None in {
615      def : Intrinsic<IVector, (args FVector:$a), (ftoi $a, IVector)>,
616            NameOverride<"vcvtq_" # IScalar>;
617
618      foreach suffix = ["a","n","p","m"] in
619        def : Intrinsic<IVector, (args FVector:$a),
620                        (IRInt<"vcvt"#suffix, [IVector, FVector]>
621                            (unsignedflag IScalar), $a)>,
622              NameOverride<"vcvt"#suffix#"q_" # IScalar>;
623    }
624    defm vcvtq: IntrinsicMX<IVector, (args FVector:$a, Predicate:$pred),
625        (IRInt<"vcvt_fp_int_predicated", [IVector, FVector, Predicate]>
626            $a, (unsignedflag IScalar), $pred, $inactive),
627        1, "_" # IScalar, PNT_2Type, PNT_None>;
628
629    foreach suffix = ["a","n","p","m"] in {
630      defm "vcvt"#suffix#"q" : IntrinsicMX<
631          IVector, (args FVector:$a, Predicate:$pred),
632          (IRInt<"vcvt"#suffix#"_predicated", [IVector, FVector, Predicate]>
633              (unsignedflag IScalar), $inactive, $a, $pred),
634          1, "_" # IScalar, PNT_2Type, PNT_None>;
635    }
636  }
637}
638
639defm "" : float_int_conversions<f32, u32, fptoui, uitofp>;
640defm "" : float_int_conversions<f16, u16, fptoui, uitofp>;
641defm "" : float_int_conversions<f32, s32, fptosi, sitofp>;
642defm "" : float_int_conversions<f16, s16, fptosi, sitofp>;
643
644multiclass vmovl<bit top> {
645  let params = [s8, u8, s16, u16] in {
646    def "": Intrinsic<DblVector, (args Vector:$a),
647      (extend (unzip $a, top), DblVector, (unsignedflag Scalar))>;
648    defm "": IntrinsicMX<DblVector, (args Vector:$a, DblPredicate:$pred),
649        (IRInt<"vmovl_predicated", [DblVector, Vector, DblPredicate]>
650         $a, (unsignedflag Scalar), top, $pred, $inactive)>;
651  }
652}
653
654defm vmovlbq: vmovl<0>;
655defm vmovltq: vmovl<1>;
656
657multiclass vmovn<bit top, dag wide_result> {
658  let params = [s16, u16, s32, u32] in {
659    def "": Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a),
660      (trunc wide_result, HalfVector)>;
661    def _m: Intrinsic<HalfVector, (args HalfVector:$inactive, Vector:$a,
662                                        Predicate:$pred),
663      (IRInt<"vmovn_predicated", [HalfVector, Vector, Predicate]>
664          $inactive, $a, top, $pred)>;
665  }
666}
667
668defm vmovntq: vmovn<1, (zip (vreinterpret $inactive, Vector), $a)>;
669defm vmovnbq: vmovn<0,
670   (zip $a, (vreinterpret (vrev $inactive, (bitsize Scalar)), Vector))>;
671
672multiclass vqmovn<bit top, Type RetScalar> {
673  defvar RetVector = VecOf<RetScalar>;
674
675  let params = [s16, u16, s32, u32] in {
676    def : Intrinsic<
677      RetVector, (args RetVector:$inactive, Vector:$a),
678      (IRInt<"vqmovn", [RetVector, Vector]>
679          $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar), top)>,
680      NameOverride<NAME>;
681    def: Intrinsic<
682      RetVector, (args RetVector:$inactive, Vector:$a, Predicate:$pred),
683      (IRInt<"vqmovn_predicated", [RetVector, Vector, Predicate]>
684          $inactive, $a, (unsignedflag RetScalar), (unsignedflag Scalar),
685          top, $pred)>,
686      NameOverride<NAME # "_m">;
687  }
688}
689
690let params = [s16, s32, u16, u32] in {
691  defm vqmovntq: vqmovn<1, HalfScalar>;
692  defm vqmovnbq: vqmovn<0, HalfScalar>;
693}
694let params = [s16, s32] in {
695  defm vqmovuntq: vqmovn<1, UHalfScalar>;
696  defm vqmovunbq: vqmovn<0, UHalfScalar>;
697}
698
699multiclass vrnd<IRIntBase ir_int, string suffix> {
700  let params = T.Float in {
701    def "": Intrinsic<Vector, (args Vector:$a), (ir_int $a)>;
702    defm "": IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
703        (IRInt<"vrint"#suffix#"_predicated", [Vector, Predicate]>
704            $a, $pred, $inactive)>;
705  }
706}
707
708defm vrndq:  vrnd<IRIntBase<"trunc", [Vector]>, "z">;
709defm vrndmq: vrnd<IRIntBase<"floor", [Vector]>, "m">;
710defm vrndpq: vrnd<IRIntBase<"ceil",  [Vector]>, "p">;
711defm vrndaq: vrnd<IRIntBase<"round", [Vector]>, "a">;
712defm vrndxq: vrnd<IRIntBase<"rint",  [Vector]>, "x">;
713defm vrndnq: vrnd<IRInt<"vrintn",    [Vector]>, "n">;
714
715multiclass compare_with_pred<string condname, dag arguments,
716                             dag cmp, string suffix> {
717  // Make the predicated and unpredicated versions of a single comparison.
718  def: Intrinsic<Predicate, arguments,
719                 (u16 (IRInt<"pred_v2i", [Predicate]> cmp))>,
720       NameOverride<"vcmp" # condname # "q" # suffix>;
721  def: Intrinsic<Predicate, !con(arguments, (args Predicate:$inpred)),
722                 (u16 (IRInt<"pred_v2i", [Predicate]> (and $inpred, cmp)))>,
723       NameOverride<"vcmp" # condname # "q_m" # suffix>;
724}
725
726multiclass compare<string condname, IRBuilder cmpop> {
727  // Make all four variants of a comparison: the vector/vector and
728  // vector/scalar forms, each using compare_with_pred to make a
729  // predicated and unpredicated version.
730  defm: compare_with_pred<condname, (args Vector:$va, Vector:$vb),
731                          (cmpop $va, $vb), "">;
732  let pnt = PNT_NType in {
733    defm: compare_with_pred<condname, (args Vector:$va, unpromoted<Scalar>:$sb),
734                            (cmpop $va, (splat $sb)), "_n">;
735  }
736}
737let params = T.Int in {
738  defm: compare<"eq", icmp_eq>;
739  defm: compare<"ne", icmp_ne>;
740}
741let params = T.Signed in {
742  defm: compare<"gt", icmp_sgt>;
743  defm: compare<"ge", icmp_sge>;
744  defm: compare<"lt", icmp_slt>;
745  defm: compare<"le", icmp_sle>;
746}
747let params = T.Unsigned in {
748  defm: compare<"hi", icmp_ugt>;
749  defm: compare<"cs", icmp_uge>;
750}
751let params = T.Float in {
752  defm: compare<"eq", fcmp_eq>;
753  defm: compare<"ne", fcmp_ne>;
754  defm: compare<"gt", fcmp_gt>;
755  defm: compare<"ge", fcmp_ge>;
756  defm: compare<"lt", fcmp_lt>;
757  defm: compare<"le", fcmp_le>;
758}
759
760let params = T.Signed in {
761  def vminq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
762                               (select (icmp_sle $a, $b), $a, $b)>;
763  def vmaxq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
764                               (select (icmp_sge $a, $b), $a, $b)>;
765  def vminaq: Intrinsic<UVector, (args UVector:$a, Vector:$b),
766                                 (seq (select (icmp_slt $b, (zeroinit Vector)),
767                                              (sub (zeroinit Vector), $b), $b):$absb,
768                                      (select (icmp_ule $a, $absb), $a, $absb))>;
769  def vmaxaq: Intrinsic<UVector, (args UVector:$a, Vector:$b),
770                                 (seq (select (icmp_slt $b, (zeroinit Vector)),
771                                              (sub (zeroinit Vector), $b), $b):$absb,
772                                      (select (icmp_uge $a, $absb), $a, $absb))>;
773}
774let params = T.Unsigned in {
775  def vminqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
776                                (select (icmp_ule $a, $b), $a, $b)>,
777              NameOverride<"vminq">;
778  def vmaxqu: Intrinsic<Vector, (args Vector:$a, Vector:$b),
779                                (select (icmp_uge $a, $b), $a, $b)>,
780              NameOverride<"vmaxq">;
781}
782let params = T.Float in {
783  def vminnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
784                                 (IRIntBase<"minnum", [Vector]> $a, $b)>;
785  def vmaxnmq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
786                                 (IRIntBase<"maxnum", [Vector]> $a, $b)>;
787  def vminnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
788                                  (IRIntBase<"minnum", [Vector]>
789                                   (IRIntBase<"fabs", [Vector]> $a),
790                                   (IRIntBase<"fabs", [Vector]> $b))>;
791  def vmaxnmaq: Intrinsic<Vector, (args Vector:$a, Vector:$b),
792                                  (IRIntBase<"maxnum", [Vector]>
793                                   (IRIntBase<"fabs", [Vector]> $a),
794                                   (IRIntBase<"fabs", [Vector]> $b))>;
795}
796
797def vpselq: Intrinsic<Vector, (args Vector:$t, Vector:$f, Predicate:$pred),
798                      (select $pred, $t, $f)> { let params = T.Usual; }
799def vpselq_64: Intrinsic<
800    Vector, (args Vector:$t, Vector:$f, PredOf<u32>:$pred),
801            (bitcast (select $pred, (bitcast $t, VecOf<u32>),
802                                    (bitcast $f, VecOf<u32>)), Vector)>,
803    NameOverride<"vpselq"> { let params = T.All64; }
804
805let params = [Void], pnt = PNT_None in {
806
807  multiclass vctp<Type pred, string intname> {
808    def "": Intrinsic<pred, (args u32:$val),
809        (u16 (IRInt<"pred_v2i", [pred]> (IRIntBase<intname> $val)))>;
810    def _m: Intrinsic<pred, (args u32:$val, pred:$inpred),
811        (u16 (IRInt<"pred_v2i", [pred]> (and $inpred,
812                                         (IRIntBase<intname> $val))))>;
813  }
814  defm vctp8q:  vctp<PredOf<u8>,  "arm_mve_vctp8">;
815  defm vctp16q: vctp<PredOf<u16>, "arm_mve_vctp16">;
816  defm vctp32q: vctp<PredOf<u32>, "arm_mve_vctp32">;
817  defm vctp64q: vctp<PredOf<u64>, "arm_mve_vctp64">;
818
819  def vpnot: Intrinsic<PredOf<u8>, (args unpromoted<PredOf<u8>>:$pred),
820                       (xor $pred, (u16 65535))>;
821
822}
823
824multiclass contiguous_load<string mnemonic, PrimitiveType memtype,
825                           list<Type> same_size, list<Type> wider> {
826  // Intrinsics named with explicit memory and element sizes that match:
827  // vldrbq_?8, vldrhq_?16, vldrwq_?32.
828  let params = same_size, pnt = PNT_None in {
829    def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
830                   (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
831         NameOverride<mnemonic>;
832    def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
833                                 Predicate:$pred),
834                   (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
835                        (CPtr<Vector> $addr), !srl(memtype.size,3),
836                        $pred, (zeroinit Vector))>,
837         NameOverride<mnemonic # "_z">;
838  }
839
840  // Synonyms for the above, with the generic name vld1q that just means
841  // 'memory and element sizes match', and allows convenient polymorphism with
842  // the memory and element types covariant.
843  let params = same_size in {
844    def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
845                   (load (address (CPtr<Vector> $addr), !srl(memtype.size,3)))>,
846         NameOverride<"vld1q">;
847    def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
848                                 Predicate:$pred),
849                   (IRIntBase<"masked_load", [Vector, CPtr<Vector>]>
850                        (CPtr<Vector> $addr), !srl(memtype.size,3),
851                        $pred, (zeroinit Vector))>,
852         NameOverride<"vld1q_z">;
853  }
854
855  // Intrinsics with the memory size narrower than the vector element, so that
856  // they load less than 128 bits of memory and sign/zero extend each loaded
857  // value into a wider vector lane.
858  let params = wider, pnt = PNT_None in {
859    def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr),
860                   (extend (load (address (CPtr<NarrowedVecOf<memtype,Vector>>
861                                           $addr), !srl(memtype.size,3))),
862                           Vector, (unsignedflag Scalar))>,
863         NameOverride<mnemonic>;
864    def: Intrinsic<Vector, (args CPtr<CopyKind<same_size[0], Scalar>>:$addr,
865                                 Predicate:$pred),
866                   (extend (IRIntBase<"masked_load",
867                                      [NarrowedVecOf<memtype,Vector>,
868                                      CPtr<NarrowedVecOf<memtype,Vector>>]>
869                                (CPtr<NarrowedVecOf<memtype,Vector>> $addr),
870                                !srl(memtype.size,3), $pred,
871                                (zeroinit NarrowedVecOf<memtype,Vector>)),
872                           Vector, (unsignedflag Scalar))>,
873         NameOverride<mnemonic # "_z">;
874  }
875}
876
877defm: contiguous_load<"vldrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
878defm: contiguous_load<"vldrhq", u16, T.All16, T.Int32>;
879defm: contiguous_load<"vldrwq", u32, T.All32, []>;
880
881multiclass contiguous_store<string mnemonic, PrimitiveType memtype,
882                           list<Type> same_size, list<Type> wider> {
883  // Intrinsics named with explicit memory and element sizes that match:
884  // vstrbq_?8, vstrhq_?16, vstrwq_?32.
885  let params = same_size in {
886    def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
887                               Vector:$value),
888                   (store $value,
889                          (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
890         NameOverride<mnemonic>;
891    def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
892                               Vector:$value, Predicate:$pred),
893                   (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
894                        $value, (Ptr<Vector> $addr),
895                        !srl(memtype.size,3), $pred)>,
896         NameOverride<mnemonic # "_p">;
897  }
898
899  // Synonyms for the above, with the generic name vst1q that just means
900  // 'memory and element sizes match', and allows convenient polymorphism with
901  // the memory and element types covariant.
902  let params = same_size in {
903    def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
904                               Vector:$value),
905                   (store $value,
906                          (address (Ptr<Vector> $addr), !srl(memtype.size,3)))>,
907         NameOverride<"vst1q">;
908    def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
909                               Vector:$value, Predicate:$pred),
910                   (IRIntBase<"masked_store", [Vector, Ptr<Vector>]>
911                        $value, (Ptr<Vector> $addr),
912                        !srl(memtype.size,3), $pred)>,
913         NameOverride<"vst1q_p">;
914  }
915
916  // Intrinsics with the memory size narrower than the vector element, so that
917  // they store less than 128 bits of memory, truncating each vector lane into
918  // a narrower value to store.
919  let params = wider in {
920    def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
921                               Vector:$value),
922                   (store (trunc $value, NarrowedVecOf<memtype,Vector>),
923                          (address (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
924                                   !srl(memtype.size,3)))>,
925         NameOverride<mnemonic>;
926    def: Intrinsic<Void, (args Ptr<CopyKind<same_size[0], Scalar>>:$addr,
927                               Vector:$value, Predicate:$pred),
928                   (IRIntBase<"masked_store",
929                              [NarrowedVecOf<memtype,Vector>,
930                               Ptr<NarrowedVecOf<memtype,Vector>>]>
931                        (trunc $value, NarrowedVecOf<memtype,Vector>),
932                        (Ptr<NarrowedVecOf<memtype,Vector>> $addr),
933                        !srl(memtype.size,3), $pred)>,
934         NameOverride<mnemonic # "_p">;
935  }
936}
937
938defm: contiguous_store<"vstrbq", u8, T.All8, !listconcat(T.Int16, T.Int32)>;
939defm: contiguous_store<"vstrhq", u16, T.All16, T.Int32>;
940defm: contiguous_store<"vstrwq", u32, T.All32, []>;
941
942multiclass gather_base<list<Type> types, int size> {
943  let params = types, pnt = PNT_None in {
944    def _gather_base: Intrinsic<
945      Vector, (args UVector:$addr, imm_mem7bit<size>:$offset),
946      (IRInt<"vldr_gather_base", [Vector, UVector]> $addr, $offset)>;
947
948    def _gather_base_z: Intrinsic<
949      Vector, (args UVector:$addr, imm_mem7bit<size>:$offset, Predicate:$pred),
950      (IRInt<"vldr_gather_base_predicated", [Vector, UVector, Predicate]>
951          $addr, $offset, $pred)>;
952
953    def _gather_base_wb: Intrinsic<
954      Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset),
955      (seq (IRInt<"vldr_gather_base_wb", [Vector, UVector]>
956               (load $addr), $offset):$pair,
957           (store (xval $pair, 1), $addr),
958           (xval $pair, 0))>;
959
960    def _gather_base_wb_z: Intrinsic<
961      Vector, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
962                    Predicate:$pred),
963      (seq (IRInt<"vldr_gather_base_wb_predicated",
964                  [Vector, UVector, Predicate]>
965               (load $addr), $offset, $pred):$pair,
966           (store (xval $pair, 1), $addr),
967           (xval $pair, 0))>;
968  }
969}
970
971defm vldrwq: gather_base<T.All32, 4>;
972defm vldrdq: gather_base<T.All64, 8>;
973
974multiclass scatter_base<list<Type> types, int size> {
975  let params = types in {
976    def _scatter_base: Intrinsic<
977      Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data),
978      (IRInt<"vstr_scatter_base", [UVector, Vector]> $addr, $offset, $data)>;
979
980    def _scatter_base_p: Intrinsic<
981      Void, (args UVector:$addr, imm_mem7bit<size>:$offset, Vector:$data,
982                  Predicate:$pred),
983      (IRInt<"vstr_scatter_base_predicated", [UVector, Vector, Predicate]>
984          $addr, $offset, $data, $pred)>;
985
986    def _scatter_base_wb: Intrinsic<
987      Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset, Vector:$data),
988      (seq (IRInt<"vstr_scatter_base_wb", [UVector, Vector]>
989                 (load $addr), $offset, $data):$wbaddr,
990           (store $wbaddr, $addr))>;
991
992    def _scatter_base_wb_p: Intrinsic<
993      Void, (args Ptr<UVector>:$addr, imm_mem7bit<size>:$offset,
994                    Vector:$data, Predicate:$pred),
995      (seq (IRInt<"vstr_scatter_base_wb_predicated",
996                  [UVector, Vector, Predicate]>
997               (load $addr), $offset, $data, $pred):$wbaddr,
998           (store $wbaddr, $addr))>;
999  }
1000}
1001
1002defm vstrwq: scatter_base<T.All32, 4>;
1003defm vstrdq: scatter_base<T.All64, 8>;
1004
1005multiclass gather_offset_unshifted<list<Type> types, PrimitiveType memtype> {
1006  let params = types in {
1007    def _gather_offset: Intrinsic<
1008      Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
1009      (IRInt<"vldr_gather_offset",
1010             [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
1011          $base, $offsets, memtype.size, 0, (unsignedflag Scalar))>;
1012    def _gather_offset_z: Intrinsic<
1013      Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1014                    Predicate:$pred),
1015      (IRInt<"vldr_gather_offset_predicated",
1016             [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
1017          $base, $offsets, memtype.size, 0, (unsignedflag Scalar), $pred)>;
1018  }
1019}
1020
1021multiclass gather_offset_shifted<list<Type> types, PrimitiveType memtype,
1022                                 int shift> {
1023  let params = types in {
1024    def _gather_shifted_offset: Intrinsic<
1025      Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets),
1026      (IRInt<"vldr_gather_offset",
1027             [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector]>
1028          $base, $offsets, memtype.size, shift, (unsignedflag Scalar))>;
1029    def _gather_shifted_offset_z: Intrinsic<
1030      Vector, (args CPtr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1031                    Predicate:$pred),
1032      (IRInt<"vldr_gather_offset_predicated",
1033             [Vector, CPtr<CopyKind<memtype, Scalar>>, UVector, Predicate]>
1034          $base, $offsets, memtype.size, shift, (unsignedflag Scalar), $pred)>;
1035  }
1036}
1037
1038multiclass gather_offset_both<list<Type> types, PrimitiveType memtype,
1039                              int shift> {
1040  defm "": gather_offset_unshifted<types, memtype>;
1041  defm "": gather_offset_shifted<types, memtype, shift>;
1042}
1043
1044defm vldrbq: gather_offset_unshifted<!listconcat(T.All8, T.Int16, T.Int32), u8>;
1045defm vldrhq: gather_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
1046defm vldrwq: gather_offset_both<T.All32, u32, 2>;
1047defm vldrdq: gather_offset_both<T.Int64, u64, 3>;
1048
1049multiclass scatter_offset_unshifted<list<Type> types, PrimitiveType memtype> {
1050  let params = types in {
1051    def _scatter_offset: Intrinsic<
1052      Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1053                  Vector:$data),
1054      (IRInt<"vstr_scatter_offset",
1055             [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
1056          $base, $offsets, $data, memtype.size, 0)>;
1057    def _scatter_offset_p: Intrinsic<
1058      Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1059                  Vector:$data, Predicate:$pred),
1060      (IRInt<"vstr_scatter_offset_predicated",
1061             [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
1062          $base, $offsets, $data, memtype.size, 0, $pred)>;
1063  }
1064}
1065
1066multiclass scatter_offset_shifted<list<Type> types, PrimitiveType memtype,
1067                                  int shift> {
1068  let params = types in {
1069    def _scatter_shifted_offset: Intrinsic<
1070      Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1071                  Vector:$data),
1072      (IRInt<"vstr_scatter_offset",
1073             [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector]>
1074          $base, $offsets, $data, memtype.size, shift)>;
1075    def _scatter_shifted_offset_p: Intrinsic<
1076      Void, (args Ptr<CopyKind<memtype, Scalar>>:$base, UVector:$offsets,
1077                  Vector:$data, Predicate:$pred),
1078      (IRInt<"vstr_scatter_offset_predicated",
1079             [Ptr<CopyKind<memtype, Scalar>>, UVector, Vector, Predicate]>
1080          $base, $offsets, $data, memtype.size, shift, $pred)>;
1081  }
1082}
1083
1084multiclass scatter_offset_both<list<Type> types, PrimitiveType memtype,
1085                               int shift> {
1086  defm "": scatter_offset_unshifted<types, memtype>;
1087  defm "": scatter_offset_shifted<types, memtype, shift>;
1088}
1089
1090defm vstrbq: scatter_offset_unshifted<!listconcat(T.All8,T.Int16,T.Int32), u8>;
1091defm vstrhq: scatter_offset_both<!listconcat(T.All16, T.Int32), u16, 1>;
1092defm vstrwq: scatter_offset_both<T.All32, u32, 2>;
1093defm vstrdq: scatter_offset_both<T.Int64, u64, 3>;
1094
1095let params = T.Int in {
1096  def vshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
1097                         (shl $v, (splat (Scalar $sh)))>;
1098  defm vshlq: IntrinsicMX<Vector, (args Vector:$v, imm_0toNm1:$sh,
1099                                        Predicate:$pred),
1100      (IRInt<"shl_imm_predicated", [Vector, Predicate]>
1101           $v, $sh, $pred, $inactive), 1, "_n">;
1102
1103  let pnt = PNT_NType in {
1104    def vshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
1105                            (immshr $v, $sh, (unsignedflag Scalar))>;
1106    defm vshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
1107                                          Predicate:$pred),
1108        (IRInt<"shr_imm_predicated", [Vector, Predicate]>
1109             $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
1110  }
1111}
1112
1113let params = T.Int in {
1114  def vqshlq_n: Intrinsic<Vector, (args Vector:$v, imm_0toNm1:$sh),
1115       (IRInt<"vqshl_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
1116  def vqshlq_m_n: Intrinsic<Vector, (args Vector:$inactive, Vector:$v,
1117                                          imm_0toNm1:$sh, Predicate:$pred),
1118       (IRInt<"vqshl_imm_predicated", [Vector, Predicate]>
1119            $v, $sh, (unsignedflag Scalar), $pred, $inactive)>;
1120
1121  let pnt = PNT_NType in {
1122    def vrshrq_n: Intrinsic<Vector, (args Vector:$v, imm_1toN:$sh),
1123         (IRInt<"vrshr_imm", [Vector]> $v, $sh, (unsignedflag Scalar))>;
1124    defm vrshrq: IntrinsicMX<Vector, (args Vector:$v, imm_1toN:$sh,
1125                                           Predicate:$pred),
1126         (IRInt<"vrshr_imm_predicated", [Vector, Predicate]>
1127              $v, $sh, (unsignedflag Scalar), $pred, $inactive), 1, "_n">;
1128  }
1129}
1130
1131let params = T.Signed, pnt = PNT_NType in {
1132  def vqshluq_n: Intrinsic<UVector, (args Vector:$v, imm_0toNm1:$sh),
1133       (IRInt<"vqshlu_imm", [Vector]> $v, $sh)>;
1134  def vqshluq_m_n: Intrinsic<UVector, (args UVector:$inactive, Vector:$v,
1135                                            imm_0toNm1:$sh, Predicate:$pred),
1136       (IRInt<"vqshlu_imm_predicated", [Vector, Predicate]>
1137            $v, $sh, $pred, $inactive)>;
1138}
1139
1140multiclass vshll_imm<int top> {
1141  let params = !listconcat(T.Int8, T.Int16), pnt = PNT_NType in {
1142    def _n: Intrinsic<DblVector, (args Vector:$v, imm_1toN:$sh),
1143        (IRInt<"vshll_imm", [DblVector, Vector]>
1144            $v, $sh, (unsignedflag Scalar), top)>;
1145    defm "": IntrinsicMX<DblVector, (args Vector:$v, imm_1toN:$sh,
1146                                          DblPredicate:$pred),
1147        (IRInt<"vshll_imm_predicated", [DblVector, Vector, DblPredicate]>
1148            $v, $sh, (unsignedflag Scalar), top, $pred, $inactive), 1, "_n">;
1149  }
1150}
1151defm vshllbq : vshll_imm<0>;
1152defm vshlltq : vshll_imm<1>;
1153
1154multiclass DyadicImmShift<Type outtype, Immediate imm, string intname = NAME,
1155                          dag extraargs = (?)> {
1156  defvar intparams = !if(!eq(!cast<string>(outtype), !cast<string>(Vector)),
1157                         [Vector], [outtype, Vector]);
1158
1159  def q_n: Intrinsic<
1160      outtype, (args outtype:$a, Vector:$b, imm:$sh),
1161      !con((IRInt<intname, intparams> $a, $b, $sh), extraargs)>;
1162
1163  def q_m_n: Intrinsic<
1164      outtype, (args outtype:$a, Vector:$b, imm:$sh, Predicate:$pred),
1165      !con((IRInt<intname # "_predicated", intparams # [Predicate]>
1166               $a, $b, $sh), extraargs, (? $pred))>;
1167}
1168
1169multiclass VSHRN<Type outtype, Immediate imm, dag extraargs> {
1170  defm b: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 0))>;
1171  defm t: DyadicImmShift<outtype, imm, "vshrn", !con(extraargs, (? 1))>;
1172}
1173
1174let params = [s16, s32, u16, u32], pnt = PNT_NType in {
1175  defvar U = (unsignedflag Scalar);
1176  defm vshrn   : VSHRN<HalfVector, imm_1toHalfN, (? 0,0,U,U)>;
1177  defm vqshrn  : VSHRN<HalfVector, imm_1toHalfN, (? 1,0,U,U)>;
1178  defm vrshrn  : VSHRN<HalfVector, imm_1toHalfN, (? 0,1,U,U)>;
1179  defm vqrshrn : VSHRN<HalfVector, imm_1toHalfN, (? 1,1,U,U)>;
1180}
1181let params = [s16, s32], pnt = PNT_NType in {
1182  defm vqshrun  : VSHRN<UHalfVector, imm_1toHalfN, (? 1,0,1,0)>;
1183  defm vqrshrun : VSHRN<UHalfVector, imm_1toHalfN, (? 1,1,1,0)>;
1184}
1185let params = T.Int, pnt = PNT_NType in {
1186  defm vsli : DyadicImmShift<Vector, imm_0toNm1>;
1187  defm vsri : DyadicImmShift<Vector, imm_1toN>;
1188}
1189
1190multiclass VSHL_non_imm<string scalarSuffix, int q, int r,
1191                        PolymorphicNameType pnt_scalar_unpred = PNT_Type> {
1192  let pnt = pnt_scalar_unpred in {
1193    def scalarSuffix: Intrinsic<
1194      Vector, (args Vector:$in, s32:$sh),
1195      (IRInt<"vshl_scalar", [Vector]> $in, $sh,
1196           q, r, (unsignedflag Scalar))>;
1197  }
1198  def "_m" # scalarSuffix: Intrinsic<
1199    Vector, (args Vector:$in, s32:$sh, Predicate:$pred),
1200    (IRInt<"vshl_scalar_predicated", [Vector, Predicate]> $in, $sh,
1201         q, r, (unsignedflag Scalar), $pred)>;
1202
1203  def "": Intrinsic<
1204    Vector, (args Vector:$in, SVector:$sh),
1205    (IRInt<"vshl_vector", [Vector, SVector]> $in, $sh,
1206         q, r, (unsignedflag Scalar))>;
1207  defm "": IntrinsicMX<
1208    Vector, (args Vector:$in, SVector:$sh, Predicate:$pred),
1209    (IRInt<"vshl_vector_predicated", [Vector, SVector, Predicate]> $in, $sh,
1210         q, r, (unsignedflag Scalar), $pred, $inactive),
1211    // The saturating shift intrinsics don't have an x variant, so we
1212    // set wantXVariant to 1 iff q == 0
1213    !eq(q, 0)>;
1214}
1215
1216let params = T.Int in {
1217  defm vshlq   : VSHL_non_imm<"_r", 0, 0>;
1218  defm vqshlq  : VSHL_non_imm<"_r", 1, 0>;
1219  defm vrshlq  : VSHL_non_imm<"_n", 0, 1, PNT_NType>;
1220  defm vqrshlq : VSHL_non_imm<"_n", 1, 1, PNT_NType>;
1221}
1222
1223// Base class for the scalar shift intrinsics.
1224class ScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
1225  Intrinsic<argtype, !con((args argtype:$value), shiftCountArg), shiftCodeGen> {
1226  let params = [Void];
1227  let pnt = PNT_None;
1228}
1229
1230// Subclass that includes the machinery to take a 64-bit input apart
1231// into halves, retrieve the two halves of a shifted output as a pair,
1232// and glue the pieces of the pair back into an i64 for output.
1233class LongScalarShift<Type argtype, dag shiftCountArg, dag shiftCodeGen>:
1234   ScalarShift<argtype, shiftCountArg,
1235               (seq (u32 (lshr $value, (argtype 32))):$hi,
1236                    (u32 $value):$lo,
1237                    shiftCodeGen:$pair,
1238                    (or (shl (u64 (xval $pair, 1)), (u64 32)),
1239                             (u64 (xval $pair, 0))))>;
1240
1241// The family of saturating/rounding scalar shifts that take an
1242// immediate shift count. They come in matched 32- and 64-bit pairs.
1243multiclass ScalarSaturatingShiftImm<Type arg32, Type arg64> {
1244  def "": ScalarShift<arg32, (args imm_1to32:$sh),
1245                      (IRInt<NAME> $value, $sh)>;
1246  def l:  LongScalarShift<arg64, (args imm_1to32:$sh),
1247                          (IRInt<NAME # "l"> $lo, $hi, $sh)>;
1248}
1249defm uqshl: ScalarSaturatingShiftImm<u32, u64>;
1250defm urshr: ScalarSaturatingShiftImm<u32, u64>;
1251defm sqshl: ScalarSaturatingShiftImm<s32, s64>;
1252defm srshr: ScalarSaturatingShiftImm<s32, s64>;
1253
1254// The family of saturating/rounding scalar shifts that take a
1255// register shift count. They also have 32- and 64-bit forms, but the
1256// 64-bit form also has a version that saturates to 48 bits, so the IR
1257// intrinsic takes an extra saturation-type operand.
1258multiclass ScalarSaturatingShiftReg<Type arg32, Type arg64> {
1259  def "":          ScalarShift<arg32, (args s32:$sh),
1260                               (IRInt<NAME> $value, $sh)>;
1261  def l:       LongScalarShift<arg64, (args s32:$sh),
1262                               (IRInt<NAME # "l"> $lo, $hi, $sh, 64)>;
1263  def l_sat48: LongScalarShift<arg64, (args s32:$sh),
1264                               (IRInt<NAME # "l"> $lo, $hi, $sh, 48)>;
1265}
1266defm uqrshl: ScalarSaturatingShiftReg<u32, u64>;
1267defm sqrshr: ScalarSaturatingShiftReg<s32, s64>;
1268
1269// The intrinsics for LSLL and ASRL come in 64-bit versions only, with
1270// no saturation count.
1271def lsll: LongScalarShift<u64, (args s32:$sh), (IRInt<"lsll"> $lo, $hi, $sh)>;
1272def asrl: LongScalarShift<s64, (args s32:$sh), (IRInt<"asrl"> $lo, $hi, $sh)>;
1273
1274multiclass vadcsbc {
1275  def q: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
1276      (seq (IRInt<NAME, [Vector]> $a, $b, (shl (load $carry), 29)):$pair,
1277           (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1278           (xval $pair, 0))>;
1279  def iq: Intrinsic<Vector, (args Vector:$a, Vector:$b, Ptr<uint>:$carry),
1280      (seq (IRInt<NAME, [Vector]> $a, $b, 0):$pair,
1281           (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1282           (xval $pair, 0))>;
1283  def q_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
1284                                   Ptr<uint>:$carry, Predicate:$pred),
1285      (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
1286               (shl (load $carry), 29), $pred):$pair,
1287           (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1288           (xval $pair, 0))>;
1289  def iq_m: Intrinsic<Vector, (args Vector:$inactive, Vector:$a, Vector:$b,
1290                                    Ptr<uint>:$carry, Predicate:$pred),
1291      (seq (IRInt<NAME # "_predicated", [Vector, Predicate]> $inactive, $a, $b,
1292               0, $pred):$pair,
1293           (store (and 1, (lshr (xval $pair, 1), 29)), $carry),
1294           (xval $pair, 0))>;
1295}
1296let params = T.Int32 in {
1297  defm vadc: vadcsbc;
1298  defm vsbc: vadcsbc;
1299}
1300
1301let params = T.Int in {
1302  def vshlcq: Intrinsic<
1303    Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm),
1304    (seq (load $ps):$s,
1305         (IRInt<"vshlc", [Vector]> $v, $s, $imm):$pair,
1306         (store (xval $pair, 0), $ps),
1307         (xval $pair, 1))>;
1308  def vshlcq_m: Intrinsic<
1309    Vector, (args Vector:$v, Ptr<u32>:$ps, imm_1to32:$imm, Predicate:$pred),
1310    (seq (load $ps):$s,
1311         (IRInt<"vshlc_predicated", [Vector, Predicate]>
1312              $v, $s, $imm, $pred):$pair,
1313         (store (xval $pair, 0), $ps),
1314         (xval $pair, 1))>;
1315}
1316
1317multiclass VectorComplexAddPred<dag not_halving, dag angle> {
1318  def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
1319     (IRInt<"vcaddq", [Vector]> not_halving, angle, $a, $b)>;
1320  defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
1321     (IRInt<"vcaddq_predicated", [Vector, Predicate]>
1322       not_halving, angle, $inactive, $a, $b, $pred)>;
1323}
1324
1325multiclass VectorComplexMulPred<dag angle> {
1326  def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b),
1327    (IRInt<"vcmulq", [Vector]> angle, $a, $b)>;
1328  defm "" : IntrinsicMX<Vector, (args Vector:$a, Vector:$b, Predicate:$pred),
1329    (IRInt<"vcmulq_predicated", [Vector, Predicate]> angle, $inactive, $a, $b,
1330      $pred)>;
1331}
1332
1333multiclass VectorComplexMLAPred<dag angle> {
1334  def "" : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c),
1335    (IRInt<"vcmlaq", [Vector]> angle, $a, $b, $c)>;
1336  def _m : Intrinsic<Vector, (args Vector:$a, Vector:$b, Vector:$c,
1337                                   Predicate:$pred),
1338    (IRInt<"vcmlaq_predicated", [Vector, Predicate]> angle, $a, $b, $c, $pred)>;
1339}
1340
1341multiclass VectorComplexAddAngle<dag not_halving> {
1342  defm _rot90 : VectorComplexAddPred<not_halving, (u32 0)>;
1343  defm _rot270 : VectorComplexAddPred<not_halving, (u32 1)>;
1344}
1345
1346multiclass VectorComplexMulAngle {
1347  defm ""      : VectorComplexMulPred<(u32 0)>;
1348  defm _rot90  : VectorComplexMulPred<(u32 1)>;
1349  defm _rot180 : VectorComplexMulPred<(u32 2)>;
1350  defm _rot270 : VectorComplexMulPred<(u32 3)>;
1351}
1352
1353multiclass VectorComplexMLAAngle {
1354  defm ""      : VectorComplexMLAPred<(u32 0)>;
1355  defm _rot90  : VectorComplexMLAPred<(u32 1)>;
1356  defm _rot180 : VectorComplexMLAPred<(u32 2)>;
1357  defm _rot270 : VectorComplexMLAPred<(u32 3)>;
1358}
1359
1360let params = T.Usual in
1361defm vcaddq : VectorComplexAddAngle<(u32 1)>;
1362
1363let params = T.Signed in
1364defm vhcaddq : VectorComplexAddAngle<(u32 0)>;
1365
1366let params = T.Float in {
1367defm vcmulq : VectorComplexMulAngle;
1368defm vcmlaq : VectorComplexMLAAngle;
1369}
1370
1371multiclass MVEBinaryVectorHoriz32<dag subtract, dag exchange, string xsuffix> {
1372  def xsuffix#"q"
1373    : Intrinsic<Scalar32, (args Vector:$a, Vector:$b),
1374                          (IRInt<"vmldava", [Vector]>
1375                           (unsignedflag Scalar), subtract, exchange,
1376                           (zeroinit Scalar32), $a, $b)>;
1377  def xsuffix#"q_p"
1378    : Intrinsic<Scalar32, (args Vector:$a, Vector:$b, Predicate:$pred),
1379                          (IRInt<"vmldava_predicated", [Vector, Predicate]>
1380                           (unsignedflag Scalar), subtract, exchange,
1381                           (zeroinit Scalar32), $a, $b, $pred)>;
1382
1383  def "a"#xsuffix#"q"
1384    : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c),
1385                          (IRInt<"vmldava", [Vector]>
1386                           (unsignedflag Scalar), subtract, exchange,
1387                           $a, $b, $c)>;
1388  def "a"#xsuffix#"q_p"
1389    : Intrinsic<Scalar32, (args Scalar32:$a, Vector:$b, Vector:$c,
1390                                Predicate:$pred),
1391                          (IRInt<"vmldava_predicated", [Vector, Predicate]>
1392                           (unsignedflag Scalar), subtract, exchange,
1393                           $a, $b, $c, $pred)>;
1394}
1395
1396class IntrSplit64<Type resty, dag args, dag codegen>
1397  : Intrinsic<resty, args,
1398              (seq (u32 (lshr $a, (u64 32))):$hi,
1399                   (u32 $a):$lo,
1400                   codegen:$pair,
1401                   (or (shl (u64 (xval $pair, 1)), (u64 32)),
1402                            (u64 (xval $pair, 0))))>;
1403
1404class IntrSplit64ZeroInit<Type resty, dag args, dag codegen>
1405  : Intrinsic<resty, args,
1406              (seq (zeroinit u32):$hi,
1407                   (zeroinit u32):$lo,
1408                   codegen:$pair,
1409                   (or (shl (u64 (xval $pair, 1)), (u64 32)),
1410                            (u64 (xval $pair, 0))))>;
1411
1412multiclass MVEBinaryVectorHoriz64Base<dag subtract, dag exchange,
1413                                      string xsuffix, string irname> {
1414  def xsuffix#"q"
1415    : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b),
1416                          (IRInt<irname, [Vector]>
1417                           (unsignedflag Scalar), subtract, exchange,
1418                           $lo, $hi, $a, $b)>;
1419  def xsuffix#"q_p"
1420    : IntrSplit64ZeroInit<Scalar64, (args Vector:$a, Vector:$b,
1421                                          Predicate:$pred),
1422                          (IRInt<irname#"_predicated", [Vector, Predicate]>
1423                           (unsignedflag Scalar), subtract, exchange,
1424                           $lo, $hi, $a, $b, $pred)>;
1425
1426  def "a"#xsuffix#"q"
1427    : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c),
1428                          (IRInt<irname, [Vector]>
1429                           (unsignedflag Scalar), subtract, exchange,
1430                           $lo, $hi, $b, $c)>;
1431  def "a"#xsuffix#"q_p"
1432    : IntrSplit64<Scalar64, (args Scalar64:$a, Vector:$b, Vector:$c,
1433                                  Predicate:$pred),
1434                          (IRInt<irname#"_predicated", [Vector, Predicate]>
1435                           (unsignedflag Scalar), subtract, exchange,
1436                           $lo, $hi, $b, $c, $pred)>;
1437}
1438
1439multiclass MVEBinaryVectorHoriz64<dag subtract, dag exchange, string xsuffix> {
1440  defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix, "vmlldava">;
1441}
1442
1443multiclass MVEBinaryVectorHoriz64R<dag subtract, dag exchange, string xsuffix> {
1444  defm "" : MVEBinaryVectorHoriz64Base<subtract, exchange, xsuffix,
1445                                       "vrmlldavha">;
1446}
1447
1448multiclass VADDV<bit acc, bit pred, string intbase, Type Scalar> {
1449  defvar accArg = !if(acc, (args Scalar:$acc), (args));
1450  defvar predArg = !if(pred, (args Predicate:$pred), (args));
1451  defvar intrinsic = !if(pred,
1452      IRInt<intbase # "_predicated", [Vector, Predicate]>,
1453      IRInt<intbase, [Vector]>);
1454  defvar intCG = !con((intrinsic $v, (unsignedflag Scalar)),
1455                      !if(pred, (? $pred), (?)));
1456  defvar accCG = !if(acc, (add intCG, $acc), intCG);
1457
1458  def "": Intrinsic<Scalar, !con(accArg, (args Vector:$v), predArg), accCG>;
1459}
1460
1461let params = T.Int in {
1462defm vaddvq    : VADDV<0, 0, "addv", Scalar32>;
1463defm vaddvaq   : VADDV<1, 0, "addv", Scalar32>;
1464defm vaddvq_p  : VADDV<0, 1, "addv", Scalar32>;
1465defm vaddvaq_p : VADDV<1, 1, "addv", Scalar32>;
1466}
1467
1468let params = [s32, u32] in {
1469defm vaddlvq    : VADDV<0, 0, "addlv", Scalar64>;
1470defm vaddlvaq   : VADDV<1, 0, "addlv", Scalar64>;
1471defm vaddlvq_p  : VADDV<0, 1, "addlv", Scalar64>;
1472defm vaddlvaq_p : VADDV<1, 1, "addlv", Scalar64>;
1473}
1474
1475let params = T.Int in {
1476def vabavq : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c),
1477    (IRInt<"vabav", [Vector]> (unsignedflag Scalar), $a, $b, $c)>;
1478def vabavq_p : Intrinsic<u32, (args u32:$a, Vector:$b, Vector:$c,
1479                                    Predicate:$pred),
1480    (IRInt<"vabav_predicated", [Vector, Predicate]>
1481                               (unsignedflag Scalar), $a, $b, $c, $pred)>;
1482
1483defm vmladav  : MVEBinaryVectorHoriz32<V.False, V.False, "">;
1484}
1485
1486let params = T.Signed in {
1487defm vmladav : MVEBinaryVectorHoriz32<V.False, V.True, "x">;
1488defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.False, "">;
1489defm vmlsdav : MVEBinaryVectorHoriz32<V.True, V.True, "x">;
1490}
1491
1492let params = [u16, s16, u32, s32] in
1493defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.False, "">;
1494
1495let params = [s16, s32] in {
1496defm vmlaldav : MVEBinaryVectorHoriz64<V.False, V.True, "x">;
1497defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.False, "">;
1498defm vmlsldav : MVEBinaryVectorHoriz64<V.True, V.True, "x">;
1499}
1500
1501let params = T.Int32 in
1502defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.False, "">;
1503
1504let params = [s32] in {
1505defm vrmlaldavh : MVEBinaryVectorHoriz64R<V.False, V.True, "x">;
1506defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.False, "">;
1507defm vrmlsldavh : MVEBinaryVectorHoriz64R<V.True, V.True, "x">;
1508}
1509
1510multiclass vrev_predicated<int revsize> {
1511  defm "" : IntrinsicMX<Vector, (args Vector:$a, Predicate:$pred),
1512      (IRInt<"vrev_predicated", [Vector, Predicate]>
1513          $a, revsize, $pred, $inactive)>;
1514}
1515
1516let params = T.All8 in {
1517  def vrev16q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 16)>;
1518  defm vrev16q: vrev_predicated<16>;
1519}
1520let params = !listconcat(T.All8, T.All16) in {
1521  def vrev32q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 32)>;
1522  defm vrev32q: vrev_predicated<32>;
1523}
1524let params = T.Usual in {
1525  def vrev64q : Intrinsic<Vector, (args Vector:$a), (vrev $a, 64)>;
1526  defm vrev64q: vrev_predicated<64>;
1527}
1528
1529foreach desttype = T.All in {
1530  // We want a vreinterpretq between every pair of supported vector types
1531  // _except_ that there shouldn't be one from a type to itself.
1532  //
1533  // So this foldl expression implements what you'd write in Python as
1534  // [srctype for srctype in T.All if srctype != desttype]
1535  let params = !foldl([]<Type>, T.All, tlist, srctype, !listconcat(tlist,
1536      !if(!eq(!cast<string>(desttype),!cast<string>(srctype)),[],[srctype])))
1537  in {
1538    def "vreinterpretq_" # desttype: Intrinsic<
1539        VecOf<desttype>, (args Vector:$x), (vreinterpret $x, VecOf<desttype>)>;
1540  }
1541}
1542
1543let params = T.All in {
1544  let pnt = PNT_None in {
1545    def vcreateq: Intrinsic<Vector, (args u64:$a, u64:$b),
1546        (bitcast (ielt_const (ielt_const (undef VecOf<u64>), $a, 0),
1547                             $b, 1), Vector)>;
1548    def vuninitializedq: Intrinsic<Vector, (args), (undef Vector)>;
1549  }
1550
1551  // This is the polymorphic form of vuninitializedq, which takes no type
1552  // suffix, but takes an _unevaluated_ vector parameter and returns an
1553  // uninitialized vector of the same vector type.
1554  //
1555  // This intrinsic has no _non_-polymorphic form exposed to the user. But each
1556  // separately typed version of it still has to have its own clang builtin id,
1557  // which can't be called vuninitializedq_u32 or similar because that would
1558  // collide with the explicit nullary versions above. So I'm calling them
1559  // vuninitializedq_polymorphic_u32 (and so on) for builtin id purposes; that
1560  // full name never appears in the header file due to the polymorphicOnly
1561  // flag, and the _polymorphic suffix is omitted from the shortened name by
1562  // the custom PolymorphicNameType here.
1563  let polymorphicOnly = 1, nonEvaluating = 1,
1564      pnt = PolymorphicNameType<1, "polymorphic"> in {
1565    def vuninitializedq_polymorphic: Intrinsic<
1566        Vector, (args Vector), (undef Vector)>;
1567  }
1568
1569  def vgetq_lane: Intrinsic<Scalar, (args Vector:$v, imm_lane:$lane),
1570                            (xelt_var $v, $lane)>;
1571  def vsetq_lane: Intrinsic<Vector, (args unpromoted<Scalar>:$e, Vector:$v, imm_lane:$lane),
1572                            (ielt_var $v, $e, $lane)>;
1573}
1574
1575foreach desttype = !listconcat(T.Int16, T.Int32, T.Float) in {
1576  defvar is_dest_float = !eq(desttype.kind, "f");
1577  defvar is_dest_unsigned = !eq(desttype.kind, "u");
1578  // First immediate operand of the LLVM intrinsic
1579  defvar unsigned_flag = !if(is_dest_float, (unsignedflag Scalar),
1580                             !if(is_dest_unsigned, V.True, V.False));
1581  // For float->int conversions _n and _x_n intrinsics are not polymorphic
1582  // because the signedness of the destination type cannot be inferred.
1583  defvar pnt_nx = !if(is_dest_float, PNT_2Type, PNT_None);
1584
1585  let params = !if(is_dest_float,
1586                   !if(!eq(desttype.size, 16), T.Int16, T.Int32),
1587                   !if(!eq(desttype.size, 16), [f16], [f32])) in {
1588    let pnt = pnt_nx in
1589      def "vcvtq_n_"#desttype : Intrinsic<VecOf<desttype>,
1590        (args Vector:$a, imm_1toN:$b),
1591        (IRInt<"vcvt_fix", [VecOf<desttype>, Vector]> unsigned_flag, $a, $b)>;
1592
1593    defm "vcvtq" : IntrinsicMX<VecOf<desttype>,
1594      (args Vector:$a, imm_1toN:$b, Predicate:$p),
1595      (IRInt<"vcvt_fix_predicated", [VecOf<desttype>, Vector, Predicate]>
1596             unsigned_flag, $inactive, $a, $b, $p),
1597      1, "_n_"#desttype, PNT_2Type, pnt_nx>;
1598  }
1599}
1600
1601let params = T.Usual in {
1602let pnt = PNT_NType in
1603def vbrsrq_n: Intrinsic<Vector, (args Vector:$a, s32:$b),
1604                        (IRInt<"vbrsr", [Vector]> $a, $b)>;
1605defm vbrsrq : IntrinsicMX<Vector, (args Vector:$a, s32:$b, Predicate:$pred),
1606                         (IRInt<"vbrsr_predicated", [Vector, Predicate]>
1607                         $inactive, $a, $b, $pred), 1, "_n",
1608                         PNT_NType, PNT_NType>;
1609}
1610