1//===-- ARMInstrMVE.td - MVE support for ARM ---------------*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file describes the ARM MVE instruction set.
10//
11//===----------------------------------------------------------------------===//
12
13// VPT condition mask
14def vpt_mask : Operand<i32> {
15  let PrintMethod = "printVPTMask";
16  let ParserMatchClass = it_mask_asmoperand;
17  let EncoderMethod = "getVPTMaskOpValue";
18  let DecoderMethod = "DecodeVPTMaskOperand";
19}
20
21// VPT/VCMP restricted predicate for sign invariant types
22def pred_restricted_i_asmoperand : AsmOperandClass {
23  let Name = "CondCodeRestrictedI";
24  let RenderMethod = "addITCondCodeOperands";
25  let PredicateMethod = "isITCondCodeRestrictedI";
26  let ParserMethod = "parseITCondCode";
27  let DiagnosticString = "condition code for sign-independent integer "#
28                         "comparison must be EQ or NE";
29}
30
31// VPT/VCMP restricted predicate for signed types
32def pred_restricted_s_asmoperand : AsmOperandClass {
33  let Name = "CondCodeRestrictedS";
34  let RenderMethod = "addITCondCodeOperands";
35  let PredicateMethod = "isITCondCodeRestrictedS";
36  let ParserMethod = "parseITCondCode";
37  let DiagnosticString = "condition code for signed integer "#
38                         "comparison must be EQ, NE, LT, GT, LE or GE";
39}
40
41// VPT/VCMP restricted predicate for unsigned types
42def pred_restricted_u_asmoperand : AsmOperandClass {
43  let Name = "CondCodeRestrictedU";
44  let RenderMethod = "addITCondCodeOperands";
45  let PredicateMethod = "isITCondCodeRestrictedU";
46  let ParserMethod = "parseITCondCode";
47  let DiagnosticString = "condition code for unsigned integer "#
48                         "comparison must be EQ, NE, HS or HI";
49}
50
51// VPT/VCMP restricted predicate for floating point
52def pred_restricted_fp_asmoperand : AsmOperandClass {
53  let Name = "CondCodeRestrictedFP";
54  let RenderMethod = "addITCondCodeOperands";
55  let PredicateMethod = "isITCondCodeRestrictedFP";
56  let ParserMethod = "parseITCondCode";
57  let DiagnosticString = "condition code for floating-point "#
58                         "comparison must be EQ, NE, LT, GT, LE or GE";
59}
60
61class VCMPPredicateOperand : Operand<i32>;
62
63def pred_basic_i : VCMPPredicateOperand {
64  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
65  let ParserMatchClass = pred_restricted_i_asmoperand;
66  let DecoderMethod = "DecodeRestrictedIPredicateOperand";
67  let EncoderMethod = "getRestrictedCondCodeOpValue";
68}
69
70def pred_basic_u : VCMPPredicateOperand {
71  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
72  let ParserMatchClass = pred_restricted_u_asmoperand;
73  let DecoderMethod = "DecodeRestrictedUPredicateOperand";
74  let EncoderMethod = "getRestrictedCondCodeOpValue";
75}
76
77def pred_basic_s : VCMPPredicateOperand {
78  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
79  let ParserMatchClass = pred_restricted_s_asmoperand;
80  let DecoderMethod = "DecodeRestrictedSPredicateOperand";
81  let EncoderMethod = "getRestrictedCondCodeOpValue";
82}
83
84def pred_basic_fp : VCMPPredicateOperand {
85  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
86  let ParserMatchClass = pred_restricted_fp_asmoperand;
87  let DecoderMethod = "DecodeRestrictedFPPredicateOperand";
88  let EncoderMethod = "getRestrictedCondCodeOpValue";
89}
90
91// Register list operands for interleaving load/stores
92def VecList2QAsmOperand : AsmOperandClass {
93  let Name = "VecListTwoMQ";
94  let ParserMethod = "parseVectorList";
95  let RenderMethod = "addMVEVecListOperands";
96  let DiagnosticString = "operand must be a list of two consecutive "#
97                         "q-registers in range [q0,q7]";
98}
99
100def VecList2Q : RegisterOperand<QQPR, "printMVEVectorListTwoQ"> {
101  let ParserMatchClass = VecList2QAsmOperand;
102  let PrintMethod = "printMVEVectorList<2>";
103}
104
105def VecList4QAsmOperand : AsmOperandClass {
106  let Name = "VecListFourMQ";
107  let ParserMethod = "parseVectorList";
108  let RenderMethod = "addMVEVecListOperands";
109  let DiagnosticString = "operand must be a list of four consecutive "#
110                         "q-registers in range [q0,q7]";
111}
112
113def VecList4Q : RegisterOperand<QQQQPR, "printMVEVectorListFourQ"> {
114  let ParserMatchClass = VecList4QAsmOperand;
115  let PrintMethod = "printMVEVectorList<4>";
116}
117
118// taddrmode_imm7  := reg[r0-r7] +/- (imm7 << shift)
119class TMemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
120  let Name = "TMemImm7Shift"#shift#"Offset";
121  let PredicateMethod = "isMemImm7ShiftedOffset<"#shift#",ARM::tGPRRegClassID>";
122  let RenderMethod = "addMemImmOffsetOperands";
123}
124
125class taddrmode_imm7<int shift> : MemOperand,
126    ComplexPattern<i32, 2, "SelectTAddrModeImm7<"#shift#">", []>  {
127  let ParserMatchClass = TMemImm7ShiftOffsetAsmOperand<shift>;
128  // They are printed the same way as the T2 imm8 version
129  let PrintMethod = "printT2AddrModeImm8Operand<false>";
130  // This can also be the same as the T2 version.
131  let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
132  let DecoderMethod = "DecodeTAddrModeImm7<"#shift#">";
133  let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
134}
135
136// t2addrmode_imm7  := reg +/- (imm7)
137class MemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
138  let Name = "MemImm7Shift"#shift#"Offset";
139  let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
140                        ",ARM::GPRnopcRegClassID>";
141  let RenderMethod = "addMemImmOffsetOperands";
142}
143
144def MemImm7Shift0OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<0>;
145def MemImm7Shift1OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<1>;
146def MemImm7Shift2OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<2>;
147class T2AddrMode_Imm7<int shift> : MemOperand,
148      ComplexPattern<i32, 2, "SelectT2AddrModeImm7<"#shift#">", []> {
149  let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
150  let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 0>";
151  let ParserMatchClass =
152    !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetAsmOperand");
153  let MIOperandInfo = (ops GPRnopc:$base, i32imm:$offsimm);
154}
155
156class t2addrmode_imm7<int shift> : T2AddrMode_Imm7<shift> {
157  // They are printed the same way as the imm8 version
158  let PrintMethod = "printT2AddrModeImm8Operand<false>";
159}
160
161class MemImm7ShiftOffsetWBAsmOperand<int shift> : AsmOperandClass {
162  let Name = "MemImm7Shift"#shift#"OffsetWB";
163  let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
164                        ",ARM::rGPRRegClassID>";
165  let RenderMethod = "addMemImmOffsetOperands";
166}
167
168def MemImm7Shift0OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<0>;
169def MemImm7Shift1OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<1>;
170def MemImm7Shift2OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<2>;
171
172class t2addrmode_imm7_pre<int shift> : T2AddrMode_Imm7<shift> {
173  // They are printed the same way as the imm8 version
174  let PrintMethod = "printT2AddrModeImm8Operand<true>";
175  let ParserMatchClass =
176    !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetWBAsmOperand");
177  let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 1>";
178  let MIOperandInfo = (ops rGPR:$base, i32imm:$offsim);
179}
180
181class t2am_imm7shiftOffsetAsmOperand<int shift>
182  : AsmOperandClass { let Name = "Imm7Shift"#shift; }
183def t2am_imm7shift0OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<0>;
184def t2am_imm7shift1OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<1>;
185def t2am_imm7shift2OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<2>;
186
187class t2am_imm7_offset<int shift> : MemOperand,
188      ComplexPattern<i32, 1, "SelectT2AddrModeImm7Offset<"#shift#">",
189                     [], [SDNPWantRoot]> {
190  // They are printed the same way as the imm8 version
191  let PrintMethod = "printT2AddrModeImm8OffsetOperand";
192  let ParserMatchClass =
193    !cast<AsmOperandClass>("t2am_imm7shift"#shift#"OffsetAsmOperand");
194  let EncoderMethod = "getT2ScaledImmOpValue<7,"#shift#">";
195  let DecoderMethod = "DecodeT2Imm7<"#shift#">";
196}
197
198// Operands for gather/scatter loads of the form [Rbase, Qoffsets]
199class MemRegRQOffsetAsmOperand<int shift> : AsmOperandClass {
200  let Name = "MemRegRQS"#shift#"Offset";
201  let PredicateMethod = "isMemRegRQOffset<"#shift#">";
202  let RenderMethod = "addMemRegRQOffsetOperands";
203}
204
205def MemRegRQS0OffsetAsmOperand : MemRegRQOffsetAsmOperand<0>;
206def MemRegRQS1OffsetAsmOperand : MemRegRQOffsetAsmOperand<1>;
207def MemRegRQS2OffsetAsmOperand : MemRegRQOffsetAsmOperand<2>;
208def MemRegRQS3OffsetAsmOperand : MemRegRQOffsetAsmOperand<3>;
209
210// mve_addr_rq_shift  := reg + vreg{ << UXTW #shift}
211class mve_addr_rq_shift<int shift> : MemOperand {
212  let EncoderMethod = "getMveAddrModeRQOpValue";
213  let PrintMethod = "printMveAddrModeRQOperand<"#shift#">";
214  let ParserMatchClass =
215    !cast<AsmOperandClass>("MemRegRQS"#shift#"OffsetAsmOperand");
216  let DecoderMethod = "DecodeMveAddrModeRQ";
217  let MIOperandInfo = (ops GPRnopc:$base, MQPR:$offsreg);
218}
219
220class MemRegQOffsetAsmOperand<int shift> : AsmOperandClass {
221  let Name = "MemRegQS"#shift#"Offset";
222  let PredicateMethod = "isMemRegQOffset<"#shift#">";
223  let RenderMethod = "addMemImmOffsetOperands";
224}
225
226def MemRegQS2OffsetAsmOperand : MemRegQOffsetAsmOperand<2>;
227def MemRegQS3OffsetAsmOperand : MemRegQOffsetAsmOperand<3>;
228
229// mve_addr_q_shift  := vreg {+ #imm7s2/4}
230class mve_addr_q_shift<int shift> : MemOperand {
231  let EncoderMethod = "getMveAddrModeQOpValue<"#shift#">";
232  // Can be printed same way as other reg + imm operands
233  let PrintMethod = "printT2AddrModeImm8Operand<false>";
234  let ParserMatchClass =
235    !cast<AsmOperandClass>("MemRegQS"#shift#"OffsetAsmOperand");
236  let DecoderMethod = "DecodeMveAddrModeQ<"#shift#">";
237  let MIOperandInfo = (ops MQPR:$base, i32imm:$imm);
238}
239
240// A family of classes wrapping up information about the vector types
241// used by MVE.
242class MVEVectorVTInfo<ValueType vec, ValueType dblvec,
243                      ValueType pred, ValueType dblpred,
244                      bits<2> size, string suffixletter, bit unsigned> {
245  // The LLVM ValueType representing the vector, so we can use it in
246  // ISel patterns.
247  ValueType Vec = vec;
248
249  // The LLVM ValueType representing a vector with elements double the size
250  // of those in Vec, so we can use it in ISel patterns. It is up to the
251  // invoker of this class to ensure that this is a correct choice.
252  ValueType DblVec = dblvec;
253
254  // An LLVM ValueType representing a corresponding vector of
255  // predicate bits, for use in ISel patterns that handle an IR
256  // intrinsic describing the predicated form of the instruction.
257  //
258  // Usually, for a vector of N things, this will be vNi1. But for
259  // vectors of 2 values, we make an exception, and use v4i1 instead
260  // of v2i1. Rationale: MVE codegen doesn't support doing all the
261  // auxiliary operations on v2i1 (vector shuffles etc), and also,
262  // there's no MVE compare instruction that will _generate_ v2i1
263  // directly.
264  ValueType Pred = pred;
265
266  // Same as Pred but for DblVec rather than Vec.
267  ValueType DblPred = dblpred;
268
269  // The most common representation of the vector element size in MVE
270  // instruction encodings: a 2-bit value V representing an (8<<V)-bit
271  // vector element.
272  bits<2> Size = size;
273
274  // For vectors explicitly mentioning a signedness of integers: 0 for
275  // signed and 1 for unsigned. For anything else, undefined.
276  bit Unsigned = unsigned;
277
278  // The number of bits in a vector element, in integer form.
279  int LaneBits = !shl(8, Size);
280
281  // The suffix used in assembly language on an instruction operating
282  // on this lane if it only cares about number of bits.
283  string BitsSuffix = !if(!eq(suffixletter, "p"),
284                          !if(!eq(unsigned, 0b0), "8", "16"),
285                          !cast<string>(LaneBits));
286
287  // The suffix used on an instruction that mentions the whole type.
288  string Suffix = suffixletter # BitsSuffix;
289
290  // The letter part of the suffix only.
291  string SuffixLetter = suffixletter;
292}
293
294// Integer vector types that don't treat signed and unsigned differently.
295def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>;
296def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "i", ?>;
297def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v4i1, 0b10, "i", ?>;
298def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?,     v4i1,  ?,    0b11, "i", ?>;
299
300// Explicitly signed and unsigned integer vectors. They map to the
301// same set of LLVM ValueTypes as above, but are represented
302// differently in assembly and instruction encodings.
303def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>;
304def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "s", 0b0>;
305def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v4i1, 0b10, "s", 0b0>;
306def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?,     v4i1,  ?,    0b11, "s", 0b0>;
307def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>;
308def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "u", 0b1>;
309def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v4i1, 0b10, "u", 0b1>;
310def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?,     v4i1,  ?,    0b11, "u", 0b1>;
311
312// FP vector types.
313def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1,  v4i1, 0b01, "f", ?>;
314def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1,  v4i1, 0b10, "f", ?>;
315def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?,     v4i1,  ?,    0b11, "f", ?>;
316
317// Polynomial vector types.
318def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>;
319def MVE_v8p16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b11, "p", 0b1>;
320
321// --------- Start of base classes for the instructions themselves
322
323class MVE_MI<dag oops, dag iops, InstrItinClass itin, string asm,
324             string ops, string cstr, list<dag> pattern>
325  : Thumb2XI<oops, iops, AddrModeNone, 4, itin, !strconcat(asm, "\t", ops), cstr,
326             pattern>,
327    Requires<[HasMVEInt]> {
328  let D = MVEDomain;
329  let DecoderNamespace = "MVE";
330}
331
332// MVE_p is used for most predicated instructions, to add the cluster
333// of input operands that provides the VPT suffix (none, T or E) and
334// the input predicate register.
335class MVE_p<dag oops, dag iops, InstrItinClass itin, string iname,
336            string suffix, string ops, vpred_ops vpred, string cstr,
337            list<dag> pattern=[]>
338  : MVE_MI<oops, !con(iops, (ins vpred:$vp)), itin,
339           // If the instruction has a suffix, like vadd.f32, then the
340           // VPT predication suffix goes before the dot, so the full
341           // name has to be "vadd${vp}.f32".
342           !strconcat(iname, "${vp}",
343                      !if(!eq(suffix, ""), "", !strconcat(".", suffix))),
344           ops, !strconcat(cstr, vpred.vpred_constraint), pattern> {
345  let Inst{31-29} = 0b111;
346  let Inst{27-26} = 0b11;
347}
348
349class MVE_f<dag oops, dag iops, InstrItinClass itin, string iname,
350            string suffix, string ops, vpred_ops vpred, string cstr,
351            list<dag> pattern=[]>
352  : MVE_p<oops, iops, itin, iname, suffix, ops, vpred, cstr, pattern> {
353  let Predicates = [HasMVEFloat];
354}
355
356class MVE_MI_with_pred<dag oops, dag iops, InstrItinClass itin, string asm,
357                       string ops, string cstr, list<dag> pattern>
358  : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm, !strconcat("\t", ops), cstr,
359             pattern>,
360    Requires<[HasV8_1MMainline, HasMVEInt]> {
361  let D = MVEDomain;
362  let DecoderNamespace = "MVE";
363}
364
365class MVE_VMOV_lane_base<dag oops, dag iops, InstrItinClass itin, string asm,
366                         string suffix, string ops, string cstr,
367                         list<dag> pattern>
368  : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm,
369            !if(!eq(suffix, ""), "", "." # suffix) # "\t" # ops,
370            cstr, pattern>,
371    Requires<[HasV8_1MMainline, HasMVEInt]> {
372  let D = MVEDomain;
373  let DecoderNamespace = "MVE";
374}
375
376class MVE_ScalarShift<string iname, dag oops, dag iops, string asm, string cstr,
377            list<dag> pattern=[]>
378  : MVE_MI_with_pred<oops, iops, NoItinerary, iname, asm, cstr, pattern> {
379  let Inst{31-20} = 0b111010100101;
380  let Inst{8} = 0b1;
381
382}
383
384class MVE_ScalarShiftSingleReg<string iname, dag iops, string asm, string cstr,
385                    list<dag> pattern=[]>
386  : MVE_ScalarShift<iname, (outs rGPR:$RdaDest), iops, asm, cstr, pattern> {
387  bits<4> RdaDest;
388
389  let Inst{19-16} = RdaDest{3-0};
390}
391
392class MVE_ScalarShiftSRegImm<string iname, bits<2> op5_4>
393  : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, long_shift:$imm),
394                     "$RdaSrc, $imm", "$RdaDest = $RdaSrc",
395                     [(set rGPR:$RdaDest,
396                          (i32 (!cast<Intrinsic>("int_arm_mve_" # iname)
397                                    (i32 rGPR:$RdaSrc), (i32 imm:$imm))))]> {
398  bits<5> imm;
399
400  let Inst{15} = 0b0;
401  let Inst{14-12} = imm{4-2};
402  let Inst{11-8} = 0b1111;
403  let Inst{7-6} = imm{1-0};
404  let Inst{5-4} = op5_4{1-0};
405  let Inst{3-0} = 0b1111;
406}
407
408def MVE_SQSHL : MVE_ScalarShiftSRegImm<"sqshl", 0b11>;
409def MVE_SRSHR : MVE_ScalarShiftSRegImm<"srshr", 0b10>;
410def MVE_UQSHL : MVE_ScalarShiftSRegImm<"uqshl", 0b00>;
411def MVE_URSHR : MVE_ScalarShiftSRegImm<"urshr", 0b01>;
412
413class MVE_ScalarShiftSRegReg<string iname, bits<2> op5_4>
414  : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, rGPR:$Rm),
415                     "$RdaSrc, $Rm", "$RdaDest = $RdaSrc",
416                     [(set rGPR:$RdaDest,
417                         (i32 (!cast<Intrinsic>("int_arm_mve_" # iname)
418                                   (i32 rGPR:$RdaSrc), (i32 rGPR:$Rm))))]> {
419  bits<4> Rm;
420
421  let Inst{15-12} = Rm{3-0};
422  let Inst{11-8} = 0b1111;
423  let Inst{7-6} = 0b00;
424  let Inst{5-4} = op5_4{1-0};
425  let Inst{3-0} = 0b1101;
426
427  let Unpredictable{8-6} = 0b111;
428}
429
430def MVE_SQRSHR : MVE_ScalarShiftSRegReg<"sqrshr", 0b10>;
431def MVE_UQRSHL : MVE_ScalarShiftSRegReg<"uqrshl", 0b00>;
432
433class MVE_ScalarShiftDoubleReg<string iname, dag iops, string asm,
434                               string cstr, list<dag> pattern=[]>
435  : MVE_ScalarShift<iname, (outs tGPREven:$RdaLo, tGPROdd:$RdaHi),
436                    iops, asm, cstr, pattern> {
437  bits<4> RdaLo;
438  bits<4> RdaHi;
439
440  let Inst{19-17} = RdaLo{3-1};
441  let Inst{11-9} = RdaHi{3-1};
442
443  let hasSideEffects = 0;
444}
445
446class MVE_ScalarShiftDRegImm<string iname, bits<2> op5_4, bit op16,
447                             list<dag> pattern=[]>
448  : MVE_ScalarShiftDoubleReg<
449      iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, long_shift:$imm),
450      "$RdaLo, $RdaHi, $imm", "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
451      pattern> {
452  bits<5> imm;
453
454  let Inst{16} = op16;
455  let Inst{15} = 0b0;
456  let Inst{14-12} = imm{4-2};
457  let Inst{7-6} = imm{1-0};
458  let Inst{5-4} = op5_4{1-0};
459  let Inst{3-0} = 0b1111;
460}
461
462class MVE_ScalarShiftDRegRegBase<string iname, dag iops, string asm,
463                                 bit op5, bit op16, list<dag> pattern=[]>
464  : MVE_ScalarShiftDoubleReg<
465     iname, iops, asm, "@earlyclobber $RdaHi,@earlyclobber $RdaLo,"
466                       "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
467     pattern> {
468  bits<4> Rm;
469
470  let Inst{16} = op16;
471  let Inst{15-12} = Rm{3-0};
472  let Inst{6} = 0b0;
473  let Inst{5} = op5;
474  let Inst{4} = 0b0;
475  let Inst{3-0} = 0b1101;
476
477  // Custom decoder method because of the following overlapping encodings:
478  // ASRL and SQRSHR
479  // LSLL and UQRSHL
480  // SQRSHRL and SQRSHR
481  // UQRSHLL and UQRSHL
482  let DecoderMethod = "DecodeMVEOverlappingLongShift";
483}
484
485class MVE_ScalarShiftDRegReg<string iname, bit op5, list<dag> pattern=[]>
486  : MVE_ScalarShiftDRegRegBase<
487     iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm),
488     "$RdaLo, $RdaHi, $Rm", op5, 0b0, pattern> {
489
490  let Inst{7} = 0b0;
491}
492
493class MVE_ScalarShiftDRegRegWithSat<string iname, bit op5, list<dag> pattern=[]>
494  : MVE_ScalarShiftDRegRegBase<
495     iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm, saturateop:$sat),
496     "$RdaLo, $RdaHi, $sat, $Rm", op5, 0b1, pattern> {
497  bit sat;
498
499  let Inst{7} = sat;
500}
501
502def MVE_ASRLr   : MVE_ScalarShiftDRegReg<"asrl",    0b1,  [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
503                                        (ARMasrl tGPREven:$RdaLo_src,
504                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
505def MVE_ASRLi   : MVE_ScalarShiftDRegImm<"asrl",    0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
506                                        (ARMasrl tGPREven:$RdaLo_src,
507                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
508def MVE_LSLLr   : MVE_ScalarShiftDRegReg<"lsll",    0b0,  [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
509                                        (ARMlsll tGPREven:$RdaLo_src,
510                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
511def MVE_LSLLi   : MVE_ScalarShiftDRegImm<"lsll",    0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
512                                        (ARMlsll tGPREven:$RdaLo_src,
513                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
514def MVE_LSRL    : MVE_ScalarShiftDRegImm<"lsrl",    0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
515                                        (ARMlsrl tGPREven:$RdaLo_src,
516                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
517
518def MVE_SQRSHRL : MVE_ScalarShiftDRegRegWithSat<"sqrshrl", 0b1>;
519def MVE_SQSHLL  : MVE_ScalarShiftDRegImm<"sqshll",  0b11, 0b1>;
520def MVE_SRSHRL  : MVE_ScalarShiftDRegImm<"srshrl",  0b10, 0b1>;
521
522def MVE_UQRSHLL : MVE_ScalarShiftDRegRegWithSat<"uqrshll", 0b0>;
523def MVE_UQSHLL  : MVE_ScalarShiftDRegImm<"uqshll",  0b00, 0b1>;
524def MVE_URSHRL  : MVE_ScalarShiftDRegImm<"urshrl",  0b01, 0b1>;
525
526// start of mve_rDest instructions
527
528class MVE_rDest<dag oops, dag iops, InstrItinClass itin,
529                string iname, string suffix,
530                string ops, string cstr, list<dag> pattern=[]>
531// Always use vpred_n and not vpred_r: with the output register being
532// a GPR and not a vector register, there can't be any question of
533// what to put in its inactive lanes.
534  : MVE_p<oops, iops, itin, iname, suffix, ops, vpred_n, cstr, pattern> {
535
536  let Inst{25-23} = 0b101;
537  let Inst{11-9} = 0b111;
538  let Inst{4} = 0b0;
539}
540
541class MVE_VABAV<string suffix, bit U, bits<2> size>
542  : MVE_rDest<(outs rGPR:$Rda), (ins rGPR:$Rda_src, MQPR:$Qn, MQPR:$Qm),
543              NoItinerary, "vabav", suffix, "$Rda, $Qn, $Qm", "$Rda = $Rda_src",
544              []> {
545  bits<4> Qm;
546  bits<4> Qn;
547  bits<4> Rda;
548
549  let Inst{28} = U;
550  let Inst{22} = 0b0;
551  let Inst{21-20} = size{1-0};
552  let Inst{19-17} = Qn{2-0};
553  let Inst{16} = 0b0;
554  let Inst{15-12} = Rda{3-0};
555  let Inst{8} = 0b1;
556  let Inst{7} = Qn{3};
557  let Inst{6} = 0b0;
558  let Inst{5} = Qm{3};
559  let Inst{3-1} = Qm{2-0};
560  let Inst{0} = 0b1;
561  let horizontalReduction = 1;
562}
563
564multiclass MVE_VABAV_m<MVEVectorVTInfo VTI> {
565  def "" : MVE_VABAV<VTI.Suffix, VTI.Unsigned, VTI.Size>;
566  defvar Inst = !cast<Instruction>(NAME);
567
568  let Predicates = [HasMVEInt] in {
569    def : Pat<(i32 (int_arm_mve_vabav
570                         (i32 VTI.Unsigned),
571                         (i32 rGPR:$Rda_src),
572                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
573              (i32 (Inst (i32 rGPR:$Rda_src),
574                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
575
576    def : Pat<(i32 (int_arm_mve_vabav_predicated
577                         (i32 VTI.Unsigned),
578                         (i32 rGPR:$Rda_src),
579                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
580                         (VTI.Pred VCCR:$mask))),
581              (i32 (Inst (i32 rGPR:$Rda_src),
582                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
583                         ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
584  }
585}
586
587defm MVE_VABAVs8  : MVE_VABAV_m<MVE_v16s8>;
588defm MVE_VABAVs16 : MVE_VABAV_m<MVE_v8s16>;
589defm MVE_VABAVs32 : MVE_VABAV_m<MVE_v4s32>;
590defm MVE_VABAVu8  : MVE_VABAV_m<MVE_v16u8>;
591defm MVE_VABAVu16 : MVE_VABAV_m<MVE_v8u16>;
592defm MVE_VABAVu32 : MVE_VABAV_m<MVE_v4u32>;
593
594class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
595              bit A, bit U, bits<2> size, list<dag> pattern=[]>
596  : MVE_rDest<(outs tGPREven:$Rda), iops, NoItinerary,
597              iname, suffix, "$Rda, $Qm", cstr, pattern> {
598  bits<3> Qm;
599  bits<4> Rda;
600
601  let Inst{28} = U;
602  let Inst{22-20} = 0b111;
603  let Inst{19-18} = size{1-0};
604  let Inst{17-16} = 0b01;
605  let Inst{15-13} = Rda{3-1};
606  let Inst{12} = 0b0;
607  let Inst{8-6} = 0b100;
608  let Inst{5} = A;
609  let Inst{3-1} = Qm{2-0};
610  let Inst{0} = 0b0;
611  let horizontalReduction = 1;
612  let validForTailPredication = 1;
613}
614
615def ARMVADDVs       : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
616def ARMVADDVu       : SDNode<"ARMISD::VADDVu", SDTVecReduce>;
617
618multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
619  def acc    : MVE_VADDV<"vaddva", VTI.Suffix,
620                         (ins tGPREven:$Rda_src, MQPR:$Qm), "$Rda = $Rda_src",
621                         0b1, VTI.Unsigned, VTI.Size>;
622  def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,
623                         (ins MQPR:$Qm), "",
624                         0b0, VTI.Unsigned, VTI.Size>;
625
626  defvar InstA = !cast<Instruction>(NAME # "acc");
627  defvar InstN = !cast<Instruction>(NAME # "no_acc");
628
629  let Predicates = [HasMVEInt] in {
630    if VTI.Unsigned then {
631      def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
632                (i32 (InstN $vec))>;
633      def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
634                (i32 (InstN $vec))>;
635      def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
636                          (i32 tGPREven:$acc))),
637                (i32 (InstA $acc, $vec))>;
638      def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
639                          (i32 tGPREven:$acc))),
640                (i32 (InstA $acc, $vec))>;
641    } else {
642      def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
643                (i32 (InstN $vec))>;
644      def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
645                          (i32 tGPREven:$acc))),
646                (i32 (InstA $acc, $vec))>;
647    }
648
649    def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
650                                                (i32 VTI.Unsigned),
651                                                (VTI.Pred VCCR:$pred))),
652              (i32 (InstN $vec, ARMVCCThen, $pred))>;
653    def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
654                                                     (i32 VTI.Unsigned),
655                                                     (VTI.Pred VCCR:$pred)),
656                        (i32 tGPREven:$acc))),
657              (i32 (InstA $acc, $vec, ARMVCCThen, $pred))>;
658  }
659}
660
661defm MVE_VADDVs8  : MVE_VADDV_A<MVE_v16s8>;
662defm MVE_VADDVs16 : MVE_VADDV_A<MVE_v8s16>;
663defm MVE_VADDVs32 : MVE_VADDV_A<MVE_v4s32>;
664defm MVE_VADDVu8  : MVE_VADDV_A<MVE_v16u8>;
665defm MVE_VADDVu16 : MVE_VADDV_A<MVE_v8u16>;
666defm MVE_VADDVu32 : MVE_VADDV_A<MVE_v4u32>;
667
668class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,
669               bit A, bit U, list<dag> pattern=[]>
670  : MVE_rDest<(outs tGPREven:$RdaLo, tGPROdd:$RdaHi), iops, NoItinerary, iname,
671              suffix, "$RdaLo, $RdaHi, $Qm", cstr, pattern> {
672  bits<3> Qm;
673  bits<4> RdaLo;
674  bits<4> RdaHi;
675
676  let Inst{28} = U;
677  let Inst{22-20} = RdaHi{3-1};
678  let Inst{19-18} = 0b10;
679  let Inst{17-16} = 0b01;
680  let Inst{15-13} = RdaLo{3-1};
681  let Inst{12} = 0b0;
682  let Inst{8-6} = 0b100;
683  let Inst{5} = A;
684  let Inst{3-1} = Qm{2-0};
685  let Inst{0} = 0b0;
686  let horizontalReduction = 1;
687}
688
689def SDTVecReduceL : SDTypeProfile<2, 1, [    // VADDLV
690  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>
691]>;
692def SDTVecReduceLA : SDTypeProfile<2, 3, [    // VADDLVA
693  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
694  SDTCisVec<4>
695]>;
696def SDTVecReduceLP : SDTypeProfile<2, 2, [    // VADDLVp
697  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<2>
698]>;
699def SDTVecReduceLPA : SDTypeProfile<2, 4, [    // VADDLVAp
700  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
701  SDTCisVec<4>, SDTCisVec<5>
702]>;
703
704multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {
705  def acc    : MVE_VADDLV<"vaddlva", VTI.Suffix,
706                        (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),
707                        "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
708                        0b1, VTI.Unsigned>;
709  def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,
710                        (ins MQPR:$Qm), "",
711                        0b0, VTI.Unsigned>;
712
713  defvar InstA = !cast<Instruction>(NAME # "acc");
714  defvar InstN = !cast<Instruction>(NAME # "no_acc");
715
716  defvar letter = VTI.SuffixLetter;
717  defvar ARMVADDLV = SDNode<"ARMISD::VADDLV" # letter, SDTVecReduceL>;
718  defvar ARMVADDLVA = SDNode<"ARMISD::VADDLVA" # letter, SDTVecReduceLA>;
719  defvar ARMVADDLVp = SDNode<"ARMISD::VADDLVp" # letter, SDTVecReduceLP>;
720  defvar ARMVADDLVAp = SDNode<"ARMISD::VADDLVAp" # letter, SDTVecReduceLPA>;
721
722  let Predicates = [HasMVEInt] in {
723    def : Pat<(ARMVADDLV (v4i32 MQPR:$vec)),
724              (InstN (v4i32 MQPR:$vec))>;
725    def : Pat<(ARMVADDLVA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec)),
726              (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec))>;
727    def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),
728              (InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred))>;
729    def : Pat<(ARMVADDLVAp tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
730                           (VTI.Pred VCCR:$pred)),
731              (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
732                     ARMVCCThen, (VTI.Pred VCCR:$pred))>;
733  }
734}
735
736defm MVE_VADDLVs32 : MVE_VADDLV_A<MVE_v4s32>;
737defm MVE_VADDLVu32 : MVE_VADDLV_A<MVE_v4u32>;
738
739class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
740                     bit bit_17, bit bit_7, list<dag> pattern=[]>
741  : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm),
742              NoItinerary, iname, suffix, "$RdaSrc, $Qm",
743              "$RdaDest = $RdaSrc", pattern> {
744  bits<3> Qm;
745  bits<4> RdaDest;
746
747  let Inst{28} = sz;
748  let Inst{22-20} = 0b110;
749  let Inst{19-18} = 0b11;
750  let Inst{17} = bit_17;
751  let Inst{16} = 0b0;
752  let Inst{15-12} = RdaDest{3-0};
753  let Inst{8} = 0b1;
754  let Inst{7} = bit_7;
755  let Inst{6-5} = 0b00;
756  let Inst{3-1} = Qm{2-0};
757  let Inst{0} = 0b0;
758  let horizontalReduction = 1;
759
760  let Predicates = [HasMVEFloat];
761  let hasSideEffects = 0;
762}
763
764multiclass MVE_VMINMAXNMV_p<string iname, bit notAbs, bit isMin,
765                            MVEVectorVTInfo VTI, string intrBaseName,
766                            ValueType Scalar, RegisterClass ScalarReg> {
767  def "": MVE_VMINMAXNMV<iname, VTI.Suffix, VTI.Size{0}, notAbs, isMin>;
768  defvar Inst        = !cast<Instruction>(NAME);
769  defvar unpred_intr = !cast<Intrinsic>(intrBaseName);
770  defvar pred_intr   = !cast<Intrinsic>(intrBaseName#"_predicated");
771
772  let Predicates = [HasMVEFloat] in {
773    def : Pat<(Scalar (unpred_intr (Scalar ScalarReg:$prev),
774                                   (VTI.Vec MQPR:$vec))),
775           (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),
776                                   (VTI.Vec MQPR:$vec)),
777                              ScalarReg)>;
778    def : Pat<(Scalar (pred_intr   (Scalar ScalarReg:$prev),
779                                   (VTI.Vec MQPR:$vec),
780                                   (VTI.Pred VCCR:$pred))),
781           (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),
782                                   (VTI.Vec MQPR:$vec),
783                                   ARMVCCThen, (VTI.Pred VCCR:$pred)),
784                              ScalarReg)>;
785  }
786}
787
788multiclass MVE_VMINMAXNMV_fty<string iname, bit notAbs, bit isMin,
789                              string intrBase> {
790  defm f32 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v4f32, intrBase,
791                              f32, SPR>;
792  defm f16 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v8f16, intrBase,
793                              f16, HPR>;
794}
795
796defm MVE_VMINNMV : MVE_VMINMAXNMV_fty<"vminnmv",  1, 1, "int_arm_mve_minnmv">;
797defm MVE_VMAXNMV : MVE_VMINMAXNMV_fty<"vmaxnmv",  1, 0, "int_arm_mve_maxnmv">;
798defm MVE_VMINNMAV: MVE_VMINMAXNMV_fty<"vminnmav", 0, 1, "int_arm_mve_minnmav">;
799defm MVE_VMAXNMAV: MVE_VMINMAXNMV_fty<"vmaxnmav", 0, 0, "int_arm_mve_maxnmav">;
800
801class MVE_VMINMAXV<string iname, string suffix, bit U, bits<2> size,
802                 bit bit_17, bit bit_7, list<dag> pattern=[]>
803  : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm), NoItinerary,
804              iname, suffix, "$RdaSrc, $Qm", "$RdaDest = $RdaSrc", pattern> {
805  bits<3> Qm;
806  bits<4> RdaDest;
807
808  let Inst{28} = U;
809  let Inst{22-20} = 0b110;
810  let Inst{19-18} = size{1-0};
811  let Inst{17} = bit_17;
812  let Inst{16} = 0b0;
813  let Inst{15-12} = RdaDest{3-0};
814  let Inst{8} = 0b1;
815  let Inst{7} = bit_7;
816  let Inst{6-5} = 0b00;
817  let Inst{3-1} = Qm{2-0};
818  let Inst{0} = 0b0;
819  let horizontalReduction = 1;
820}
821
822multiclass MVE_VMINMAXV_p<string iname, bit notAbs, bit isMin,
823                          MVEVectorVTInfo VTI, string intrBaseName> {
824  def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
825                       notAbs, isMin>;
826  defvar Inst        = !cast<Instruction>(NAME);
827  defvar unpred_intr = !cast<Intrinsic>(intrBaseName);
828  defvar pred_intr   = !cast<Intrinsic>(intrBaseName#"_predicated");
829  defvar base_args   = (? (i32 rGPR:$prev), (VTI.Vec MQPR:$vec));
830  defvar args        = !if(notAbs, !con(base_args, (? (i32 VTI.Unsigned))),
831                           base_args);
832
833  let Predicates = [HasMVEInt] in {
834    def : Pat<(i32 !con(args, (unpred_intr))),
835              (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec)))>;
836    def : Pat<(i32 !con(args, (pred_intr (VTI.Pred VCCR:$pred)))),
837              (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec),
838                         ARMVCCThen, (VTI.Pred VCCR:$pred)))>;
839  }
840}
841
842multiclass MVE_VMINMAXV_ty<string iname, bit isMin, string intrBaseName> {
843  defm s8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16s8, intrBaseName>;
844  defm s16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8s16, intrBaseName>;
845  defm s32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4s32, intrBaseName>;
846  defm u8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16u8, intrBaseName>;
847  defm u16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8u16, intrBaseName>;
848  defm u32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4u32, intrBaseName>;
849}
850
851defm MVE_VMINV : MVE_VMINMAXV_ty<"vminv", 1, "int_arm_mve_minv">;
852defm MVE_VMAXV : MVE_VMINMAXV_ty<"vmaxv", 0, "int_arm_mve_maxv">;
853
854let Predicates = [HasMVEInt] in {
855  def : Pat<(i32 (vecreduce_smax (v16i8 MQPR:$src))),
856            (i32 (MVE_VMAXVs8 (t2MVNi (i32 127)), $src))>;
857  def : Pat<(i32 (vecreduce_smax (v8i16 MQPR:$src))),
858            (i32 (MVE_VMAXVs16 (t2MOVi32imm (i32 -32768)), $src))>;
859  def : Pat<(i32 (vecreduce_smax (v4i32 MQPR:$src))),
860            (i32 (MVE_VMAXVs32 (t2MOVi (i32 -2147483648)), $src))>;
861  def : Pat<(i32 (vecreduce_umax (v16i8 MQPR:$src))),
862            (i32 (MVE_VMAXVu8 (t2MOVi (i32 0)), $src))>;
863  def : Pat<(i32 (vecreduce_umax (v8i16 MQPR:$src))),
864            (i32 (MVE_VMAXVu16 (t2MOVi (i32 0)), $src))>;
865  def : Pat<(i32 (vecreduce_umax (v4i32 MQPR:$src))),
866            (i32 (MVE_VMAXVu32 (t2MOVi (i32 0)), $src))>;
867
868  def : Pat<(i32 (vecreduce_smin (v16i8 MQPR:$src))),
869            (i32 (MVE_VMINVs8 (t2MOVi (i32 127)), $src))>;
870  def : Pat<(i32 (vecreduce_smin (v8i16 MQPR:$src))),
871            (i32 (MVE_VMINVs16 (t2MOVi16 (i32 32767)), $src))>;
872  def : Pat<(i32 (vecreduce_smin (v4i32 MQPR:$src))),
873            (i32 (MVE_VMINVs32 (t2MVNi (i32 -2147483648)), $src))>;
874  def : Pat<(i32 (vecreduce_umin (v16i8 MQPR:$src))),
875            (i32 (MVE_VMINVu8 (t2MOVi (i32 255)), $src))>;
876  def : Pat<(i32 (vecreduce_umin (v8i16 MQPR:$src))),
877            (i32 (MVE_VMINVu16 (t2MOVi16 (i32 65535)), $src))>;
878  def : Pat<(i32 (vecreduce_umin (v4i32 MQPR:$src))),
879            (i32 (MVE_VMINVu32 (t2MOVi (i32 4294967295)), $src))>;
880
881}
882
883multiclass MVE_VMINMAXAV_ty<string iname, bit isMin, string intrBaseName> {
884  defm s8 : MVE_VMINMAXV_p<iname, 0, isMin, MVE_v16s8, intrBaseName>;
885  defm s16: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v8s16, intrBaseName>;
886  defm s32: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v4s32, intrBaseName>;
887}
888
889defm MVE_VMINAV : MVE_VMINMAXAV_ty<"vminav", 1, "int_arm_mve_minav">;
890defm MVE_VMAXAV : MVE_VMINMAXAV_ty<"vmaxav", 0, "int_arm_mve_maxav">;
891
892class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,
893                   bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0>
894  : MVE_rDest<(outs tGPREven:$RdaDest), iops, NoItinerary, iname, suffix,
895              "$RdaDest, $Qn, $Qm", cstr, []> {
896  bits<4> RdaDest;
897  bits<3> Qm;
898  bits<3> Qn;
899
900  let Inst{28} = bit_28;
901  let Inst{22-20} = 0b111;
902  let Inst{19-17} = Qn{2-0};
903  let Inst{16} = sz;
904  let Inst{15-13} = RdaDest{3-1};
905  let Inst{12} = X;
906  let Inst{8} = bit_8;
907  let Inst{7-6} = 0b00;
908  let Inst{5} = A;
909  let Inst{3-1} = Qm{2-0};
910  let Inst{0} = bit_0;
911  let horizontalReduction = 1;
912  // Allow tail predication for non-exchanging versions. As this is also a
913  // horizontalReduction, ARMLowOverheadLoops will also have to check that
914  // the vector operands contain zeros in their false lanes for the instruction
915  // to be properly valid.
916  let validForTailPredication = !eq(X, 0);
917}
918
919multiclass MVE_VMLAMLSDAV_A<string iname, string x, MVEVectorVTInfo VTI,
920                            bit sz, bit bit_28, bit X, bit bit_8, bit bit_0> {
921  def ""#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # x, VTI.Suffix,
922                                   (ins MQPR:$Qn, MQPR:$Qm), "",
923                                   sz, bit_28, 0b0, X, bit_8, bit_0>;
924  def "a"#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # "a" # x, VTI.Suffix,
925                                    (ins tGPREven:$RdaSrc, MQPR:$Qn, MQPR:$Qm),
926                                    "$RdaDest = $RdaSrc",
927                                    sz, bit_28, 0b1, X, bit_8, bit_0>;
928  let Predicates = [HasMVEInt] in {
929    def : Pat<(i32 (int_arm_mve_vmldava
930                            (i32 VTI.Unsigned),
931                            (i32 bit_0) /* subtract */,
932                            (i32 X) /* exchange */,
933                            (i32 0) /* accumulator */,
934                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
935              (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
936                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
937
938    def : Pat<(i32 (int_arm_mve_vmldava_predicated
939                            (i32 VTI.Unsigned),
940                            (i32 bit_0) /* subtract */,
941                            (i32 X) /* exchange */,
942                            (i32 0) /* accumulator */,
943                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
944                            (VTI.Pred VCCR:$mask))),
945              (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
946                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
947                             ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
948
949    def : Pat<(i32 (int_arm_mve_vmldava
950                            (i32 VTI.Unsigned),
951                            (i32 bit_0) /* subtract */,
952                            (i32 X) /* exchange */,
953                            (i32 tGPREven:$RdaSrc),
954                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
955              (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
956                            (i32 tGPREven:$RdaSrc),
957                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
958
959    def : Pat<(i32 (int_arm_mve_vmldava_predicated
960                            (i32 VTI.Unsigned),
961                            (i32 bit_0) /* subtract */,
962                            (i32 X) /* exchange */,
963                            (i32 tGPREven:$RdaSrc),
964                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
965                            (VTI.Pred VCCR:$mask))),
966              (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
967                            (i32 tGPREven:$RdaSrc),
968                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
969                             ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
970  }
971}
972
973multiclass MVE_VMLAMLSDAV_AX<string iname, MVEVectorVTInfo VTI, bit sz,
974                             bit bit_28, bit bit_8, bit bit_0> {
975  defm "" : MVE_VMLAMLSDAV_A<iname, "", VTI, sz, bit_28,
976                             0b0, bit_8, bit_0>;
977  defm "" : MVE_VMLAMLSDAV_A<iname, "x", VTI, sz, bit_28,
978                             0b1, bit_8, bit_0>;
979}
980
981multiclass MVE_VMLADAV_multi<MVEVectorVTInfo SVTI, MVEVectorVTInfo UVTI,
982                             bit sz, bit bit_8> {
983  defm "" : MVE_VMLAMLSDAV_AX<"vmladav", SVTI,
984                              sz, 0b0, bit_8, 0b0>;
985  defm "" : MVE_VMLAMLSDAV_A<"vmladav", "", UVTI,
986                             sz, 0b1, 0b0, bit_8, 0b0>;
987}
988
989multiclass MVE_VMLSDAV_multi<MVEVectorVTInfo VTI, bit sz, bit bit_28> {
990  defm "" : MVE_VMLAMLSDAV_AX<"vmlsdav", VTI,
991                              sz, bit_28, 0b0, 0b1>;
992}
993
994defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v16s8, MVE_v16u8, 0b0, 0b1>;
995defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v8s16, MVE_v8u16, 0b0, 0b0>;
996defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v4s32, MVE_v4u32, 0b1, 0b0>;
997
998defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v16s8, 0b0, 0b1>;
999defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v8s16, 0b0, 0b0>;
1000defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v4s32, 0b1, 0b0>;
1001
1002def SDTVecReduce2 : SDTypeProfile<1, 2, [    // VMLAV
1003  SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>
1004]>;
1005def SDTVecReduce2L : SDTypeProfile<2, 2, [    // VMLALV
1006  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>
1007]>;
1008def SDTVecReduce2LA : SDTypeProfile<2, 4, [    // VMLALVA
1009  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
1010  SDTCisVec<4>, SDTCisVec<5>
1011]>;
1012def ARMVMLAVs       : SDNode<"ARMISD::VMLAVs", SDTVecReduce2>;
1013def ARMVMLAVu       : SDNode<"ARMISD::VMLAVu", SDTVecReduce2>;
1014def ARMVMLALVs      : SDNode<"ARMISD::VMLALVs", SDTVecReduce2L>;
1015def ARMVMLALVu      : SDNode<"ARMISD::VMLALVu", SDTVecReduce2L>;
1016def ARMVMLALVAs      : SDNode<"ARMISD::VMLALVAs", SDTVecReduce2LA>;
1017def ARMVMLALVAu      : SDNode<"ARMISD::VMLALVAu", SDTVecReduce2LA>;
1018
1019let Predicates = [HasMVEInt] in {
1020  def : Pat<(i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
1021            (i32 (MVE_VMLADAVu32 $src1, $src2))>;
1022  def : Pat<(i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
1023            (i32 (MVE_VMLADAVu16 $src1, $src2))>;
1024  def : Pat<(i32 (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1025            (i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1026  def : Pat<(i32 (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1027            (i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1028  def : Pat<(i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
1029            (i32 (MVE_VMLADAVu8 $src1, $src2))>;
1030  def : Pat<(i32 (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1031            (i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1032  def : Pat<(i32 (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1033            (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1034
1035  def : Pat<(i32 (add (i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
1036                                          (i32 tGPREven:$src3))),
1037            (i32 (MVE_VMLADAVau32 $src3, $src1, $src2))>;
1038  def : Pat<(i32 (add (i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
1039                                          (i32 tGPREven:$src3))),
1040            (i32 (MVE_VMLADAVau16 $src3, $src1, $src2))>;
1041  def : Pat<(i32 (add (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
1042            (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1043  def : Pat<(i32 (add (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
1044            (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1045  def : Pat<(i32 (add (i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
1046                                          (i32 tGPREven:$src3))),
1047            (i32 (MVE_VMLADAVau8 $src3, $src1, $src2))>;
1048  def : Pat<(i32 (add (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
1049            (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1050  def : Pat<(i32 (add (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
1051            (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1052}
1053
1054// vmlav aliases vmladav
1055foreach acc = ["", "a"] in {
1056  foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32"] in {
1057    def : MVEInstAlias<"vmlav"#acc#"${vp}."#suffix#"\t$RdaDest, $Qn, $Qm",
1058                       (!cast<Instruction>("MVE_VMLADAV"#acc#suffix)
1059                        tGPREven:$RdaDest, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1060  }
1061}
1062
1063// Base class for VMLALDAV and VMLSLDAV, VRMLALDAVH, VRMLSLDAVH
1064class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,
1065                       bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
1066                       list<dag> pattern=[]>
1067  : MVE_rDest<(outs tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest), iops, NoItinerary,
1068              iname, suffix, "$RdaLoDest, $RdaHiDest, $Qn, $Qm", cstr, pattern> {
1069  bits<4> RdaLoDest;
1070  bits<4> RdaHiDest;
1071  bits<3> Qm;
1072  bits<3> Qn;
1073
1074  let Inst{28} = bit_28;
1075  let Inst{22-20} = RdaHiDest{3-1};
1076  let Inst{19-17} = Qn{2-0};
1077  let Inst{16} = sz;
1078  let Inst{15-13} = RdaLoDest{3-1};
1079  let Inst{12} = X;
1080  let Inst{8} = bit_8;
1081  let Inst{7-6} = 0b00;
1082  let Inst{5} = A;
1083  let Inst{3-1} = Qm{2-0};
1084  let Inst{0} = bit_0;
1085  let horizontalReduction = 1;
1086  // Allow tail predication for non-exchanging versions. As this is also a
1087  // horizontalReduction, ARMLowOverheadLoops will also have to check that
1088  // the vector operands contain zeros in their false lanes for the instruction
1089  // to be properly valid.
1090  let validForTailPredication = !eq(X, 0);
1091
1092  let hasSideEffects = 0;
1093}
1094
1095multiclass MVE_VMLALDAVBase_A<string iname, string x, string suffix,
1096                               bit sz, bit bit_28, bit X, bit bit_8, bit bit_0,
1097                               list<dag> pattern=[]> {
1098  def ""#x#suffix : MVE_VMLALDAVBase<
1099     iname # x, suffix, (ins MQPR:$Qn, MQPR:$Qm), "",
1100     sz, bit_28, 0b0, X, bit_8, bit_0, pattern>;
1101  def "a"#x#suffix : MVE_VMLALDAVBase<
1102     iname # "a" # x, suffix,
1103     (ins tGPREven:$RdaLoSrc, tGPROdd:$RdaHiSrc, MQPR:$Qn, MQPR:$Qm),
1104     "$RdaLoDest = $RdaLoSrc,$RdaHiDest = $RdaHiSrc",
1105     sz, bit_28, 0b1, X, bit_8, bit_0, pattern>;
1106}
1107
1108
1109multiclass MVE_VMLALDAVBase_AX<string iname, string suffix, bit sz, bit bit_28,
1110                               bit bit_8, bit bit_0, list<dag> pattern=[]> {
1111  defm "" : MVE_VMLALDAVBase_A<iname, "", suffix, sz,
1112                               bit_28, 0b0, bit_8, bit_0, pattern>;
1113  defm "" : MVE_VMLALDAVBase_A<iname, "x", suffix, sz,
1114                               bit_28, 0b1, bit_8, bit_0, pattern>;
1115}
1116
1117multiclass MVE_VRMLALDAVH_multi<string suffix, list<dag> pattern=[]> {
1118  defm "" : MVE_VMLALDAVBase_AX<"vrmlaldavh", "s"#suffix,
1119                                0b0, 0b0, 0b1, 0b0, pattern>;
1120  defm "" : MVE_VMLALDAVBase_A<"vrmlaldavh", "", "u"#suffix,
1121                               0b0, 0b1, 0b0, 0b1, 0b0, pattern>;
1122}
1123
1124defm MVE_VRMLALDAVH : MVE_VRMLALDAVH_multi<"32">;
1125
1126// vrmlalvh aliases for vrmlaldavh
1127def : MVEInstAlias<"vrmlalvh${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
1128                  (MVE_VRMLALDAVHs32
1129                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1130                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1131def : MVEInstAlias<"vrmlalvha${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
1132                  (MVE_VRMLALDAVHas32
1133                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1134                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1135def : MVEInstAlias<"vrmlalvh${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
1136                  (MVE_VRMLALDAVHu32
1137                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1138                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1139def : MVEInstAlias<"vrmlalvha${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
1140                  (MVE_VRMLALDAVHau32
1141                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1142                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1143
1144multiclass MVE_VMLALDAV_multi<string suffix, bit sz, list<dag> pattern=[]> {
1145  defm "" : MVE_VMLALDAVBase_AX<"vmlaldav", "s"#suffix, sz, 0b0, 0b0, 0b0, pattern>;
1146  defm "" : MVE_VMLALDAVBase_A<"vmlaldav", "", "u"#suffix,
1147                               sz, 0b1, 0b0, 0b0, 0b0, pattern>;
1148}
1149
1150defm MVE_VMLALDAV : MVE_VMLALDAV_multi<"16", 0b0>;
1151defm MVE_VMLALDAV : MVE_VMLALDAV_multi<"32", 0b1>;
1152
1153let Predicates = [HasMVEInt] in {
1154  def : Pat<(ARMVMLALVs (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1155            (MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1156  def : Pat<(ARMVMLALVu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1157            (MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1158  def : Pat<(ARMVMLALVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1159            (MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1160  def : Pat<(ARMVMLALVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1161            (MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1162
1163  def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1164            (MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1165  def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1166            (MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1167  def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1168            (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1169  def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1170            (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1171}
1172
1173// vmlalv aliases vmlaldav
1174foreach acc = ["", "a"] in {
1175  foreach suffix = ["s16", "s32", "u16", "u32"] in {
1176    def : MVEInstAlias<"vmlalv" # acc # "${vp}." # suffix #
1177                          "\t$RdaLoDest, $RdaHiDest, $Qn, $Qm",
1178                       (!cast<Instruction>("MVE_VMLALDAV"#acc#suffix)
1179                       tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest,
1180                       MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1181  }
1182}
1183
1184multiclass MVE_VMLSLDAV_multi<string iname, string suffix, bit sz,
1185                              bit bit_28, list<dag> pattern=[]> {
1186  defm "" : MVE_VMLALDAVBase_AX<iname, suffix, sz, bit_28, 0b0, 0b1, pattern>;
1187}
1188
1189defm MVE_VMLSLDAV   : MVE_VMLSLDAV_multi<"vmlsldav", "s16", 0b0, 0b0>;
1190defm MVE_VMLSLDAV   : MVE_VMLSLDAV_multi<"vmlsldav", "s32", 0b1, 0b0>;
1191defm MVE_VRMLSLDAVH : MVE_VMLSLDAV_multi<"vrmlsldavh", "s32", 0b0, 0b1>;
1192
1193// end of mve_rDest instructions
1194
1195// start of mve_comp instructions
1196
1197class MVE_comp<InstrItinClass itin, string iname, string suffix,
1198               string cstr, list<dag> pattern=[]>
1199  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), itin, iname, suffix,
1200           "$Qd, $Qn, $Qm", vpred_r, cstr, pattern> {
1201  bits<4> Qd;
1202  bits<4> Qn;
1203  bits<4> Qm;
1204
1205  let Inst{22} = Qd{3};
1206  let Inst{19-17} = Qn{2-0};
1207  let Inst{16} = 0b0;
1208  let Inst{15-13} = Qd{2-0};
1209  let Inst{12} = 0b0;
1210  let Inst{10-9} = 0b11;
1211  let Inst{7} = Qn{3};
1212  let Inst{5} = Qm{3};
1213  let Inst{3-1} = Qm{2-0};
1214  let Inst{0} = 0b0;
1215}
1216
1217class MVE_VMINMAXNM<string iname, string suffix, bit sz, bit bit_21,
1218                list<dag> pattern=[]>
1219  : MVE_comp<NoItinerary, iname, suffix, "", pattern> {
1220
1221  let Inst{28} = 0b1;
1222  let Inst{25-24} = 0b11;
1223  let Inst{23} = 0b0;
1224  let Inst{21} = bit_21;
1225  let Inst{20} = sz;
1226  let Inst{11} = 0b1;
1227  let Inst{8} = 0b1;
1228  let Inst{6} = 0b1;
1229  let Inst{4} = 0b1;
1230
1231  let Predicates = [HasMVEFloat];
1232}
1233
1234def MVE_VMAXNMf32 : MVE_VMINMAXNM<"vmaxnm", "f32", 0b0, 0b0>;
1235def MVE_VMAXNMf16 : MVE_VMINMAXNM<"vmaxnm", "f16", 0b1, 0b0>;
1236
1237let Predicates = [HasMVEFloat] in {
1238  def : Pat<(v4f32 (fmaxnum (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
1239            (v4f32 (MVE_VMAXNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
1240  def : Pat<(v8f16 (fmaxnum (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
1241            (v8f16 (MVE_VMAXNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
1242  def : Pat<(v4f32 (int_arm_mve_max_predicated (v4f32 MQPR:$val1), (v4f32 MQPR:$val2), (i32 0),
1243                          (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive))),
1244            (v4f32 (MVE_VMAXNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2),
1245                          ARMVCCThen, (v4i1 VCCR:$mask),
1246                          (v4f32 MQPR:$inactive)))>;
1247  def : Pat<(v8f16 (int_arm_mve_max_predicated (v8f16 MQPR:$val1), (v8f16 MQPR:$val2), (i32 0),
1248                          (v8i1 VCCR:$mask), (v8f16 MQPR:$inactive))),
1249            (v8f16 (MVE_VMAXNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2),
1250                          ARMVCCThen, (v8i1 VCCR:$mask),
1251                          (v8f16 MQPR:$inactive)))>;
1252}
1253
1254def MVE_VMINNMf32 : MVE_VMINMAXNM<"vminnm", "f32", 0b0, 0b1>;
1255def MVE_VMINNMf16 : MVE_VMINMAXNM<"vminnm", "f16", 0b1, 0b1>;
1256
1257let Predicates = [HasMVEFloat] in {
1258  def : Pat<(v4f32 (fminnum (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
1259            (v4f32 (MVE_VMINNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
1260  def : Pat<(v8f16 (fminnum (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
1261            (v8f16 (MVE_VMINNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
1262  def : Pat<(v4f32 (int_arm_mve_min_predicated (v4f32 MQPR:$val1), (v4f32 MQPR:$val2),
1263                          (i32 0), (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive))),
1264            (v4f32 (MVE_VMINNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2),
1265                          ARMVCCThen, (v4i1 VCCR:$mask),
1266                          (v4f32 MQPR:$inactive)))>;
1267  def : Pat<(v8f16 (int_arm_mve_min_predicated (v8f16 MQPR:$val1), (v8f16 MQPR:$val2),
1268                          (i32 0), (v8i1 VCCR:$mask), (v8f16 MQPR:$inactive))),
1269            (v8f16 (MVE_VMINNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2),
1270                          ARMVCCThen, (v8i1 VCCR:$mask),
1271                          (v8f16 MQPR:$inactive)))>;
1272}
1273
1274
1275class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
1276              bit bit_4, list<dag> pattern=[]>
1277  : MVE_comp<NoItinerary, iname, suffix, "", pattern> {
1278
1279  let Inst{28} = U;
1280  let Inst{25-24} = 0b11;
1281  let Inst{23} = 0b0;
1282  let Inst{21-20} = size{1-0};
1283  let Inst{11} = 0b0;
1284  let Inst{8} = 0b0;
1285  let Inst{6} = 0b1;
1286  let Inst{4} = bit_4;
1287  let validForTailPredication = 1;
1288}
1289
1290multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,
1291                      SDNode unpred_op, Intrinsic pred_int> {
1292  def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;
1293  defvar Inst = !cast<Instruction>(NAME);
1294
1295  let Predicates = [HasMVEInt] in {
1296    // Unpredicated min/max
1297    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1298              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1299
1300    // Predicated min/max
1301    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1302                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
1303                            (VTI.Vec MQPR:$inactive))),
1304              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1305                             ARMVCCThen, (VTI.Pred VCCR:$mask),
1306                             (VTI.Vec MQPR:$inactive)))>;
1307  }
1308}
1309
1310multiclass MVE_VMAX<MVEVectorVTInfo VTI>
1311  : MVE_VMINMAX_m<"vmax", 0b0, VTI, !if(VTI.Unsigned, umax, smax), int_arm_mve_max_predicated>;
1312multiclass MVE_VMIN<MVEVectorVTInfo VTI>
1313  : MVE_VMINMAX_m<"vmin", 0b1, VTI, !if(VTI.Unsigned, umin, smin), int_arm_mve_min_predicated>;
1314
1315defm MVE_VMINs8   : MVE_VMIN<MVE_v16s8>;
1316defm MVE_VMINs16  : MVE_VMIN<MVE_v8s16>;
1317defm MVE_VMINs32  : MVE_VMIN<MVE_v4s32>;
1318defm MVE_VMINu8   : MVE_VMIN<MVE_v16u8>;
1319defm MVE_VMINu16  : MVE_VMIN<MVE_v8u16>;
1320defm MVE_VMINu32  : MVE_VMIN<MVE_v4u32>;
1321
1322defm MVE_VMAXs8   : MVE_VMAX<MVE_v16s8>;
1323defm MVE_VMAXs16  : MVE_VMAX<MVE_v8s16>;
1324defm MVE_VMAXs32  : MVE_VMAX<MVE_v4s32>;
1325defm MVE_VMAXu8   : MVE_VMAX<MVE_v16u8>;
1326defm MVE_VMAXu16  : MVE_VMAX<MVE_v8u16>;
1327defm MVE_VMAXu32  : MVE_VMAX<MVE_v4u32>;
1328
1329// end of mve_comp instructions
1330
1331// start of mve_bit instructions
1332
1333class MVE_bit_arith<dag oops, dag iops, string iname, string suffix,
1334                    string ops, string cstr, list<dag> pattern=[]>
1335  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred_r, cstr, pattern> {
1336  bits<4> Qd;
1337  bits<4> Qm;
1338
1339  let Inst{22} = Qd{3};
1340  let Inst{15-13} = Qd{2-0};
1341  let Inst{5} = Qm{3};
1342  let Inst{3-1} = Qm{2-0};
1343}
1344
1345def MVE_VBIC : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
1346                             "vbic", "", "$Qd, $Qn, $Qm", ""> {
1347  bits<4> Qn;
1348
1349  let Inst{28} = 0b0;
1350  let Inst{25-23} = 0b110;
1351  let Inst{21-20} = 0b01;
1352  let Inst{19-17} = Qn{2-0};
1353  let Inst{16} = 0b0;
1354  let Inst{12-8} = 0b00001;
1355  let Inst{7} = Qn{3};
1356  let Inst{6} = 0b1;
1357  let Inst{4} = 0b1;
1358  let Inst{0} = 0b0;
1359  let validForTailPredication = 1;
1360}
1361
1362class MVE_VREV<string iname, string suffix, bits<2> size, bits<2> bit_8_7, string cstr="">
1363  : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm), iname,
1364                  suffix, "$Qd, $Qm", cstr> {
1365
1366  let Inst{28} = 0b1;
1367  let Inst{25-23} = 0b111;
1368  let Inst{21-20} = 0b11;
1369  let Inst{19-18} = size;
1370  let Inst{17-16} = 0b00;
1371  let Inst{12-9} = 0b0000;
1372  let Inst{8-7} = bit_8_7;
1373  let Inst{6} = 0b1;
1374  let Inst{4} = 0b0;
1375  let Inst{0} = 0b0;
1376}
1377
1378def MVE_VREV64_8  : MVE_VREV<"vrev64", "8", 0b00, 0b00, "@earlyclobber $Qd">;
1379def MVE_VREV64_16 : MVE_VREV<"vrev64", "16", 0b01, 0b00, "@earlyclobber $Qd">;
1380def MVE_VREV64_32 : MVE_VREV<"vrev64", "32", 0b10, 0b00, "@earlyclobber $Qd">;
1381
1382def MVE_VREV32_8  : MVE_VREV<"vrev32", "8", 0b00, 0b01>;
1383def MVE_VREV32_16 : MVE_VREV<"vrev32", "16", 0b01, 0b01>;
1384
1385def MVE_VREV16_8  : MVE_VREV<"vrev16", "8", 0b00, 0b10>;
1386
1387let Predicates = [HasMVEInt] in {
1388  def : Pat<(v8i16 (bswap (v8i16 MQPR:$src))),
1389            (v8i16 (MVE_VREV16_8 (v8i16 MQPR:$src)))>;
1390  def : Pat<(v4i32 (bswap (v4i32 MQPR:$src))),
1391            (v4i32 (MVE_VREV32_8 (v4i32 MQPR:$src)))>;
1392}
1393
1394multiclass MVE_VREV_basic_patterns<int revbits, list<MVEVectorVTInfo> VTIs,
1395                                   Instruction Inst> {
1396  defvar unpred_op = !cast<SDNode>("ARMvrev" # revbits);
1397
1398  foreach VTI = VTIs in {
1399    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src))),
1400              (VTI.Vec (Inst (VTI.Vec MQPR:$src)))>;
1401    def : Pat<(VTI.Vec (int_arm_mve_vrev_predicated (VTI.Vec MQPR:$src),
1402                  revbits, (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1403              (VTI.Vec (Inst (VTI.Vec MQPR:$src), ARMVCCThen,
1404                  (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
1405  }
1406}
1407
1408let Predicates = [HasMVEInt] in {
1409  defm: MVE_VREV_basic_patterns<64, [MVE_v4i32, MVE_v4f32], MVE_VREV64_32>;
1410  defm: MVE_VREV_basic_patterns<64, [MVE_v8i16, MVE_v8f16], MVE_VREV64_16>;
1411  defm: MVE_VREV_basic_patterns<64, [MVE_v16i8           ], MVE_VREV64_8>;
1412
1413  defm: MVE_VREV_basic_patterns<32, [MVE_v8i16, MVE_v8f16], MVE_VREV32_16>;
1414  defm: MVE_VREV_basic_patterns<32, [MVE_v16i8           ], MVE_VREV32_8>;
1415
1416  defm: MVE_VREV_basic_patterns<16, [MVE_v16i8           ], MVE_VREV16_8>;
1417}
1418
1419def MVE_VMVN : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm),
1420                             "vmvn", "", "$Qd, $Qm", ""> {
1421  let Inst{28} = 0b1;
1422  let Inst{25-23} = 0b111;
1423  let Inst{21-16} = 0b110000;
1424  let Inst{12-6} = 0b0010111;
1425  let Inst{4} = 0b0;
1426  let Inst{0} = 0b0;
1427  let validForTailPredication = 1;
1428}
1429
1430let Predicates = [HasMVEInt] in {
1431  foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in {
1432    def : Pat<(VTI.Vec (vnotq    (VTI.Vec MQPR:$val1))),
1433              (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1)))>;
1434    def : Pat<(VTI.Vec (int_arm_mve_mvn_predicated (VTI.Vec MQPR:$val1),
1435                       (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1436              (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1), ARMVCCThen,
1437                       (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
1438  }
1439}
1440
1441class MVE_bit_ops<string iname, bits<2> bit_21_20, bit bit_28>
1442  : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
1443                  iname, "", "$Qd, $Qn, $Qm", ""> {
1444  bits<4> Qn;
1445
1446  let Inst{28} = bit_28;
1447  let Inst{25-23} = 0b110;
1448  let Inst{21-20} = bit_21_20;
1449  let Inst{19-17} = Qn{2-0};
1450  let Inst{16} = 0b0;
1451  let Inst{12-8} = 0b00001;
1452  let Inst{7} = Qn{3};
1453  let Inst{6} = 0b1;
1454  let Inst{4} = 0b1;
1455  let Inst{0} = 0b0;
1456  let validForTailPredication = 1;
1457}
1458
1459def MVE_VEOR : MVE_bit_ops<"veor", 0b00, 0b1>;
1460def MVE_VORN : MVE_bit_ops<"vorn", 0b11, 0b0>;
1461def MVE_VORR : MVE_bit_ops<"vorr", 0b10, 0b0>;
1462def MVE_VAND : MVE_bit_ops<"vand", 0b00, 0b0>;
1463
1464// add ignored suffixes as aliases
1465
1466foreach s=["s8", "s16", "s32", "u8", "u16", "u32", "i8", "i16", "i32", "f16", "f32"] in {
1467  def : MVEInstAlias<"vbic${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1468        (MVE_VBIC MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1469  def : MVEInstAlias<"veor${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1470        (MVE_VEOR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1471  def : MVEInstAlias<"vorn${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1472        (MVE_VORN MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1473  def : MVEInstAlias<"vorr${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1474        (MVE_VORR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1475  def : MVEInstAlias<"vand${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1476        (MVE_VAND MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1477}
1478
1479multiclass MVE_bit_op<MVEVectorVTInfo VTI, SDNode unpred_op, Intrinsic pred_int, MVE_bit_ops instruction> {
1480  let Predicates = [HasMVEInt] in {
1481    // Unpredicated operation
1482    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1483              (VTI.Vec (instruction (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1484    // Predicated operation
1485    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1486                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
1487              (VTI.Vec (instruction
1488                            (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1489                            ARMVCCThen, (VTI.Pred VCCR:$mask),
1490                            (VTI.Vec MQPR:$inactive)))>;
1491  }
1492}
1493
1494defm : MVE_bit_op<MVE_v16i8, and, int_arm_mve_and_predicated, MVE_VAND>;
1495defm : MVE_bit_op<MVE_v8i16, and, int_arm_mve_and_predicated, MVE_VAND>;
1496defm : MVE_bit_op<MVE_v4i32, and, int_arm_mve_and_predicated, MVE_VAND>;
1497defm : MVE_bit_op<MVE_v2i64, and, int_arm_mve_and_predicated, MVE_VAND>;
1498
1499defm : MVE_bit_op<MVE_v16i8, or, int_arm_mve_orr_predicated, MVE_VORR>;
1500defm : MVE_bit_op<MVE_v8i16, or, int_arm_mve_orr_predicated, MVE_VORR>;
1501defm : MVE_bit_op<MVE_v4i32, or, int_arm_mve_orr_predicated, MVE_VORR>;
1502defm : MVE_bit_op<MVE_v2i64, or, int_arm_mve_orr_predicated, MVE_VORR>;
1503
1504defm : MVE_bit_op<MVE_v16i8, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
1505defm : MVE_bit_op<MVE_v8i16, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
1506defm : MVE_bit_op<MVE_v4i32, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
1507defm : MVE_bit_op<MVE_v2i64, xor, int_arm_mve_eor_predicated, MVE_VEOR>;
1508
1509multiclass MVE_bit_op_with_inv<MVEVectorVTInfo VTI, SDNode unpred_op, Intrinsic pred_int, MVE_bit_ops instruction> {
1510  let Predicates = [HasMVEInt] in {
1511    // Unpredicated operation
1512    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (vnotq (VTI.Vec MQPR:$Qn)))),
1513              (VTI.Vec (instruction (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1514    // Predicated operation
1515    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1516                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
1517              (VTI.Vec (instruction
1518                            (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1519                            ARMVCCThen, (VTI.Pred VCCR:$mask),
1520                            (VTI.Vec MQPR:$inactive)))>;
1521  }
1522}
1523
1524defm : MVE_bit_op_with_inv<MVE_v16i8, and, int_arm_mve_bic_predicated, MVE_VBIC>;
1525defm : MVE_bit_op_with_inv<MVE_v8i16, and, int_arm_mve_bic_predicated, MVE_VBIC>;
1526defm : MVE_bit_op_with_inv<MVE_v4i32, and, int_arm_mve_bic_predicated, MVE_VBIC>;
1527defm : MVE_bit_op_with_inv<MVE_v2i64, and, int_arm_mve_bic_predicated, MVE_VBIC>;
1528
1529defm : MVE_bit_op_with_inv<MVE_v16i8, or, int_arm_mve_orn_predicated, MVE_VORN>;
1530defm : MVE_bit_op_with_inv<MVE_v8i16, or, int_arm_mve_orn_predicated, MVE_VORN>;
1531defm : MVE_bit_op_with_inv<MVE_v4i32, or, int_arm_mve_orn_predicated, MVE_VORN>;
1532defm : MVE_bit_op_with_inv<MVE_v2i64, or, int_arm_mve_orn_predicated, MVE_VORN>;
1533
1534class MVE_bit_cmode<string iname, string suffix, bit halfword, dag inOps>
1535  : MVE_p<(outs MQPR:$Qd), inOps, NoItinerary,
1536          iname, suffix, "$Qd, $imm", vpred_n, "$Qd = $Qd_src"> {
1537  bits<12> imm;
1538  bits<4> Qd;
1539
1540  let Inst{28} = imm{7};
1541  let Inst{27-23} = 0b11111;
1542  let Inst{22} = Qd{3};
1543  let Inst{21-19} = 0b000;
1544  let Inst{18-16} = imm{6-4};
1545  let Inst{15-13} = Qd{2-0};
1546  let Inst{12} = 0b0;
1547  let Inst{11} = halfword;
1548  let Inst{10} = !if(halfword, 0, imm{10});
1549  let Inst{9} = imm{9};
1550  let Inst{8} = 0b1;
1551  let Inst{7-6} = 0b01;
1552  let Inst{4} = 0b1;
1553  let Inst{3-0} = imm{3-0};
1554}
1555
1556multiclass MVE_bit_cmode_p<string iname, bit opcode,
1557                           MVEVectorVTInfo VTI, Operand imm_type, SDNode op> {
1558  def "" : MVE_bit_cmode<iname, VTI.Suffix, VTI.Size{0},
1559                         (ins MQPR:$Qd_src, imm_type:$imm)> {
1560    let Inst{5} = opcode;
1561    let validForTailPredication = 1;
1562  }
1563
1564  defvar Inst = !cast<Instruction>(NAME);
1565  defvar UnpredPat = (VTI.Vec (op (VTI.Vec MQPR:$src), timm:$simm));
1566
1567  let Predicates = [HasMVEInt] in {
1568    def : Pat<UnpredPat, (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm))>;
1569    def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
1570                          UnpredPat, (VTI.Vec MQPR:$src))),
1571              (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm,
1572                             ARMVCCThen, (VTI.Pred VCCR:$pred)))>;
1573  }
1574}
1575
1576multiclass MVE_VORRimm<MVEVectorVTInfo VTI, Operand imm_type> {
1577  defm "": MVE_bit_cmode_p<"vorr", 0, VTI, imm_type, ARMvorrImm>;
1578}
1579multiclass MVE_VBICimm<MVEVectorVTInfo VTI, Operand imm_type> {
1580  defm "": MVE_bit_cmode_p<"vbic", 1, VTI, imm_type, ARMvbicImm>;
1581}
1582
1583defm MVE_VORRimmi16 : MVE_VORRimm<MVE_v8i16, nImmSplatI16>;
1584defm MVE_VORRimmi32 : MVE_VORRimm<MVE_v4i32, nImmSplatI32>;
1585defm MVE_VBICimmi16 : MVE_VBICimm<MVE_v8i16, nImmSplatI16>;
1586defm MVE_VBICimmi32 : MVE_VBICimm<MVE_v4i32, nImmSplatI32>;
1587
1588def MVE_VORNimmi16 : MVEInstAlias<"vorn${vp}.i16\t$Qd, $imm",
1589    (MVE_VORRimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;
1590def MVE_VORNimmi32 : MVEInstAlias<"vorn${vp}.i32\t$Qd, $imm",
1591    (MVE_VORRimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;
1592
1593def MVE_VANDimmi16 : MVEInstAlias<"vand${vp}.i16\t$Qd, $imm",
1594    (MVE_VBICimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;
1595def MVE_VANDimmi32 : MVEInstAlias<"vand${vp}.i32\t$Qd, $imm",
1596    (MVE_VBICimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;
1597
1598def MVE_VMOV : MVEInstAlias<"vmov${vp}\t$Qd, $Qm",
1599    (MVE_VORR MQPR:$Qd, MQPR:$Qm, MQPR:$Qm, vpred_r:$vp)>;
1600
1601class MVE_VMOV_lane_direction {
1602  bit bit_20;
1603  dag oops;
1604  dag iops;
1605  string ops;
1606  string cstr;
1607}
1608def MVE_VMOV_from_lane : MVE_VMOV_lane_direction {
1609  let bit_20 = 0b1;
1610  let oops = (outs rGPR:$Rt);
1611  let iops = (ins MQPR:$Qd);
1612  let ops = "$Rt, $Qd$Idx";
1613  let cstr = "";
1614}
1615def MVE_VMOV_to_lane : MVE_VMOV_lane_direction {
1616  let bit_20 = 0b0;
1617  let oops = (outs MQPR:$Qd);
1618  let iops = (ins MQPR:$Qd_src, rGPR:$Rt);
1619  let ops = "$Qd$Idx, $Rt";
1620  let cstr = "$Qd = $Qd_src";
1621}
1622
1623class MVE_VMOV_lane<string suffix, bit U, dag indexop,
1624                    MVE_VMOV_lane_direction dir>
1625  : MVE_VMOV_lane_base<dir.oops, !con(dir.iops, indexop), NoItinerary,
1626                       "vmov", suffix, dir.ops, dir.cstr, []> {
1627  bits<4> Qd;
1628  bits<4> Rt;
1629
1630  let Inst{31-24} = 0b11101110;
1631  let Inst{23} = U;
1632  let Inst{20} = dir.bit_20;
1633  let Inst{19-17} = Qd{2-0};
1634  let Inst{15-12} = Rt{3-0};
1635  let Inst{11-8} = 0b1011;
1636  let Inst{7} = Qd{3};
1637  let Inst{4-0} = 0b10000;
1638
1639  let hasSideEffects = 0;
1640}
1641
1642class MVE_VMOV_lane_32<MVE_VMOV_lane_direction dir>
1643    : MVE_VMOV_lane<"32", 0b0, (ins MVEVectorIndex<4>:$Idx), dir> {
1644  bits<2> Idx;
1645  let Inst{22} = 0b0;
1646  let Inst{6-5} = 0b00;
1647  let Inst{16} = Idx{1};
1648  let Inst{21} = Idx{0};
1649
1650  let Predicates = [HasFPRegsV8_1M];
1651}
1652
1653class MVE_VMOV_lane_16<string suffix, bit U, MVE_VMOV_lane_direction dir>
1654  : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<8>:$Idx), dir> {
1655  bits<3> Idx;
1656  let Inst{22} = 0b0;
1657  let Inst{5} = 0b1;
1658  let Inst{16} = Idx{2};
1659  let Inst{21} = Idx{1};
1660  let Inst{6} = Idx{0};
1661}
1662
1663class MVE_VMOV_lane_8<string suffix, bit U, MVE_VMOV_lane_direction dir>
1664  : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<16>:$Idx), dir> {
1665  bits<4> Idx;
1666  let Inst{22} = 0b1;
1667  let Inst{16} = Idx{3};
1668  let Inst{21} = Idx{2};
1669  let Inst{6} = Idx{1};
1670  let Inst{5} = Idx{0};
1671}
1672
1673def MVE_VMOV_from_lane_32  : MVE_VMOV_lane_32<            MVE_VMOV_from_lane>;
1674def MVE_VMOV_to_lane_32    : MVE_VMOV_lane_32<            MVE_VMOV_to_lane>;
1675def MVE_VMOV_from_lane_s16 : MVE_VMOV_lane_16<"s16", 0b0, MVE_VMOV_from_lane>;
1676def MVE_VMOV_from_lane_u16 : MVE_VMOV_lane_16<"u16", 0b1, MVE_VMOV_from_lane>;
1677def MVE_VMOV_to_lane_16    : MVE_VMOV_lane_16< "16", 0b0, MVE_VMOV_to_lane>;
1678def MVE_VMOV_from_lane_s8  : MVE_VMOV_lane_8 < "s8", 0b0, MVE_VMOV_from_lane>;
1679def MVE_VMOV_from_lane_u8  : MVE_VMOV_lane_8 < "u8", 0b1, MVE_VMOV_from_lane>;
1680def MVE_VMOV_to_lane_8     : MVE_VMOV_lane_8 <  "8", 0b0, MVE_VMOV_to_lane>;
1681
1682let Predicates = [HasMVEInt] in {
1683  def : Pat<(extractelt (v2f64 MQPR:$src), imm:$lane),
1684            (f64 (EXTRACT_SUBREG MQPR:$src, (DSubReg_f64_reg imm:$lane)))>;
1685  def : Pat<(insertelt (v2f64 MQPR:$src1), DPR:$src2, imm:$lane),
1686            (INSERT_SUBREG (v2f64 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), DPR:$src2, (DSubReg_f64_reg imm:$lane))>;
1687
1688  def : Pat<(extractelt (v4i32 MQPR:$src), imm:$lane),
1689            (COPY_TO_REGCLASS
1690              (i32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), rGPR)>;
1691  def : Pat<(insertelt (v4i32 MQPR:$src1), rGPR:$src2, imm:$lane),
1692            (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$src2, imm:$lane)>;
1693
1694  def : Pat<(vector_insert (v16i8 MQPR:$src1), rGPR:$src2, imm:$lane),
1695            (MVE_VMOV_to_lane_8  MQPR:$src1, rGPR:$src2, imm:$lane)>;
1696  def : Pat<(vector_insert (v8i16 MQPR:$src1), rGPR:$src2, imm:$lane),
1697            (MVE_VMOV_to_lane_16 MQPR:$src1, rGPR:$src2, imm:$lane)>;
1698
1699  def : Pat<(ARMvgetlanes (v16i8 MQPR:$src), imm:$lane),
1700            (MVE_VMOV_from_lane_s8 MQPR:$src, imm:$lane)>;
1701  def : Pat<(ARMvgetlanes (v8i16 MQPR:$src), imm:$lane),
1702            (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
1703  def : Pat<(ARMvgetlanes (v8f16 MQPR:$src), imm:$lane),
1704            (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
1705  def : Pat<(ARMvgetlaneu (v16i8 MQPR:$src), imm:$lane),
1706            (MVE_VMOV_from_lane_u8 MQPR:$src, imm:$lane)>;
1707  def : Pat<(ARMvgetlaneu (v8i16 MQPR:$src), imm:$lane),
1708            (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
1709  def : Pat<(ARMvgetlaneu (v8f16 MQPR:$src), imm:$lane),
1710            (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
1711
1712  def : Pat<(v16i8 (scalar_to_vector GPR:$src)),
1713            (MVE_VMOV_to_lane_8  (v16i8 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1714  def : Pat<(v8i16 (scalar_to_vector GPR:$src)),
1715            (MVE_VMOV_to_lane_16 (v8i16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1716  def : Pat<(v4i32 (scalar_to_vector GPR:$src)),
1717            (MVE_VMOV_to_lane_32 (v4i32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1718
1719  // Floating point patterns, still enabled under HasMVEInt
1720  def : Pat<(extractelt (v4f32 MQPR:$src), imm:$lane),
1721            (COPY_TO_REGCLASS (f32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), SPR)>;
1722  def : Pat<(insertelt (v4f32 MQPR:$src1), (f32 SPR:$src2), imm:$lane),
1723            (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), SPR:$src2, (SSubReg_f32_reg imm:$lane))>;
1724
1725  def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm:$lane),
1726            (MVE_VMOV_to_lane_16 MQPR:$src1, (COPY_TO_REGCLASS (f16 HPR:$src2), rGPR), imm:$lane)>;
1727  def : Pat<(extractelt (v8f16 MQPR:$src), imm_even:$lane),
1728            (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_even:$lane))>;
1729  def : Pat<(extractelt (v8f16 MQPR:$src), imm_odd:$lane),
1730            (COPY_TO_REGCLASS
1731              (VMOVH (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_odd:$lane))),
1732              HPR)>;
1733
1734  def : Pat<(v4f32 (scalar_to_vector SPR:$src)),
1735            (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), SPR:$src, ssub_0)>;
1736  def : Pat<(v4f32 (scalar_to_vector GPR:$src)),
1737            (MVE_VMOV_to_lane_32 (v4f32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1738  def : Pat<(v8f16 (scalar_to_vector (f16 HPR:$src))),
1739            (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), (f16 HPR:$src), ssub_0)>;
1740  def : Pat<(v8f16 (scalar_to_vector GPR:$src)),
1741            (MVE_VMOV_to_lane_16 (v8f16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1742}
1743
1744// end of mve_bit instructions
1745
1746// start of MVE Integer instructions
1747
1748class MVE_int<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
1749  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
1750          iname, suffix, "$Qd, $Qn, $Qm", vpred_r, "", pattern> {
1751  bits<4> Qd;
1752  bits<4> Qn;
1753  bits<4> Qm;
1754
1755  let Inst{22} = Qd{3};
1756  let Inst{21-20} = size;
1757  let Inst{19-17} = Qn{2-0};
1758  let Inst{15-13} = Qd{2-0};
1759  let Inst{7} = Qn{3};
1760  let Inst{6} = 0b1;
1761  let Inst{5} = Qm{3};
1762  let Inst{3-1} = Qm{2-0};
1763}
1764
1765class MVE_VMULt1<string iname, string suffix, bits<2> size,
1766                   list<dag> pattern=[]>
1767  : MVE_int<iname, suffix, size, pattern> {
1768
1769  let Inst{28} = 0b0;
1770  let Inst{25-23} = 0b110;
1771  let Inst{16} = 0b0;
1772  let Inst{12-8} = 0b01001;
1773  let Inst{4} = 0b1;
1774  let Inst{0} = 0b0;
1775  let validForTailPredication = 1;
1776}
1777
1778multiclass MVE_VMUL_m<string iname, MVEVectorVTInfo VTI,
1779                      SDNode unpred_op, Intrinsic pred_int> {
1780  def "" : MVE_VMULt1<iname, VTI.Suffix, VTI.Size>;
1781  defvar Inst = !cast<Instruction>(NAME);
1782
1783  let Predicates = [HasMVEInt] in {
1784    // Unpredicated multiply
1785    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1786              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1787
1788    // Predicated multiply
1789    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1790                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
1791              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1792                             ARMVCCThen, (VTI.Pred VCCR:$mask),
1793                             (VTI.Vec MQPR:$inactive)))>;
1794  }
1795}
1796
1797multiclass MVE_VMUL<MVEVectorVTInfo VTI>
1798  : MVE_VMUL_m<"vmul", VTI, mul, int_arm_mve_mul_predicated>;
1799
1800defm MVE_VMULi8  : MVE_VMUL<MVE_v16i8>;
1801defm MVE_VMULi16 : MVE_VMUL<MVE_v8i16>;
1802defm MVE_VMULi32 : MVE_VMUL<MVE_v4i32>;
1803
1804class MVE_VQxDMULH_Base<string iname, string suffix, bits<2> size, bit rounding,
1805                  list<dag> pattern=[]>
1806  : MVE_int<iname, suffix, size, pattern> {
1807
1808  let Inst{28} = rounding;
1809  let Inst{25-23} = 0b110;
1810  let Inst{16} = 0b0;
1811  let Inst{12-8} = 0b01011;
1812  let Inst{4} = 0b0;
1813  let Inst{0} = 0b0;
1814}
1815
1816multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,
1817                      SDNode unpred_op, Intrinsic pred_int,
1818                      bit rounding> {
1819  def "" : MVE_VQxDMULH_Base<iname, VTI.Suffix, VTI.Size, rounding>;
1820  defvar Inst = !cast<Instruction>(NAME);
1821
1822  let Predicates = [HasMVEInt] in {
1823    // Unpredicated multiply
1824    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1825              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1826
1827    // Predicated multiply
1828    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1829                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
1830              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1831                             ARMVCCThen, (VTI.Pred VCCR:$mask),
1832                             (VTI.Vec MQPR:$inactive)))>;
1833  }
1834}
1835
1836multiclass MVE_VQxDMULH<string iname, MVEVectorVTInfo VTI, bit rounding>
1837  : MVE_VQxDMULH_m<iname, VTI, !if(rounding, int_arm_mve_vqrdmulh,
1838                                             int_arm_mve_vqdmulh),
1839                               !if(rounding, int_arm_mve_qrdmulh_predicated,
1840                                             int_arm_mve_qdmulh_predicated),
1841                   rounding>;
1842
1843defm MVE_VQDMULHi8  : MVE_VQxDMULH<"vqdmulh", MVE_v16s8, 0b0>;
1844defm MVE_VQDMULHi16 : MVE_VQxDMULH<"vqdmulh", MVE_v8s16, 0b0>;
1845defm MVE_VQDMULHi32 : MVE_VQxDMULH<"vqdmulh", MVE_v4s32, 0b0>;
1846
1847defm MVE_VQRDMULHi8  : MVE_VQxDMULH<"vqrdmulh", MVE_v16s8, 0b1>;
1848defm MVE_VQRDMULHi16 : MVE_VQxDMULH<"vqrdmulh", MVE_v8s16, 0b1>;
1849defm MVE_VQRDMULHi32 : MVE_VQxDMULH<"vqrdmulh", MVE_v4s32, 0b1>;
1850
1851class MVE_VADDSUB<string iname, string suffix, bits<2> size, bit subtract,
1852                    list<dag> pattern=[]>
1853  : MVE_int<iname, suffix, size, pattern> {
1854
1855  let Inst{28} = subtract;
1856  let Inst{25-23} = 0b110;
1857  let Inst{16} = 0b0;
1858  let Inst{12-8} = 0b01000;
1859  let Inst{4} = 0b0;
1860  let Inst{0} = 0b0;
1861  let validForTailPredication = 1;
1862}
1863
1864multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,
1865                         SDNode unpred_op, Intrinsic pred_int> {
1866  def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;
1867  defvar Inst = !cast<Instruction>(NAME);
1868
1869  let Predicates = [HasMVEInt] in {
1870    // Unpredicated add/subtract
1871    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1872              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1873
1874    // Predicated add/subtract
1875    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1876                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
1877              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1878                             ARMVCCThen, (VTI.Pred VCCR:$mask),
1879                             (VTI.Vec MQPR:$inactive)))>;
1880  }
1881}
1882
1883multiclass MVE_VADD<MVEVectorVTInfo VTI>
1884  : MVE_VADDSUB_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
1885multiclass MVE_VSUB<MVEVectorVTInfo VTI>
1886  : MVE_VADDSUB_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
1887
1888defm MVE_VADDi8  : MVE_VADD<MVE_v16i8>;
1889defm MVE_VADDi16 : MVE_VADD<MVE_v8i16>;
1890defm MVE_VADDi32 : MVE_VADD<MVE_v4i32>;
1891
1892defm MVE_VSUBi8  : MVE_VSUB<MVE_v16i8>;
1893defm MVE_VSUBi16 : MVE_VSUB<MVE_v8i16>;
1894defm MVE_VSUBi32 : MVE_VSUB<MVE_v4i32>;
1895
1896class MVE_VQADDSUB<string iname, string suffix, bit U, bit subtract,
1897                   bits<2> size>
1898  : MVE_int<iname, suffix, size, []> {
1899
1900  let Inst{28} = U;
1901  let Inst{25-23} = 0b110;
1902  let Inst{16} = 0b0;
1903  let Inst{12-10} = 0b000;
1904  let Inst{9} = subtract;
1905  let Inst{8} = 0b0;
1906  let Inst{4} = 0b1;
1907  let Inst{0} = 0b0;
1908  let validForTailPredication = 1;
1909}
1910
1911class MVE_VQADD_<string suffix, bit U, bits<2> size>
1912  : MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size>;
1913class MVE_VQSUB_<string suffix, bit U, bits<2> size>
1914  : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size>;
1915
1916multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,
1917                      SDNode unpred_op, Intrinsic pred_int> {
1918  def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
1919  defvar Inst = !cast<Instruction>(NAME);
1920
1921  let Predicates = [HasMVEInt] in {
1922    // Unpredicated saturating add
1923    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1924              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1925
1926    // Predicated saturating add
1927    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1928                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
1929                            (VTI.Vec MQPR:$inactive))),
1930              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1931                             ARMVCCThen, (VTI.Pred VCCR:$mask),
1932                             (VTI.Vec MQPR:$inactive)))>;
1933  }
1934}
1935
1936multiclass MVE_VQADD<MVEVectorVTInfo VTI, SDNode unpred_op>
1937  : MVE_VQADD_m<VTI, unpred_op, int_arm_mve_qadd_predicated>;
1938
1939defm MVE_VQADDs8  : MVE_VQADD<MVE_v16s8, saddsat>;
1940defm MVE_VQADDs16 : MVE_VQADD<MVE_v8s16, saddsat>;
1941defm MVE_VQADDs32 : MVE_VQADD<MVE_v4s32, saddsat>;
1942defm MVE_VQADDu8  : MVE_VQADD<MVE_v16u8, uaddsat>;
1943defm MVE_VQADDu16 : MVE_VQADD<MVE_v8u16, uaddsat>;
1944defm MVE_VQADDu32 : MVE_VQADD<MVE_v4u32, uaddsat>;
1945
1946multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,
1947                      SDNode unpred_op, Intrinsic pred_int> {
1948  def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
1949  defvar Inst = !cast<Instruction>(NAME);
1950
1951  let Predicates = [HasMVEInt] in {
1952    // Unpredicated saturating subtract
1953    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1954              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1955
1956    // Predicated saturating subtract
1957    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1958                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
1959                            (VTI.Vec MQPR:$inactive))),
1960              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1961                             ARMVCCThen, (VTI.Pred VCCR:$mask),
1962                             (VTI.Vec MQPR:$inactive)))>;
1963  }
1964}
1965
1966multiclass MVE_VQSUB<MVEVectorVTInfo VTI, SDNode unpred_op>
1967  : MVE_VQSUB_m<VTI, unpred_op, int_arm_mve_qsub_predicated>;
1968
1969defm MVE_VQSUBs8  : MVE_VQSUB<MVE_v16s8, ssubsat>;
1970defm MVE_VQSUBs16 : MVE_VQSUB<MVE_v8s16, ssubsat>;
1971defm MVE_VQSUBs32 : MVE_VQSUB<MVE_v4s32, ssubsat>;
1972defm MVE_VQSUBu8  : MVE_VQSUB<MVE_v16u8, usubsat>;
1973defm MVE_VQSUBu16 : MVE_VQSUB<MVE_v8u16, usubsat>;
1974defm MVE_VQSUBu32 : MVE_VQSUB<MVE_v4u32, usubsat>;
1975
1976class MVE_VABD_int<string suffix, bit U, bits<2> size,
1977                     list<dag> pattern=[]>
1978  : MVE_int<"vabd", suffix, size, pattern> {
1979
1980  let Inst{28} = U;
1981  let Inst{25-23} = 0b110;
1982  let Inst{16} = 0b0;
1983  let Inst{12-8} = 0b00111;
1984  let Inst{4} = 0b0;
1985  let Inst{0} = 0b0;
1986  let validForTailPredication = 1;
1987}
1988
1989multiclass MVE_VABD_m<MVEVectorVTInfo VTI,
1990                      Intrinsic unpred_int, Intrinsic pred_int> {
1991  def "" : MVE_VABD_int<VTI.Suffix, VTI.Unsigned, VTI.Size>;
1992  defvar Inst = !cast<Instruction>(NAME);
1993
1994  let Predicates = [HasMVEInt] in {
1995    // Unpredicated absolute difference
1996    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
1997                            (i32 VTI.Unsigned))),
1998              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1999
2000    // Predicated absolute difference
2001    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2002                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2003                            (VTI.Vec MQPR:$inactive))),
2004              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2005                             ARMVCCThen, (VTI.Pred VCCR:$mask),
2006                             (VTI.Vec MQPR:$inactive)))>;
2007  }
2008}
2009
2010multiclass MVE_VABD<MVEVectorVTInfo VTI>
2011  : MVE_VABD_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
2012
2013defm MVE_VABDs8  : MVE_VABD<MVE_v16s8>;
2014defm MVE_VABDs16 : MVE_VABD<MVE_v8s16>;
2015defm MVE_VABDs32 : MVE_VABD<MVE_v4s32>;
2016defm MVE_VABDu8  : MVE_VABD<MVE_v16u8>;
2017defm MVE_VABDu16 : MVE_VABD<MVE_v8u16>;
2018defm MVE_VABDu32 : MVE_VABD<MVE_v4u32>;
2019
2020class MVE_VRHADD_Base<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
2021  : MVE_int<"vrhadd", suffix, size, pattern> {
2022
2023  let Inst{28} = U;
2024  let Inst{25-23} = 0b110;
2025  let Inst{16} = 0b0;
2026  let Inst{12-8} = 0b00001;
2027  let Inst{4} = 0b0;
2028  let Inst{0} = 0b0;
2029  let validForTailPredication = 1;
2030}
2031
2032def addnuw : PatFrag<(ops node:$lhs, node:$rhs),
2033                     (add node:$lhs, node:$rhs), [{
2034  return N->getFlags().hasNoUnsignedWrap();
2035}]>;
2036
2037def addnsw : PatFrag<(ops node:$lhs, node:$rhs),
2038                     (add node:$lhs, node:$rhs), [{
2039  return N->getFlags().hasNoSignedWrap();
2040}]>;
2041
2042def subnuw : PatFrag<(ops node:$lhs, node:$rhs),
2043                     (sub node:$lhs, node:$rhs), [{
2044  return N->getFlags().hasNoUnsignedWrap();
2045}]>;
2046
2047def subnsw : PatFrag<(ops node:$lhs, node:$rhs),
2048                     (sub node:$lhs, node:$rhs), [{
2049  return N->getFlags().hasNoSignedWrap();
2050}]>;
2051
2052multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI,
2053                      SDNode unpred_op, Intrinsic pred_int> {
2054  def "" : MVE_VRHADD_Base<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2055  defvar Inst = !cast<Instruction>(NAME);
2056
2057  let Predicates = [HasMVEInt] in {
2058    // Unpredicated rounding add-with-divide-by-two
2059    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2060                            (i32 VTI.Unsigned))),
2061              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2062
2063    // Predicated add-with-divide-by-two
2064    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2065                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2066                            (VTI.Vec MQPR:$inactive))),
2067              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2068                             ARMVCCThen, (VTI.Pred VCCR:$mask),
2069                             (VTI.Vec MQPR:$inactive)))>;
2070  }
2071}
2072
2073multiclass MVE_VRHADD<MVEVectorVTInfo VTI>
2074  : MVE_VRHADD_m<VTI, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;
2075
2076defm MVE_VRHADDs8  : MVE_VRHADD<MVE_v16s8>;
2077defm MVE_VRHADDs16 : MVE_VRHADD<MVE_v8s16>;
2078defm MVE_VRHADDs32 : MVE_VRHADD<MVE_v4s32>;
2079defm MVE_VRHADDu8  : MVE_VRHADD<MVE_v16u8>;
2080defm MVE_VRHADDu16 : MVE_VRHADD<MVE_v8u16>;
2081defm MVE_VRHADDu32 : MVE_VRHADD<MVE_v4u32>;
2082
2083// Rounding Halving Add perform the arithemtic operation with an extra bit of
2084// precision, before performing the shift, to void clipping errors. We're not
2085// modelling that here with these patterns, but we're using no wrap forms of
2086// add to ensure that the extra bit of information is not needed for the
2087// arithmetic or the rounding.
2088def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
2089                                      (v16i8 (ARMvmovImm (i32 3585)))),
2090                                   (i32 1))),
2091              (MVE_VRHADDs8 MQPR:$Qm, MQPR:$Qn)>;
2092def : Pat<(v8i16 (ARMvshrsImm (addnsw (addnsw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
2093                                      (v8i16 (ARMvmovImm (i32 2049)))),
2094                                   (i32 1))),
2095              (MVE_VRHADDs16 MQPR:$Qm, MQPR:$Qn)>;
2096def : Pat<(v4i32 (ARMvshrsImm (addnsw (addnsw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
2097                                      (v4i32 (ARMvmovImm (i32 1)))),
2098                                   (i32 1))),
2099              (MVE_VRHADDs32 MQPR:$Qm, MQPR:$Qn)>;
2100def : Pat<(v16i8 (ARMvshruImm (addnuw (addnuw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
2101                                      (v16i8 (ARMvmovImm (i32 3585)))),
2102                                   (i32 1))),
2103              (MVE_VRHADDu8 MQPR:$Qm, MQPR:$Qn)>;
2104def : Pat<(v8i16 (ARMvshruImm (addnuw (addnuw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
2105                                      (v8i16 (ARMvmovImm (i32 2049)))),
2106                                   (i32 1))),
2107              (MVE_VRHADDu16 MQPR:$Qm, MQPR:$Qn)>;
2108def : Pat<(v4i32 (ARMvshruImm (addnuw (addnuw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
2109                                      (v4i32 (ARMvmovImm (i32 1)))),
2110                                   (i32 1))),
2111             (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>;
2112
2113
2114class MVE_VHADDSUB<string iname, string suffix, bit U, bit subtract,
2115                   bits<2> size, list<dag> pattern=[]>
2116  : MVE_int<iname, suffix, size, pattern> {
2117
2118  let Inst{28} = U;
2119  let Inst{25-23} = 0b110;
2120  let Inst{16} = 0b0;
2121  let Inst{12-10} = 0b000;
2122  let Inst{9} = subtract;
2123  let Inst{8} = 0b0;
2124  let Inst{4} = 0b0;
2125  let Inst{0} = 0b0;
2126  let validForTailPredication = 1;
2127}
2128
2129class MVE_VHADD_<string suffix, bit U, bits<2> size,
2130              list<dag> pattern=[]>
2131  : MVE_VHADDSUB<"vhadd", suffix, U, 0b0, size, pattern>;
2132class MVE_VHSUB_<string suffix, bit U, bits<2> size,
2133              list<dag> pattern=[]>
2134  : MVE_VHADDSUB<"vhsub", suffix, U, 0b1, size, pattern>;
2135
2136multiclass MVE_VHADD_m<MVEVectorVTInfo VTI,
2137                      SDNode unpred_op, Intrinsic pred_int, PatFrag add_op,
2138                      SDNode shift_op> {
2139  def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2140  defvar Inst = !cast<Instruction>(NAME);
2141
2142  let Predicates = [HasMVEInt] in {
2143    // Unpredicated add-and-divide-by-two
2144    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),
2145              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2146
2147    def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2148              (Inst MQPR:$Qm, MQPR:$Qn)>;
2149
2150    // Predicated add-and-divide-by-two
2151    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned),
2152                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
2153              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2154                             ARMVCCThen, (VTI.Pred VCCR:$mask),
2155                             (VTI.Vec MQPR:$inactive)))>;
2156  }
2157}
2158
2159multiclass MVE_VHADD<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op>
2160  : MVE_VHADD_m<VTI, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, add_op,
2161                shift_op>;
2162
2163// Halving add/sub perform the arithemtic operation with an extra bit of
2164// precision, before performing the shift, to void clipping errors. We're not
2165// modelling that here with these patterns, but we're using no wrap forms of
2166// add/sub to ensure that the extra bit of information is not needed.
2167defm MVE_VHADDs8  : MVE_VHADD<MVE_v16s8, addnsw, ARMvshrsImm>;
2168defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16, addnsw, ARMvshrsImm>;
2169defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32, addnsw, ARMvshrsImm>;
2170defm MVE_VHADDu8  : MVE_VHADD<MVE_v16u8, addnuw, ARMvshruImm>;
2171defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16, addnuw, ARMvshruImm>;
2172defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32, addnuw, ARMvshruImm>;
2173
2174multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,
2175                      SDNode unpred_op, Intrinsic pred_int, PatFrag sub_op,
2176                      SDNode shift_op> {
2177  def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2178  defvar Inst = !cast<Instruction>(NAME);
2179
2180  let Predicates = [HasMVEInt] in {
2181    // Unpredicated subtract-and-divide-by-two
2182    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2183                            (i32 VTI.Unsigned))),
2184              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2185
2186    def : Pat<(VTI.Vec (shift_op (sub_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2187              (Inst MQPR:$Qm, MQPR:$Qn)>;
2188
2189
2190    // Predicated subtract-and-divide-by-two
2191    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2192                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2193                            (VTI.Vec MQPR:$inactive))),
2194              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2195                             ARMVCCThen, (VTI.Pred VCCR:$mask),
2196                             (VTI.Vec MQPR:$inactive)))>;
2197  }
2198}
2199
2200multiclass MVE_VHSUB<MVEVectorVTInfo VTI, PatFrag sub_op, SDNode shift_op>
2201  : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, sub_op,
2202                shift_op>;
2203
2204defm MVE_VHSUBs8  : MVE_VHSUB<MVE_v16s8, subnsw, ARMvshrsImm>;
2205defm MVE_VHSUBs16 : MVE_VHSUB<MVE_v8s16, subnsw, ARMvshrsImm>;
2206defm MVE_VHSUBs32 : MVE_VHSUB<MVE_v4s32, subnsw, ARMvshrsImm>;
2207defm MVE_VHSUBu8  : MVE_VHSUB<MVE_v16u8, subnuw, ARMvshruImm>;
2208defm MVE_VHSUBu16 : MVE_VHSUB<MVE_v8u16, subnuw, ARMvshruImm>;
2209defm MVE_VHSUBu32 : MVE_VHSUB<MVE_v4u32, subnuw, ARMvshruImm>;
2210
2211class MVE_VDUP<string suffix, bit B, bit E, list<dag> pattern=[]>
2212  : MVE_p<(outs MQPR:$Qd), (ins rGPR:$Rt), NoItinerary,
2213          "vdup", suffix, "$Qd, $Rt", vpred_r, "", pattern> {
2214  bits<4> Qd;
2215  bits<4> Rt;
2216
2217  let Inst{28} = 0b0;
2218  let Inst{25-23} = 0b101;
2219  let Inst{22} = B;
2220  let Inst{21-20} = 0b10;
2221  let Inst{19-17} = Qd{2-0};
2222  let Inst{16} = 0b0;
2223  let Inst{15-12} = Rt;
2224  let Inst{11-8} = 0b1011;
2225  let Inst{7} = Qd{3};
2226  let Inst{6} = 0b0;
2227  let Inst{5} = E;
2228  let Inst{4-0} = 0b10000;
2229  let validForTailPredication = 1;
2230}
2231
2232def MVE_VDUP32 : MVE_VDUP<"32", 0b0, 0b0>;
2233def MVE_VDUP16 : MVE_VDUP<"16", 0b0, 0b1>;
2234def MVE_VDUP8  : MVE_VDUP<"8",  0b1, 0b0>;
2235
2236let Predicates = [HasMVEInt] in {
2237  def : Pat<(v16i8 (ARMvdup (i32 rGPR:$elem))),
2238            (MVE_VDUP8  rGPR:$elem)>;
2239  def : Pat<(v8i16 (ARMvdup (i32 rGPR:$elem))),
2240            (MVE_VDUP16 rGPR:$elem)>;
2241  def : Pat<(v4i32 (ARMvdup (i32 rGPR:$elem))),
2242            (MVE_VDUP32 rGPR:$elem)>;
2243
2244  def : Pat<(v8f16 (ARMvdup (i32 rGPR:$elem))),
2245            (MVE_VDUP16 rGPR:$elem)>;
2246  def : Pat<(v4f32 (ARMvdup (i32 rGPR:$elem))),
2247            (MVE_VDUP32 rGPR:$elem)>;
2248
2249  // Match a vselect with an ARMvdup as a predicated MVE_VDUP
2250  def : Pat<(v16i8 (vselect (v16i1 VCCR:$pred),
2251                            (v16i8 (ARMvdup (i32 rGPR:$elem))),
2252                            (v16i8 MQPR:$inactive))),
2253            (MVE_VDUP8  rGPR:$elem, ARMVCCThen, (v16i1 VCCR:$pred),
2254                        (v16i8 MQPR:$inactive))>;
2255  def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred),
2256                            (v8i16 (ARMvdup (i32 rGPR:$elem))),
2257                            (v8i16 MQPR:$inactive))),
2258            (MVE_VDUP16 rGPR:$elem, ARMVCCThen, (v8i1 VCCR:$pred),
2259                            (v8i16 MQPR:$inactive))>;
2260  def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred),
2261                            (v4i32 (ARMvdup (i32 rGPR:$elem))),
2262                            (v4i32 MQPR:$inactive))),
2263            (MVE_VDUP32 rGPR:$elem, ARMVCCThen, (v4i1 VCCR:$pred),
2264                            (v4i32 MQPR:$inactive))>;
2265  def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred),
2266                            (v4f32 (ARMvdup (i32 rGPR:$elem))),
2267                            (v4f32 MQPR:$inactive))),
2268            (MVE_VDUP32 rGPR:$elem, ARMVCCThen, (v4i1 VCCR:$pred),
2269                            (v4f32 MQPR:$inactive))>;
2270  def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred),
2271                            (v8f16 (ARMvdup (i32 rGPR:$elem))),
2272                            (v8f16 MQPR:$inactive))),
2273            (MVE_VDUP16 rGPR:$elem, ARMVCCThen, (v8i1 VCCR:$pred),
2274                            (v8f16 MQPR:$inactive))>;
2275}
2276
2277
2278class MVEIntSingleSrc<string iname, string suffix, bits<2> size,
2279                         list<dag> pattern=[]>
2280  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm), NoItinerary,
2281          iname, suffix, "$Qd, $Qm", vpred_r, "", pattern> {
2282  bits<4> Qd;
2283  bits<4> Qm;
2284
2285  let Inst{22} = Qd{3};
2286  let Inst{19-18} = size{1-0};
2287  let Inst{15-13} = Qd{2-0};
2288  let Inst{5} = Qm{3};
2289  let Inst{3-1} = Qm{2-0};
2290}
2291
2292class MVE_VCLSCLZ<string iname, string suffix, bits<2> size,
2293                   bit count_zeroes, list<dag> pattern=[]>
2294  : MVEIntSingleSrc<iname, suffix, size, pattern> {
2295
2296  let Inst{28} = 0b1;
2297  let Inst{25-23} = 0b111;
2298  let Inst{21-20} = 0b11;
2299  let Inst{17-16} = 0b00;
2300  let Inst{12-8} = 0b00100;
2301  let Inst{7} = count_zeroes;
2302  let Inst{6} = 0b1;
2303  let Inst{4} = 0b0;
2304  let Inst{0} = 0b0;
2305  let validForTailPredication = 1;
2306}
2307
2308multiclass MVE_VCLSCLZ_p<string opname, bit opcode, MVEVectorVTInfo VTI,
2309                         SDNode unpred_op> {
2310  def "": MVE_VCLSCLZ<"v"#opname, VTI.Suffix, VTI.Size, opcode>;
2311
2312  defvar Inst     = !cast<Instruction>(NAME);
2313  defvar pred_int = !cast<Intrinsic>("int_arm_mve_"#opname#"_predicated");
2314
2315  let Predicates = [HasMVEInt] in {
2316    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
2317              (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
2318    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
2319                                 (VTI.Vec MQPR:$inactive))),
2320              (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
2321                             (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
2322  }
2323}
2324
2325defm MVE_VCLSs8  : MVE_VCLSCLZ_p<"cls", 0, MVE_v16s8, int_arm_mve_vcls>;
2326defm MVE_VCLSs16 : MVE_VCLSCLZ_p<"cls", 0, MVE_v8s16, int_arm_mve_vcls>;
2327defm MVE_VCLSs32 : MVE_VCLSCLZ_p<"cls", 0, MVE_v4s32, int_arm_mve_vcls>;
2328
2329defm MVE_VCLZs8  : MVE_VCLSCLZ_p<"clz", 1, MVE_v16i8, ctlz>;
2330defm MVE_VCLZs16 : MVE_VCLSCLZ_p<"clz", 1, MVE_v8i16, ctlz>;
2331defm MVE_VCLZs32 : MVE_VCLSCLZ_p<"clz", 1, MVE_v4i32, ctlz>;
2332
2333class MVE_VABSNEG_int<string iname, string suffix, bits<2> size, bit negate,
2334                      bit saturate, list<dag> pattern=[]>
2335  : MVEIntSingleSrc<iname, suffix, size, pattern> {
2336
2337  let Inst{28} = 0b1;
2338  let Inst{25-23} = 0b111;
2339  let Inst{21-20} = 0b11;
2340  let Inst{17} = 0b0;
2341  let Inst{16} = !eq(saturate, 0);
2342  let Inst{12-11} = 0b00;
2343  let Inst{10} = saturate;
2344  let Inst{9-8} = 0b11;
2345  let Inst{7} = negate;
2346  let Inst{6} = 0b1;
2347  let Inst{4} = 0b0;
2348  let Inst{0} = 0b0;
2349  let validForTailPredication = 1;
2350}
2351
2352multiclass MVE_VABSNEG_int_m<string iname, bit negate, bit saturate,
2353                             SDNode unpred_op, Intrinsic pred_int,
2354                             MVEVectorVTInfo VTI> {
2355  def "" : MVE_VABSNEG_int<iname, VTI.Suffix, VTI.Size, negate, saturate>;
2356  defvar Inst = !cast<Instruction>(NAME);
2357
2358  let Predicates = [HasMVEInt] in {
2359    // VQABS and VQNEG have more difficult isel patterns defined elsewhere
2360    if !eq(saturate, 0) then {
2361      def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))), (VTI.Vec (Inst $v))>;
2362    }
2363
2364    def : Pat<(VTI.Vec (pred_int  (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
2365                                  (VTI.Vec MQPR:$inactive))),
2366              (VTI.Vec (Inst $v, ARMVCCThen, $mask, $inactive))>;
2367  }
2368}
2369
2370foreach VTI = [ MVE_v16s8, MVE_v8s16, MVE_v4s32 ] in {
2371  defm "MVE_VABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2372     "vabs",  0, 0, abs,   int_arm_mve_abs_predicated,  VTI>;
2373  defm "MVE_VQABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2374     "vqabs", 0, 1, ?,     int_arm_mve_qabs_predicated, VTI>;
2375  defm "MVE_VNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2376     "vneg",  1, 0, vnegq, int_arm_mve_neg_predicated,  VTI>;
2377  defm "MVE_VQNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2378     "vqneg", 1, 1, ?,     int_arm_mve_qneg_predicated, VTI>;
2379}
2380
2381// int_min/int_max: vector containing INT_MIN/INT_MAX VTI.Size times
2382// zero_vec: v4i32-initialized zero vector, potentially wrapped in a bitconvert
2383multiclass vqabsneg_pattern<MVEVectorVTInfo VTI, dag int_min, dag int_max,
2384                         dag zero_vec,  MVE_VABSNEG_int vqabs_instruction,
2385                         MVE_VABSNEG_int vqneg_instruction> {
2386  let Predicates = [HasMVEInt] in {
2387    // The below tree can be replaced by a vqabs instruction, as it represents
2388    // the following vectorized expression (r being the value in $reg):
2389    // r > 0 ? r : (r == INT_MIN ? INT_MAX : -r)
2390    def : Pat<(VTI.Vec (vselect
2391                      (VTI.Pred (ARMvcmpz (VTI.Vec MQPR:$reg), ARMCCgt)),
2392                      (VTI.Vec MQPR:$reg),
2393                      (VTI.Vec (vselect
2394                                (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2395                                int_max,
2396                                (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))))),
2397            (VTI.Vec (vqabs_instruction (VTI.Vec MQPR:$reg)))>;
2398    // Similarly, this tree represents vqneg, i.e. the following vectorized expression:
2399    // r == INT_MIN ? INT_MAX : -r
2400    def : Pat<(VTI.Vec (vselect
2401                        (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2402                        int_max,
2403                        (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))),
2404               (VTI.Vec (vqneg_instruction (VTI.Vec MQPR:$reg)))>;
2405  }
2406}
2407
2408defm MVE_VQABSNEG_Ps8  : vqabsneg_pattern<MVE_v16i8,
2409                                    (v16i8 (ARMvmovImm (i32 3712))),
2410                                    (v16i8 (ARMvmovImm (i32 3711))),
2411                                    (bitconvert (v4i32 (ARMvmovImm (i32 0)))),
2412                                    MVE_VQABSs8, MVE_VQNEGs8>;
2413defm MVE_VQABSNEG_Ps16 : vqabsneg_pattern<MVE_v8i16,
2414                                    (v8i16 (ARMvmovImm (i32 2688))),
2415                                    (v8i16 (ARMvmvnImm (i32 2688))),
2416                                    (bitconvert (v4i32 (ARMvmovImm (i32 0)))),
2417                                    MVE_VQABSs16, MVE_VQNEGs16>;
2418defm MVE_VQABSNEG_Ps32 : vqabsneg_pattern<MVE_v4i32,
2419                                    (v4i32 (ARMvmovImm (i32 1664))),
2420                                    (v4i32 (ARMvmvnImm (i32 1664))),
2421                                    (ARMvmovImm (i32 0)),
2422                                    MVE_VQABSs32, MVE_VQNEGs32>;
2423
2424class MVE_mod_imm<string iname, string suffix, bits<4> cmode, bit op,
2425                  dag iops, list<dag> pattern=[]>
2426  : MVE_p<(outs MQPR:$Qd), iops, NoItinerary, iname, suffix, "$Qd, $imm",
2427          vpred_r, "", pattern> {
2428  bits<13> imm;
2429  bits<4> Qd;
2430
2431  let Inst{28} = imm{7};
2432  let Inst{25-23} = 0b111;
2433  let Inst{22} = Qd{3};
2434  let Inst{21-19} = 0b000;
2435  let Inst{18-16} = imm{6-4};
2436  let Inst{15-13} = Qd{2-0};
2437  let Inst{12} = 0b0;
2438  let Inst{11-8} = cmode{3-0};
2439  let Inst{7-6} = 0b01;
2440  let Inst{5} = op;
2441  let Inst{4} = 0b1;
2442  let Inst{3-0} = imm{3-0};
2443
2444  let DecoderMethod = "DecodeMVEModImmInstruction";
2445  let validForTailPredication = 1;
2446}
2447
2448let isReMaterializable = 1 in {
2449let isAsCheapAsAMove = 1 in {
2450def MVE_VMOVimmi8  : MVE_mod_imm<"vmov", "i8",  {1,1,1,0}, 0b0, (ins nImmSplatI8:$imm)>;
2451def MVE_VMOVimmi16 : MVE_mod_imm<"vmov", "i16", {1,0,?,0}, 0b0, (ins nImmSplatI16:$imm)> {
2452  let Inst{9} = imm{9};
2453}
2454def MVE_VMOVimmi32 : MVE_mod_imm<"vmov", "i32", {?,?,?,?}, 0b0, (ins nImmVMOVI32:$imm)> {
2455  let Inst{11-8} = imm{11-8};
2456}
2457def MVE_VMOVimmi64 : MVE_mod_imm<"vmov", "i64", {1,1,1,0}, 0b1, (ins nImmSplatI64:$imm)>;
2458def MVE_VMOVimmf32 : MVE_mod_imm<"vmov", "f32", {1,1,1,1}, 0b0, (ins nImmVMOVF32:$imm)>;
2459} // let isAsCheapAsAMove = 1
2460
2461def MVE_VMVNimmi16 : MVE_mod_imm<"vmvn", "i16", {1,0,?,0}, 0b1, (ins nImmSplatI16:$imm)> {
2462  let Inst{9} = imm{9};
2463}
2464def MVE_VMVNimmi32 : MVE_mod_imm<"vmvn", "i32", {?,?,?,?}, 0b1, (ins nImmVMOVI32:$imm)> {
2465  let Inst{11-8} = imm{11-8};
2466}
2467} // let isReMaterializable = 1
2468
2469let Predicates = [HasMVEInt] in {
2470  def : Pat<(v16i8 (ARMvmovImm timm:$simm)),
2471            (v16i8 (MVE_VMOVimmi8  nImmSplatI8:$simm))>;
2472  def : Pat<(v8i16 (ARMvmovImm timm:$simm)),
2473            (v8i16 (MVE_VMOVimmi16 nImmSplatI16:$simm))>;
2474  def : Pat<(v4i32 (ARMvmovImm timm:$simm)),
2475            (v4i32 (MVE_VMOVimmi32 nImmVMOVI32:$simm))>;
2476  def : Pat<(v2i64 (ARMvmovImm timm:$simm)),
2477            (v2i64 (MVE_VMOVimmi64 nImmSplatI64:$simm))>;
2478
2479  def : Pat<(v8i16 (ARMvmvnImm timm:$simm)),
2480            (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm))>;
2481  def : Pat<(v4i32 (ARMvmvnImm timm:$simm)),
2482            (v4i32 (MVE_VMVNimmi32 nImmVMOVI32:$simm))>;
2483
2484  def : Pat<(v4f32 (ARMvmovFPImm timm:$simm)),
2485            (v4f32 (MVE_VMOVimmf32 nImmVMOVF32:$simm))>;
2486
2487  def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred), (ARMvmvnImm timm:$simm),
2488                            MQPR:$inactive)),
2489            (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm,
2490                            ARMVCCThen, VCCR:$pred, MQPR:$inactive))>;
2491  def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (ARMvmvnImm timm:$simm),
2492                            MQPR:$inactive)),
2493            (v4i32 (MVE_VMVNimmi32 nImmSplatI32:$simm,
2494                            ARMVCCThen, VCCR:$pred, MQPR:$inactive))>;
2495}
2496
2497class MVE_VMINMAXA<string iname, string suffix, bits<2> size,
2498                   bit bit_12, list<dag> pattern=[]>
2499  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
2500          NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",
2501          pattern> {
2502  bits<4> Qd;
2503  bits<4> Qm;
2504
2505  let Inst{28} = 0b0;
2506  let Inst{25-23} = 0b100;
2507  let Inst{22} = Qd{3};
2508  let Inst{21-20} = 0b11;
2509  let Inst{19-18} = size;
2510  let Inst{17-16} = 0b11;
2511  let Inst{15-13} = Qd{2-0};
2512  let Inst{12} = bit_12;
2513  let Inst{11-6} = 0b111010;
2514  let Inst{5} = Qm{3};
2515  let Inst{4} = 0b0;
2516  let Inst{3-1} = Qm{2-0};
2517  let Inst{0} = 0b1;
2518  let validForTailPredication = 1;
2519}
2520
2521multiclass MVE_VMINMAXA_m<string iname, MVEVectorVTInfo VTI,
2522                      SDNode unpred_op, Intrinsic pred_int, bit bit_12> {
2523  def "" : MVE_VMINMAXA<iname, VTI.Suffix, VTI.Size, bit_12>;
2524  defvar Inst = !cast<Instruction>(NAME);
2525
2526  let Predicates = [HasMVEInt] in {
2527    // Unpredicated v(min|max)a
2528    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qd), (abs (VTI.Vec MQPR:$Qm)))),
2529              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
2530
2531    // Predicated v(min|max)a
2532    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2533                            (VTI.Pred VCCR:$mask))),
2534              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2535                            ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
2536  }
2537}
2538
2539multiclass MVE_VMINA<MVEVectorVTInfo VTI>
2540  : MVE_VMINMAXA_m<"vmina", VTI, umin, int_arm_mve_vmina_predicated, 0b1>;
2541
2542defm MVE_VMINAs8  : MVE_VMINA<MVE_v16s8>;
2543defm MVE_VMINAs16 : MVE_VMINA<MVE_v8s16>;
2544defm MVE_VMINAs32 : MVE_VMINA<MVE_v4s32>;
2545
2546multiclass MVE_VMAXA<MVEVectorVTInfo VTI>
2547  : MVE_VMINMAXA_m<"vmaxa", VTI, umax, int_arm_mve_vmaxa_predicated, 0b0>;
2548
2549defm MVE_VMAXAs8  : MVE_VMAXA<MVE_v16s8>;
2550defm MVE_VMAXAs16 : MVE_VMAXA<MVE_v8s16>;
2551defm MVE_VMAXAs32 : MVE_VMAXA<MVE_v4s32>;
2552
2553// end of MVE Integer instructions
2554
2555// start of mve_imm_shift instructions
2556
2557def MVE_VSHLC : MVE_p<(outs rGPR:$RdmDest, MQPR:$Qd),
2558                      (ins MQPR:$QdSrc, rGPR:$RdmSrc, long_shift:$imm),
2559                      NoItinerary, "vshlc", "", "$QdSrc, $RdmSrc, $imm",
2560                      vpred_n, "$RdmDest = $RdmSrc,$Qd = $QdSrc"> {
2561  bits<5> imm;
2562  bits<4> Qd;
2563  bits<4> RdmDest;
2564
2565  let Inst{28} = 0b0;
2566  let Inst{25-23} = 0b101;
2567  let Inst{22} = Qd{3};
2568  let Inst{21} = 0b1;
2569  let Inst{20-16} = imm{4-0};
2570  let Inst{15-13} = Qd{2-0};
2571  let Inst{12-4} = 0b011111100;
2572  let Inst{3-0} = RdmDest{3-0};
2573}
2574
2575class MVE_shift_imm<dag oops, dag iops, string iname, string suffix,
2576                    string ops, vpred_ops vpred, string cstr,
2577                    list<dag> pattern=[]>
2578  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
2579  bits<4> Qd;
2580  bits<4> Qm;
2581
2582  let Inst{22} = Qd{3};
2583  let Inst{15-13} = Qd{2-0};
2584  let Inst{5} = Qm{3};
2585  let Inst{3-1} = Qm{2-0};
2586}
2587
2588class MVE_VMOVL<string iname, string suffix, bits<2> sz, bit U, bit top,
2589              list<dag> pattern=[]>
2590  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),
2591                  iname, suffix, "$Qd, $Qm", vpred_r, "",
2592                  pattern> {
2593  let Inst{28} = U;
2594  let Inst{25-23} = 0b101;
2595  let Inst{21} = 0b1;
2596  let Inst{20-19} = sz{1-0};
2597  let Inst{18-16} = 0b000;
2598  let Inst{12} = top;
2599  let Inst{11-6} = 0b111101;
2600  let Inst{4} = 0b0;
2601  let Inst{0} = 0b0;
2602  let doubleWidthResult = 1;
2603}
2604
2605multiclass MVE_VMOVL_m<bit top, string chr, MVEVectorVTInfo OutVTI,
2606                       MVEVectorVTInfo InVTI> {
2607  def "": MVE_VMOVL<"vmovl" # chr, InVTI.Suffix, OutVTI.Size,
2608                    InVTI.Unsigned, top>;
2609  defvar Inst = !cast<Instruction>(NAME);
2610
2611  def : Pat<(OutVTI.Vec (int_arm_mve_vmovl_predicated (InVTI.Vec MQPR:$src),
2612                            (i32 InVTI.Unsigned), (i32 top),
2613                            (OutVTI.Pred VCCR:$pred),
2614                            (OutVTI.Vec MQPR:$inactive))),
2615            (OutVTI.Vec (Inst (InVTI.Vec MQPR:$src), ARMVCCThen,
2616                            (OutVTI.Pred VCCR:$pred),
2617                            (OutVTI.Vec MQPR:$inactive)))>;
2618}
2619
2620defm MVE_VMOVLs8bh  : MVE_VMOVL_m<0, "b", MVE_v8s16, MVE_v16s8>;
2621defm MVE_VMOVLs8th  : MVE_VMOVL_m<1, "t", MVE_v8s16, MVE_v16s8>;
2622defm MVE_VMOVLu8bh  : MVE_VMOVL_m<0, "b", MVE_v8u16, MVE_v16u8>;
2623defm MVE_VMOVLu8th  : MVE_VMOVL_m<1, "t", MVE_v8u16, MVE_v16u8>;
2624defm MVE_VMOVLs16bh : MVE_VMOVL_m<0, "b", MVE_v4s32, MVE_v8s16>;
2625defm MVE_VMOVLs16th : MVE_VMOVL_m<1, "t", MVE_v4s32, MVE_v8s16>;
2626defm MVE_VMOVLu16bh : MVE_VMOVL_m<0, "b", MVE_v4s32, MVE_v8u16>;
2627defm MVE_VMOVLu16th : MVE_VMOVL_m<1, "t", MVE_v4s32, MVE_v8u16>;
2628
2629let Predicates = [HasMVEInt] in {
2630  def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i16),
2631            (MVE_VMOVLs16bh MQPR:$src)>;
2632  def : Pat<(sext_inreg (v8i16 MQPR:$src), v8i8),
2633            (MVE_VMOVLs8bh MQPR:$src)>;
2634  def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i8),
2635            (MVE_VMOVLs16bh (MVE_VMOVLs8bh MQPR:$src))>;
2636
2637  def : Pat<(sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), v8i8),
2638            (MVE_VMOVLs8th MQPR:$src)>;
2639  def : Pat<(sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))), v4i16),
2640            (MVE_VMOVLs16th MQPR:$src)>;
2641
2642  // zext_inreg 8 -> 16
2643  def : Pat<(ARMvbicImm (v8i16 MQPR:$src), (i32 0xAFF)),
2644            (MVE_VMOVLu8bh MQPR:$src)>;
2645  // zext_inreg 16 -> 32
2646  def : Pat<(and (v4i32 MQPR:$src), (v4i32 (ARMvmovImm (i32 0xCFF)))),
2647            (MVE_VMOVLu16bh MQPR:$src)>;
2648  // Same zext_inreg with vrevs, picking the top half
2649  def : Pat<(ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), (i32 0xAFF)),
2650            (MVE_VMOVLu8th MQPR:$src)>;
2651  def : Pat<(and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))),
2652                 (v4i32 (ARMvmovImm (i32 0xCFF)))),
2653            (MVE_VMOVLu16th MQPR:$src)>;
2654}
2655
2656
2657class MVE_VSHLL_imm<string iname, string suffix, bit U, bit th,
2658                    Operand immtype, list<dag> pattern=[]>
2659  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm, immtype:$imm),
2660                  iname, suffix, "$Qd, $Qm, $imm", vpred_r, "", pattern> {
2661  let Inst{28} = U;
2662  let Inst{25-23} = 0b101;
2663  let Inst{21} = 0b1;
2664  let Inst{12} = th;
2665  let Inst{11-6} = 0b111101;
2666  let Inst{4} = 0b0;
2667  let Inst{0} = 0b0;
2668
2669  // For the MVE_VSHLL_patterns multiclass to refer to
2670  Operand immediateType = immtype;
2671
2672  let doubleWidthResult = 1;
2673}
2674
2675// The immediate VSHLL instructions accept shift counts from 1 up to
2676// the lane width (8 or 16), but the full-width shifts have an
2677// entirely separate encoding, given below with 'lw' in the name.
2678
2679class MVE_VSHLL_imm8<string iname, string suffix,
2680                     bit U, bit th, list<dag> pattern=[]>
2681  : MVE_VSHLL_imm<iname, suffix, U, th, mve_shift_imm1_7, pattern> {
2682  bits<3> imm;
2683  let Inst{20-19} = 0b01;
2684  let Inst{18-16} = imm;
2685}
2686
2687class MVE_VSHLL_imm16<string iname, string suffix,
2688                      bit U, bit th, list<dag> pattern=[]>
2689  : MVE_VSHLL_imm<iname, suffix, U, th, mve_shift_imm1_15, pattern> {
2690  bits<4> imm;
2691  let Inst{20} = 0b1;
2692  let Inst{19-16} = imm;
2693}
2694
2695def MVE_VSHLL_imms8bh  : MVE_VSHLL_imm8 <"vshllb", "s8", 0b0, 0b0>;
2696def MVE_VSHLL_imms8th  : MVE_VSHLL_imm8 <"vshllt", "s8", 0b0, 0b1>;
2697def MVE_VSHLL_immu8bh  : MVE_VSHLL_imm8 <"vshllb", "u8", 0b1, 0b0>;
2698def MVE_VSHLL_immu8th  : MVE_VSHLL_imm8 <"vshllt", "u8", 0b1, 0b1>;
2699def MVE_VSHLL_imms16bh : MVE_VSHLL_imm16<"vshllb", "s16", 0b0, 0b0>;
2700def MVE_VSHLL_imms16th : MVE_VSHLL_imm16<"vshllt", "s16", 0b0, 0b1>;
2701def MVE_VSHLL_immu16bh : MVE_VSHLL_imm16<"vshllb", "u16", 0b1, 0b0>;
2702def MVE_VSHLL_immu16th : MVE_VSHLL_imm16<"vshllt", "u16", 0b1, 0b1>;
2703
2704class MVE_VSHLL_by_lane_width<string iname, string suffix, bits<2> size,
2705                              bit U, string ops, list<dag> pattern=[]>
2706  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),
2707                  iname, suffix, ops, vpred_r, "", pattern> {
2708  let Inst{28} = U;
2709  let Inst{25-23} = 0b100;
2710  let Inst{21-20} = 0b11;
2711  let Inst{19-18} = size{1-0};
2712  let Inst{17-16} = 0b01;
2713  let Inst{11-6} = 0b111000;
2714  let Inst{4} = 0b0;
2715  let Inst{0} = 0b1;
2716  let doubleWidthResult = 1;
2717}
2718
2719multiclass MVE_VSHLL_lw<string iname, string suffix, bits<2> sz, bit U,
2720                              string ops, list<dag> pattern=[]> {
2721  def bh : MVE_VSHLL_by_lane_width<iname#"b", suffix, sz, U, ops, pattern> {
2722    let Inst{12} = 0b0;
2723  }
2724  def th : MVE_VSHLL_by_lane_width<iname#"t", suffix, sz, U, ops, pattern> {
2725    let Inst{12} = 0b1;
2726  }
2727}
2728
2729defm MVE_VSHLL_lws8  : MVE_VSHLL_lw<"vshll", "s8",  0b00, 0b0, "$Qd, $Qm, #8">;
2730defm MVE_VSHLL_lws16 : MVE_VSHLL_lw<"vshll", "s16", 0b01, 0b0, "$Qd, $Qm, #16">;
2731defm MVE_VSHLL_lwu8  : MVE_VSHLL_lw<"vshll", "u8",  0b00, 0b1, "$Qd, $Qm, #8">;
2732defm MVE_VSHLL_lwu16 : MVE_VSHLL_lw<"vshll", "u16", 0b01, 0b1, "$Qd, $Qm, #16">;
2733
2734multiclass MVE_VSHLL_patterns<MVEVectorVTInfo VTI, int top> {
2735  defvar suffix     = !strconcat(VTI.Suffix, !if(top, "th", "bh"));
2736  defvar inst_imm   = !cast<MVE_VSHLL_imm>("MVE_VSHLL_imm" # suffix);
2737  defvar inst_lw    = !cast<MVE_VSHLL_by_lane_width>("MVE_VSHLL_lw" # suffix);
2738  defvar unpred_int = int_arm_mve_vshll_imm;
2739  defvar pred_int   = int_arm_mve_vshll_imm_predicated;
2740  defvar imm        = inst_imm.immediateType;
2741
2742  def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), imm:$imm,
2743                                    (i32 VTI.Unsigned), (i32 top))),
2744            (VTI.DblVec (inst_imm   (VTI.Vec MQPR:$src), imm:$imm))>;
2745  def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2746                                    (i32 VTI.Unsigned), (i32 top))),
2747            (VTI.DblVec (inst_lw    (VTI.Vec MQPR:$src)))>;
2748
2749  def : Pat<(VTI.DblVec (pred_int   (VTI.Vec MQPR:$src), imm:$imm,
2750                                    (i32 VTI.Unsigned), (i32 top),
2751                                    (VTI.DblPred VCCR:$mask),
2752                                    (VTI.DblVec MQPR:$inactive))),
2753            (VTI.DblVec (inst_imm   (VTI.Vec MQPR:$src), imm:$imm,
2754                                    ARMVCCThen, (VTI.DblPred VCCR:$mask),
2755                                    (VTI.DblVec MQPR:$inactive)))>;
2756  def : Pat<(VTI.DblVec (pred_int   (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2757                                    (i32 VTI.Unsigned), (i32 top),
2758                                    (VTI.DblPred VCCR:$mask),
2759                                    (VTI.DblVec MQPR:$inactive))),
2760            (VTI.DblVec (inst_lw    (VTI.Vec MQPR:$src), ARMVCCThen,
2761                                    (VTI.DblPred VCCR:$mask),
2762                                    (VTI.DblVec MQPR:$inactive)))>;
2763}
2764
2765foreach VTI = [MVE_v16s8, MVE_v8s16, MVE_v16u8, MVE_v8u16] in
2766  foreach top = [0, 1] in
2767    defm : MVE_VSHLL_patterns<VTI, top>;
2768
2769class MVE_shift_imm_partial<Operand imm, string iname, string suffix>
2770  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$QdSrc, MQPR:$Qm, imm:$imm),
2771                  iname, suffix, "$Qd, $Qm, $imm", vpred_n, "$Qd = $QdSrc"> {
2772  Operand immediateType = imm;
2773}
2774
2775class MVE_VxSHRN<string iname, string suffix, bit bit_12, bit bit_28,
2776                 Operand imm, list<dag> pattern=[]>
2777  : MVE_shift_imm_partial<imm, iname, suffix> {
2778  bits<5> imm;
2779
2780  let Inst{28} = bit_28;
2781  let Inst{25-23} = 0b101;
2782  let Inst{21} = 0b0;
2783  let Inst{20-16} = imm{4-0};
2784  let Inst{12} = bit_12;
2785  let Inst{11-6} = 0b111111;
2786  let Inst{4} = 0b0;
2787  let Inst{0} = 0b1;
2788  let validForTailPredication = 1;
2789  let retainsPreviousHalfElement = 1;
2790}
2791
2792def MVE_VRSHRNi16bh : MVE_VxSHRN<"vrshrnb", "i16", 0b0, 0b1, shr_imm8> {
2793  let Inst{20-19} = 0b01;
2794}
2795def MVE_VRSHRNi16th : MVE_VxSHRN<"vrshrnt", "i16", 0b1, 0b1, shr_imm8> {
2796  let Inst{20-19} = 0b01;
2797}
2798def MVE_VRSHRNi32bh : MVE_VxSHRN<"vrshrnb", "i32", 0b0, 0b1, shr_imm16> {
2799  let Inst{20} = 0b1;
2800}
2801def MVE_VRSHRNi32th : MVE_VxSHRN<"vrshrnt", "i32", 0b1, 0b1, shr_imm16> {
2802  let Inst{20} = 0b1;
2803}
2804
2805def MVE_VSHRNi16bh : MVE_VxSHRN<"vshrnb", "i16", 0b0, 0b0, shr_imm8> {
2806  let Inst{20-19} = 0b01;
2807}
2808def MVE_VSHRNi16th : MVE_VxSHRN<"vshrnt", "i16", 0b1, 0b0, shr_imm8> {
2809  let Inst{20-19} = 0b01;
2810}
2811def MVE_VSHRNi32bh : MVE_VxSHRN<"vshrnb", "i32", 0b0, 0b0, shr_imm16> {
2812  let Inst{20} = 0b1;
2813}
2814def MVE_VSHRNi32th : MVE_VxSHRN<"vshrnt", "i32", 0b1, 0b0, shr_imm16> {
2815  let Inst{20} = 0b1;
2816}
2817
2818class MVE_VxQRSHRUN<string iname, string suffix, bit bit_28, bit bit_12,
2819                    Operand imm, list<dag> pattern=[]>
2820  : MVE_shift_imm_partial<imm, iname, suffix> {
2821  bits<5> imm;
2822
2823  let Inst{28} = bit_28;
2824  let Inst{25-23} = 0b101;
2825  let Inst{21} = 0b0;
2826  let Inst{20-16} = imm{4-0};
2827  let Inst{12} = bit_12;
2828  let Inst{11-6} = 0b111111;
2829  let Inst{4} = 0b0;
2830  let Inst{0} = 0b0;
2831  let validForTailPredication = 1;
2832  let retainsPreviousHalfElement = 1;
2833}
2834
2835def MVE_VQRSHRUNs16bh : MVE_VxQRSHRUN<
2836    "vqrshrunb", "s16", 0b1, 0b0, shr_imm8> {
2837  let Inst{20-19} = 0b01;
2838}
2839def MVE_VQRSHRUNs16th : MVE_VxQRSHRUN<
2840    "vqrshrunt", "s16", 0b1, 0b1, shr_imm8> {
2841  let Inst{20-19} = 0b01;
2842}
2843def MVE_VQRSHRUNs32bh : MVE_VxQRSHRUN<
2844    "vqrshrunb", "s32", 0b1, 0b0, shr_imm16> {
2845  let Inst{20} = 0b1;
2846}
2847def MVE_VQRSHRUNs32th : MVE_VxQRSHRUN<
2848    "vqrshrunt", "s32", 0b1, 0b1, shr_imm16> {
2849  let Inst{20} = 0b1;
2850}
2851
2852def MVE_VQSHRUNs16bh : MVE_VxQRSHRUN<
2853    "vqshrunb", "s16", 0b0, 0b0, shr_imm8> {
2854  let Inst{20-19} = 0b01;
2855}
2856def MVE_VQSHRUNs16th : MVE_VxQRSHRUN<
2857    "vqshrunt", "s16", 0b0, 0b1, shr_imm8> {
2858  let Inst{20-19} = 0b01;
2859}
2860def MVE_VQSHRUNs32bh : MVE_VxQRSHRUN<
2861    "vqshrunb", "s32", 0b0, 0b0, shr_imm16> {
2862  let Inst{20} = 0b1;
2863}
2864def MVE_VQSHRUNs32th : MVE_VxQRSHRUN<
2865    "vqshrunt", "s32", 0b0, 0b1, shr_imm16> {
2866  let Inst{20} = 0b1;
2867}
2868
2869class MVE_VxQRSHRN<string iname, string suffix, bit bit_0, bit bit_12,
2870                   Operand imm, list<dag> pattern=[]>
2871  : MVE_shift_imm_partial<imm, iname, suffix> {
2872  bits<5> imm;
2873
2874  let Inst{25-23} = 0b101;
2875  let Inst{21} = 0b0;
2876  let Inst{20-16} = imm{4-0};
2877  let Inst{12} = bit_12;
2878  let Inst{11-6} = 0b111101;
2879  let Inst{4} = 0b0;
2880  let Inst{0} = bit_0;
2881  let validForTailPredication = 1;
2882  let retainsPreviousHalfElement = 1;
2883}
2884
2885multiclass MVE_VxQRSHRN_types<string iname, bit bit_0, bit bit_12> {
2886  def s16 : MVE_VxQRSHRN<iname, "s16", bit_0, bit_12, shr_imm8> {
2887    let Inst{28} = 0b0;
2888    let Inst{20-19} = 0b01;
2889  }
2890  def u16 : MVE_VxQRSHRN<iname, "u16", bit_0, bit_12, shr_imm8> {
2891    let Inst{28} = 0b1;
2892    let Inst{20-19} = 0b01;
2893  }
2894  def s32 : MVE_VxQRSHRN<iname, "s32", bit_0, bit_12, shr_imm16> {
2895    let Inst{28} = 0b0;
2896    let Inst{20} = 0b1;
2897  }
2898  def u32 : MVE_VxQRSHRN<iname, "u32", bit_0, bit_12, shr_imm16> {
2899    let Inst{28} = 0b1;
2900    let Inst{20} = 0b1;
2901  }
2902}
2903
2904defm MVE_VQRSHRNbh : MVE_VxQRSHRN_types<"vqrshrnb", 0b1, 0b0>;
2905defm MVE_VQRSHRNth : MVE_VxQRSHRN_types<"vqrshrnt", 0b1, 0b1>;
2906defm MVE_VQSHRNbh  : MVE_VxQRSHRN_types<"vqshrnb", 0b0, 0b0>;
2907defm MVE_VQSHRNth  : MVE_VxQRSHRN_types<"vqshrnt", 0b0, 0b1>;
2908
2909multiclass MVE_VSHRN_patterns<MVE_shift_imm_partial inst,
2910                              MVEVectorVTInfo OutVTI, MVEVectorVTInfo InVTI,
2911                              bit q, bit r, bit top> {
2912  defvar inparams = (? (OutVTI.Vec MQPR:$QdSrc), (InVTI.Vec MQPR:$Qm),
2913                       (inst.immediateType:$imm), (i32 q), (i32 r),
2914                       (i32 OutVTI.Unsigned), (i32 InVTI.Unsigned), (i32 top));
2915  defvar outparams = (inst (OutVTI.Vec MQPR:$QdSrc), (InVTI.Vec MQPR:$Qm),
2916                           (imm:$imm));
2917
2918  def : Pat<(OutVTI.Vec !setop(inparams, int_arm_mve_vshrn)),
2919            (OutVTI.Vec outparams)>;
2920  def : Pat<(OutVTI.Vec !con(inparams, (int_arm_mve_vshrn_predicated
2921                                           (InVTI.Pred VCCR:$pred)))),
2922            (OutVTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred)))>;
2923}
2924
2925defm : MVE_VSHRN_patterns<MVE_VSHRNi16bh,    MVE_v16s8, MVE_v8s16, 0,0,0>;
2926defm : MVE_VSHRN_patterns<MVE_VSHRNi16th,    MVE_v16s8, MVE_v8s16, 0,0,1>;
2927defm : MVE_VSHRN_patterns<MVE_VSHRNi32bh,    MVE_v8s16, MVE_v4s32, 0,0,0>;
2928defm : MVE_VSHRN_patterns<MVE_VSHRNi32th,    MVE_v8s16, MVE_v4s32, 0,0,1>;
2929defm : MVE_VSHRN_patterns<MVE_VSHRNi16bh,    MVE_v16u8, MVE_v8u16, 0,0,0>;
2930defm : MVE_VSHRN_patterns<MVE_VSHRNi16th,    MVE_v16u8, MVE_v8u16, 0,0,1>;
2931defm : MVE_VSHRN_patterns<MVE_VSHRNi32bh,    MVE_v8u16, MVE_v4u32, 0,0,0>;
2932defm : MVE_VSHRN_patterns<MVE_VSHRNi32th,    MVE_v8u16, MVE_v4u32, 0,0,1>;
2933defm : MVE_VSHRN_patterns<MVE_VRSHRNi16bh,   MVE_v16s8, MVE_v8s16, 0,1,0>;
2934defm : MVE_VSHRN_patterns<MVE_VRSHRNi16th,   MVE_v16s8, MVE_v8s16, 0,1,1>;
2935defm : MVE_VSHRN_patterns<MVE_VRSHRNi32bh,   MVE_v8s16, MVE_v4s32, 0,1,0>;
2936defm : MVE_VSHRN_patterns<MVE_VRSHRNi32th,   MVE_v8s16, MVE_v4s32, 0,1,1>;
2937defm : MVE_VSHRN_patterns<MVE_VRSHRNi16bh,   MVE_v16u8, MVE_v8u16, 0,1,0>;
2938defm : MVE_VSHRN_patterns<MVE_VRSHRNi16th,   MVE_v16u8, MVE_v8u16, 0,1,1>;
2939defm : MVE_VSHRN_patterns<MVE_VRSHRNi32bh,   MVE_v8u16, MVE_v4u32, 0,1,0>;
2940defm : MVE_VSHRN_patterns<MVE_VRSHRNi32th,   MVE_v8u16, MVE_v4u32, 0,1,1>;
2941defm : MVE_VSHRN_patterns<MVE_VQSHRNbhs16,   MVE_v16s8, MVE_v8s16, 1,0,0>;
2942defm : MVE_VSHRN_patterns<MVE_VQSHRNths16,   MVE_v16s8, MVE_v8s16, 1,0,1>;
2943defm : MVE_VSHRN_patterns<MVE_VQSHRNbhs32,   MVE_v8s16, MVE_v4s32, 1,0,0>;
2944defm : MVE_VSHRN_patterns<MVE_VQSHRNths32,   MVE_v8s16, MVE_v4s32, 1,0,1>;
2945defm : MVE_VSHRN_patterns<MVE_VQSHRNbhu16,   MVE_v16u8, MVE_v8u16, 1,0,0>;
2946defm : MVE_VSHRN_patterns<MVE_VQSHRNthu16,   MVE_v16u8, MVE_v8u16, 1,0,1>;
2947defm : MVE_VSHRN_patterns<MVE_VQSHRNbhu32,   MVE_v8u16, MVE_v4u32, 1,0,0>;
2948defm : MVE_VSHRN_patterns<MVE_VQSHRNthu32,   MVE_v8u16, MVE_v4u32, 1,0,1>;
2949defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhs16,  MVE_v16s8, MVE_v8s16, 1,1,0>;
2950defm : MVE_VSHRN_patterns<MVE_VQRSHRNths16,  MVE_v16s8, MVE_v8s16, 1,1,1>;
2951defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhs32,  MVE_v8s16, MVE_v4s32, 1,1,0>;
2952defm : MVE_VSHRN_patterns<MVE_VQRSHRNths32,  MVE_v8s16, MVE_v4s32, 1,1,1>;
2953defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhu16,  MVE_v16u8, MVE_v8u16, 1,1,0>;
2954defm : MVE_VSHRN_patterns<MVE_VQRSHRNthu16,  MVE_v16u8, MVE_v8u16, 1,1,1>;
2955defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhu32,  MVE_v8u16, MVE_v4u32, 1,1,0>;
2956defm : MVE_VSHRN_patterns<MVE_VQRSHRNthu32,  MVE_v8u16, MVE_v4u32, 1,1,1>;
2957defm : MVE_VSHRN_patterns<MVE_VQSHRUNs16bh,  MVE_v16u8, MVE_v8s16, 1,0,0>;
2958defm : MVE_VSHRN_patterns<MVE_VQSHRUNs16th,  MVE_v16u8, MVE_v8s16, 1,0,1>;
2959defm : MVE_VSHRN_patterns<MVE_VQSHRUNs32bh,  MVE_v8u16, MVE_v4s32, 1,0,0>;
2960defm : MVE_VSHRN_patterns<MVE_VQSHRUNs32th,  MVE_v8u16, MVE_v4s32, 1,0,1>;
2961defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs16bh, MVE_v16u8, MVE_v8s16, 1,1,0>;
2962defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs16th, MVE_v16u8, MVE_v8s16, 1,1,1>;
2963defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs32bh, MVE_v8u16, MVE_v4s32, 1,1,0>;
2964defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs32th, MVE_v8u16, MVE_v4s32, 1,1,1>;
2965
2966// end of mve_imm_shift instructions
2967
2968// start of mve_shift instructions
2969
2970class MVE_shift_by_vec<string iname, string suffix, bit U,
2971                       bits<2> size, bit bit_4, bit bit_8>
2972  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm, MQPR:$Qn), NoItinerary,
2973           iname, suffix, "$Qd, $Qm, $Qn", vpred_r, "", []> {
2974  // Shift instructions which take a vector of shift counts
2975  bits<4> Qd;
2976  bits<4> Qm;
2977  bits<4> Qn;
2978
2979  let Inst{28} = U;
2980  let Inst{25-24} = 0b11;
2981  let Inst{23} = 0b0;
2982  let Inst{22} = Qd{3};
2983  let Inst{21-20} = size;
2984  let Inst{19-17} = Qn{2-0};
2985  let Inst{16} = 0b0;
2986  let Inst{15-13} = Qd{2-0};
2987  let Inst{12-9} = 0b0010;
2988  let Inst{8} = bit_8;
2989  let Inst{7} = Qn{3};
2990  let Inst{6} = 0b1;
2991  let Inst{5} = Qm{3};
2992  let Inst{4} = bit_4;
2993  let Inst{3-1} = Qm{2-0};
2994  let Inst{0} = 0b0;
2995  let validForTailPredication = 1;
2996}
2997
2998multiclass MVE_shift_by_vec_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
2999  def "" : MVE_shift_by_vec<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
3000  defvar Inst = !cast<Instruction>(NAME);
3001
3002  def : Pat<(VTI.Vec (int_arm_mve_vshl_vector
3003                         (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3004                         (i32 q), (i32 r), (i32 VTI.Unsigned))),
3005            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh)))>;
3006
3007  def : Pat<(VTI.Vec (int_arm_mve_vshl_vector_predicated
3008                         (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3009                         (i32 q), (i32 r), (i32 VTI.Unsigned),
3010                         (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
3011            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3012                           ARMVCCThen, (VTI.Pred VCCR:$mask),
3013                           (VTI.Vec MQPR:$inactive)))>;
3014}
3015
3016multiclass mve_shift_by_vec_multi<string iname, bit bit_4, bit bit_8> {
3017  defm s8  : MVE_shift_by_vec_p<iname, MVE_v16s8, bit_4, bit_8>;
3018  defm s16 : MVE_shift_by_vec_p<iname, MVE_v8s16, bit_4, bit_8>;
3019  defm s32 : MVE_shift_by_vec_p<iname, MVE_v4s32, bit_4, bit_8>;
3020  defm u8  : MVE_shift_by_vec_p<iname, MVE_v16u8, bit_4, bit_8>;
3021  defm u16 : MVE_shift_by_vec_p<iname, MVE_v8u16, bit_4, bit_8>;
3022  defm u32 : MVE_shift_by_vec_p<iname, MVE_v4u32, bit_4, bit_8>;
3023}
3024
3025defm MVE_VSHL_by_vec   : mve_shift_by_vec_multi<"vshl",   0b0, 0b0>;
3026defm MVE_VQSHL_by_vec  : mve_shift_by_vec_multi<"vqshl",  0b1, 0b0>;
3027defm MVE_VQRSHL_by_vec : mve_shift_by_vec_multi<"vqrshl", 0b1, 0b1>;
3028defm MVE_VRSHL_by_vec  : mve_shift_by_vec_multi<"vrshl",  0b0, 0b1>;
3029
3030let Predicates = [HasMVEInt] in {
3031  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
3032            (v4i32 (MVE_VSHL_by_vecu32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
3033  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
3034            (v8i16 (MVE_VSHL_by_vecu16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
3035  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
3036            (v16i8 (MVE_VSHL_by_vecu8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
3037
3038  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
3039            (v4i32 (MVE_VSHL_by_vecs32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
3040  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
3041            (v8i16 (MVE_VSHL_by_vecs16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
3042  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
3043            (v16i8 (MVE_VSHL_by_vecs8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
3044}
3045
3046class MVE_shift_with_imm<string iname, string suffix, dag oops, dag iops,
3047                         string ops, vpred_ops vpred, string cstr,
3048                         list<dag> pattern=[]>
3049  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
3050  bits<4> Qd;
3051  bits<4> Qm;
3052
3053  let Inst{23} = 0b1;
3054  let Inst{22} = Qd{3};
3055  let Inst{15-13} = Qd{2-0};
3056  let Inst{12-11} = 0b00;
3057  let Inst{7-6} = 0b01;
3058  let Inst{5} = Qm{3};
3059  let Inst{4} = 0b1;
3060  let Inst{3-1} = Qm{2-0};
3061  let Inst{0} = 0b0;
3062  let validForTailPredication = 1;
3063
3064  // For the MVE_shift_imm_patterns multiclass to refer to
3065  MVEVectorVTInfo VTI;
3066  Operand immediateType;
3067  Intrinsic unpred_int;
3068  Intrinsic pred_int;
3069  dag unsignedFlag = (?);
3070}
3071
3072class MVE_VSxI_imm<string iname, string suffix, bit bit_8, Operand immType>
3073  : MVE_shift_with_imm<iname, suffix, (outs MQPR:$Qd),
3074                       (ins MQPR:$Qd_src, MQPR:$Qm, immType:$imm),
3075                       "$Qd, $Qm, $imm", vpred_n, "$Qd = $Qd_src"> {
3076  bits<6> imm;
3077  let Inst{28} = 0b1;
3078  let Inst{25-24} = 0b11;
3079  let Inst{21-16} = imm;
3080  let Inst{10-9} = 0b10;
3081  let Inst{8} = bit_8;
3082  let validForTailPredication = 1;
3083
3084  Operand immediateType = immType;
3085}
3086
3087def MVE_VSRIimm8 : MVE_VSxI_imm<"vsri", "8", 0b0, shr_imm8> {
3088  let Inst{21-19} = 0b001;
3089}
3090
3091def MVE_VSRIimm16 : MVE_VSxI_imm<"vsri", "16", 0b0, shr_imm16> {
3092  let Inst{21-20} = 0b01;
3093}
3094
3095def MVE_VSRIimm32 : MVE_VSxI_imm<"vsri", "32", 0b0, shr_imm32> {
3096  let Inst{21} = 0b1;
3097}
3098
3099def MVE_VSLIimm8 : MVE_VSxI_imm<"vsli", "8", 0b1, imm0_7> {
3100  let Inst{21-19} = 0b001;
3101}
3102
3103def MVE_VSLIimm16 : MVE_VSxI_imm<"vsli", "16", 0b1, imm0_15> {
3104  let Inst{21-20} = 0b01;
3105}
3106
3107def MVE_VSLIimm32 : MVE_VSxI_imm<"vsli", "32", 0b1,imm0_31> {
3108  let Inst{21} = 0b1;
3109}
3110
3111multiclass MVE_VSxI_patterns<MVE_VSxI_imm inst, string name,
3112                              MVEVectorVTInfo VTI> {
3113  defvar inparams = (? (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3114                       (inst.immediateType:$imm));
3115  defvar outparams = (inst (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3116                           (inst.immediateType:$imm));
3117  defvar unpred_int = !cast<Intrinsic>("int_arm_mve_" # name);
3118  defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # name # "_predicated");
3119
3120  def : Pat<(VTI.Vec !setop(inparams, unpred_int)),
3121            (VTI.Vec outparams)>;
3122  def : Pat<(VTI.Vec !con(inparams, (pred_int (VTI.Pred VCCR:$pred)))),
3123            (VTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred)))>;
3124}
3125
3126defm : MVE_VSxI_patterns<MVE_VSLIimm8,  "vsli", MVE_v16i8>;
3127defm : MVE_VSxI_patterns<MVE_VSLIimm16, "vsli", MVE_v8i16>;
3128defm : MVE_VSxI_patterns<MVE_VSLIimm32, "vsli", MVE_v4i32>;
3129defm : MVE_VSxI_patterns<MVE_VSRIimm8,  "vsri", MVE_v16i8>;
3130defm : MVE_VSxI_patterns<MVE_VSRIimm16, "vsri", MVE_v8i16>;
3131defm : MVE_VSxI_patterns<MVE_VSRIimm32, "vsri", MVE_v4i32>;
3132
3133class MVE_VQSHL_imm<MVEVectorVTInfo VTI_, Operand immType>
3134  : MVE_shift_with_imm<"vqshl", VTI_.Suffix, (outs MQPR:$Qd),
3135                       (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",
3136                       vpred_r, ""> {
3137  bits<6> imm;
3138
3139  let Inst{28} = VTI_.Unsigned;
3140  let Inst{25-24} = 0b11;
3141  let Inst{21-16} = imm;
3142  let Inst{10-8} = 0b111;
3143
3144  let VTI = VTI_;
3145  let immediateType = immType;
3146  let unsignedFlag = (? (i32 VTI.Unsigned));
3147}
3148
3149let unpred_int = int_arm_mve_vqshl_imm,
3150    pred_int = int_arm_mve_vqshl_imm_predicated in {
3151  def MVE_VQSHLimms8 : MVE_VQSHL_imm<MVE_v16s8, imm0_7> {
3152    let Inst{21-19} = 0b001;
3153  }
3154  def MVE_VQSHLimmu8 : MVE_VQSHL_imm<MVE_v16u8, imm0_7> {
3155    let Inst{21-19} = 0b001;
3156  }
3157
3158  def MVE_VQSHLimms16 : MVE_VQSHL_imm<MVE_v8s16, imm0_15> {
3159    let Inst{21-20} = 0b01;
3160  }
3161  def MVE_VQSHLimmu16 : MVE_VQSHL_imm<MVE_v8u16, imm0_15> {
3162    let Inst{21-20} = 0b01;
3163  }
3164
3165  def MVE_VQSHLimms32 : MVE_VQSHL_imm<MVE_v4s32, imm0_31> {
3166    let Inst{21} = 0b1;
3167  }
3168  def MVE_VQSHLimmu32 : MVE_VQSHL_imm<MVE_v4u32, imm0_31> {
3169    let Inst{21} = 0b1;
3170  }
3171}
3172
3173class MVE_VQSHLU_imm<MVEVectorVTInfo VTI_, Operand immType>
3174  : MVE_shift_with_imm<"vqshlu", VTI_.Suffix, (outs MQPR:$Qd),
3175                       (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",
3176                       vpred_r, ""> {
3177  bits<6> imm;
3178
3179  let Inst{28} = 0b1;
3180  let Inst{25-24} = 0b11;
3181  let Inst{21-16} = imm;
3182  let Inst{10-8} = 0b110;
3183
3184  let VTI = VTI_;
3185  let immediateType = immType;
3186}
3187
3188let unpred_int = int_arm_mve_vqshlu_imm,
3189    pred_int = int_arm_mve_vqshlu_imm_predicated in {
3190  def MVE_VQSHLU_imms8 : MVE_VQSHLU_imm<MVE_v16s8, imm0_7> {
3191    let Inst{21-19} = 0b001;
3192  }
3193
3194  def MVE_VQSHLU_imms16 : MVE_VQSHLU_imm<MVE_v8s16, imm0_15> {
3195    let Inst{21-20} = 0b01;
3196  }
3197
3198  def MVE_VQSHLU_imms32 : MVE_VQSHLU_imm<MVE_v4s32, imm0_31> {
3199    let Inst{21} = 0b1;
3200  }
3201}
3202
3203class MVE_VRSHR_imm<MVEVectorVTInfo VTI_, Operand immType>
3204  : MVE_shift_with_imm<"vrshr", VTI_.Suffix, (outs MQPR:$Qd),
3205                       (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",
3206                       vpred_r, ""> {
3207  bits<6> imm;
3208
3209  let Inst{28} = VTI_.Unsigned;
3210  let Inst{25-24} = 0b11;
3211  let Inst{21-16} = imm;
3212  let Inst{10-8} = 0b010;
3213
3214  let VTI = VTI_;
3215  let immediateType = immType;
3216  let unsignedFlag = (? (i32 VTI.Unsigned));
3217}
3218
3219let unpred_int = int_arm_mve_vrshr_imm,
3220    pred_int = int_arm_mve_vrshr_imm_predicated in {
3221  def MVE_VRSHR_imms8 : MVE_VRSHR_imm<MVE_v16s8, shr_imm8> {
3222    let Inst{21-19} = 0b001;
3223  }
3224
3225  def MVE_VRSHR_immu8 : MVE_VRSHR_imm<MVE_v16u8, shr_imm8> {
3226    let Inst{21-19} = 0b001;
3227  }
3228
3229  def MVE_VRSHR_imms16 : MVE_VRSHR_imm<MVE_v8s16, shr_imm16> {
3230    let Inst{21-20} = 0b01;
3231  }
3232
3233  def MVE_VRSHR_immu16 : MVE_VRSHR_imm<MVE_v8u16, shr_imm16> {
3234    let Inst{21-20} = 0b01;
3235  }
3236
3237  def MVE_VRSHR_imms32 : MVE_VRSHR_imm<MVE_v4s32, shr_imm32> {
3238    let Inst{21} = 0b1;
3239  }
3240
3241  def MVE_VRSHR_immu32 : MVE_VRSHR_imm<MVE_v4u32, shr_imm32> {
3242    let Inst{21} = 0b1;
3243  }
3244}
3245
3246multiclass MVE_shift_imm_patterns<MVE_shift_with_imm inst> {
3247  def : Pat<(inst.VTI.Vec !con((inst.unpred_int (inst.VTI.Vec MQPR:$src),
3248                                                inst.immediateType:$imm),
3249                               inst.unsignedFlag)),
3250            (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3251                                inst.immediateType:$imm))>;
3252
3253  def : Pat<(inst.VTI.Vec !con((inst.pred_int (inst.VTI.Vec MQPR:$src),
3254                                              inst.immediateType:$imm),
3255                               inst.unsignedFlag,
3256                               (? (inst.VTI.Pred VCCR:$mask),
3257                                  (inst.VTI.Vec MQPR:$inactive)))),
3258            (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3259                                inst.immediateType:$imm,
3260                                ARMVCCThen, (inst.VTI.Pred VCCR:$mask),
3261                                (inst.VTI.Vec MQPR:$inactive)))>;
3262}
3263
3264defm : MVE_shift_imm_patterns<MVE_VQSHLimms8>;
3265defm : MVE_shift_imm_patterns<MVE_VQSHLimmu8>;
3266defm : MVE_shift_imm_patterns<MVE_VQSHLimms16>;
3267defm : MVE_shift_imm_patterns<MVE_VQSHLimmu16>;
3268defm : MVE_shift_imm_patterns<MVE_VQSHLimms32>;
3269defm : MVE_shift_imm_patterns<MVE_VQSHLimmu32>;
3270defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms8>;
3271defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms16>;
3272defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms32>;
3273defm : MVE_shift_imm_patterns<MVE_VRSHR_imms8>;
3274defm : MVE_shift_imm_patterns<MVE_VRSHR_immu8>;
3275defm : MVE_shift_imm_patterns<MVE_VRSHR_imms16>;
3276defm : MVE_shift_imm_patterns<MVE_VRSHR_immu16>;
3277defm : MVE_shift_imm_patterns<MVE_VRSHR_imms32>;
3278defm : MVE_shift_imm_patterns<MVE_VRSHR_immu32>;
3279
3280class MVE_VSHR_imm<string suffix, dag imm>
3281  : MVE_shift_with_imm<"vshr", suffix, (outs MQPR:$Qd),
3282                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
3283                       vpred_r, ""> {
3284  bits<6> imm;
3285
3286  let Inst{25-24} = 0b11;
3287  let Inst{21-16} = imm;
3288  let Inst{10-8} = 0b000;
3289}
3290
3291def MVE_VSHR_imms8 : MVE_VSHR_imm<"s8", (ins shr_imm8:$imm)> {
3292  let Inst{28} = 0b0;
3293  let Inst{21-19} = 0b001;
3294}
3295
3296def MVE_VSHR_immu8 : MVE_VSHR_imm<"u8", (ins shr_imm8:$imm)> {
3297  let Inst{28} = 0b1;
3298  let Inst{21-19} = 0b001;
3299}
3300
3301def MVE_VSHR_imms16 : MVE_VSHR_imm<"s16", (ins shr_imm16:$imm)> {
3302  let Inst{28} = 0b0;
3303  let Inst{21-20} = 0b01;
3304}
3305
3306def MVE_VSHR_immu16 : MVE_VSHR_imm<"u16", (ins shr_imm16:$imm)> {
3307  let Inst{28} = 0b1;
3308  let Inst{21-20} = 0b01;
3309}
3310
3311def MVE_VSHR_imms32 : MVE_VSHR_imm<"s32", (ins shr_imm32:$imm)> {
3312  let Inst{28} = 0b0;
3313  let Inst{21} = 0b1;
3314}
3315
3316def MVE_VSHR_immu32 : MVE_VSHR_imm<"u32", (ins shr_imm32:$imm)> {
3317  let Inst{28} = 0b1;
3318  let Inst{21} = 0b1;
3319}
3320
3321class MVE_VSHL_imm<string suffix, dag imm>
3322  : MVE_shift_with_imm<"vshl", suffix, (outs MQPR:$Qd),
3323                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
3324                       vpred_r, ""> {
3325  bits<6> imm;
3326
3327  let Inst{28} = 0b0;
3328  let Inst{25-24} = 0b11;
3329  let Inst{21-16} = imm;
3330  let Inst{10-8} = 0b101;
3331}
3332
3333def MVE_VSHL_immi8 : MVE_VSHL_imm<"i8", (ins imm0_7:$imm)> {
3334  let Inst{21-19} = 0b001;
3335}
3336
3337def MVE_VSHL_immi16 : MVE_VSHL_imm<"i16", (ins imm0_15:$imm)> {
3338  let Inst{21-20} = 0b01;
3339}
3340
3341def MVE_VSHL_immi32 : MVE_VSHL_imm<"i32", (ins imm0_31:$imm)> {
3342  let Inst{21} = 0b1;
3343}
3344
3345multiclass MVE_immediate_shift_patterns_inner<
3346    MVEVectorVTInfo VTI, Operand imm_operand_type, SDNode unpred_op,
3347    Intrinsic pred_int, Instruction inst, list<int> unsignedFlag = []> {
3348
3349  def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src), imm_operand_type:$imm)),
3350            (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm))>;
3351
3352  def : Pat<(VTI.Vec !con((pred_int (VTI.Vec MQPR:$src), imm_operand_type:$imm),
3353                          !dag(pred_int, unsignedFlag, ?),
3354                          (pred_int (VTI.Pred VCCR:$mask),
3355                                   (VTI.Vec MQPR:$inactive)))),
3356            (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm,
3357                           ARMVCCThen, (VTI.Pred VCCR:$mask),
3358                           (VTI.Vec MQPR:$inactive)))>;
3359}
3360
3361multiclass MVE_immediate_shift_patterns<MVEVectorVTInfo VTI,
3362                                        Operand imm_operand_type> {
3363  defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3364      ARMvshlImm, int_arm_mve_shl_imm_predicated,
3365      !cast<Instruction>("MVE_VSHL_immi" # VTI.BitsSuffix)>;
3366  defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3367      ARMvshruImm, int_arm_mve_shr_imm_predicated,
3368      !cast<Instruction>("MVE_VSHR_immu" # VTI.BitsSuffix), [1]>;
3369  defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3370      ARMvshrsImm, int_arm_mve_shr_imm_predicated,
3371      !cast<Instruction>("MVE_VSHR_imms" # VTI.BitsSuffix), [0]>;
3372}
3373
3374let Predicates = [HasMVEInt] in {
3375  defm : MVE_immediate_shift_patterns<MVE_v16i8, imm0_7>;
3376  defm : MVE_immediate_shift_patterns<MVE_v8i16, imm0_15>;
3377  defm : MVE_immediate_shift_patterns<MVE_v4i32, imm0_31>;
3378}
3379
3380// end of mve_shift instructions
3381
3382// start of MVE Floating Point instructions
3383
3384class MVE_float<string iname, string suffix, dag oops, dag iops, string ops,
3385                vpred_ops vpred, string cstr, list<dag> pattern=[]>
3386  : MVE_f<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
3387  bits<4> Qm;
3388
3389  let Inst{12} = 0b0;
3390  let Inst{6} = 0b1;
3391  let Inst{5} = Qm{3};
3392  let Inst{3-1} = Qm{2-0};
3393  let Inst{0} = 0b0;
3394}
3395
3396class MVE_VRINT<string rmode, bits<3> op, string suffix, bits<2> size,
3397                list<dag> pattern=[]>
3398  : MVE_float<!strconcat("vrint", rmode), suffix, (outs MQPR:$Qd),
3399              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
3400  bits<4> Qd;
3401
3402  let Inst{28} = 0b1;
3403  let Inst{25-23} = 0b111;
3404  let Inst{22} = Qd{3};
3405  let Inst{21-20} = 0b11;
3406  let Inst{19-18} = size;
3407  let Inst{17-16} = 0b10;
3408  let Inst{15-13} = Qd{2-0};
3409  let Inst{11-10} = 0b01;
3410  let Inst{9-7} = op{2-0};
3411  let Inst{4} = 0b0;
3412  let validForTailPredication = 1;
3413
3414}
3415
3416multiclass MVE_VRINT_m<MVEVectorVTInfo VTI, string suffix, bits<3> opcode,
3417                       SDNode unpred_op> {
3418  def "": MVE_VRINT<suffix, opcode, VTI.Suffix, VTI.Size>;
3419  defvar Inst = !cast<Instruction>(NAME);
3420  defvar pred_int = !cast<Intrinsic>("int_arm_mve_vrint"#suffix#"_predicated");
3421
3422  let Predicates = [HasMVEFloat] in {
3423    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
3424              (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
3425    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
3426                                 (VTI.Vec MQPR:$inactive))),
3427              (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
3428                             (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
3429  }
3430}
3431
3432multiclass MVE_VRINT_ops<MVEVectorVTInfo VTI> {
3433  defm N : MVE_VRINT_m<VTI, "n", 0b000, int_arm_mve_vrintn>;
3434  defm X : MVE_VRINT_m<VTI, "x", 0b001, frint>;
3435  defm A : MVE_VRINT_m<VTI, "a", 0b010, fround>;
3436  defm Z : MVE_VRINT_m<VTI, "z", 0b011, ftrunc>;
3437  defm M : MVE_VRINT_m<VTI, "m", 0b101, ffloor>;
3438  defm P : MVE_VRINT_m<VTI, "p", 0b111, fceil>;
3439}
3440
3441defm MVE_VRINTf16 : MVE_VRINT_ops<MVE_v8f16>;
3442defm MVE_VRINTf32 : MVE_VRINT_ops<MVE_v4f32>;
3443
3444class MVEFloatArithNeon<string iname, string suffix, bit size,
3445                           dag oops, dag iops, string ops,
3446                           vpred_ops vpred, string cstr, list<dag> pattern=[]>
3447  : MVE_float<iname, suffix, oops, iops, ops, vpred, cstr, pattern> {
3448  let Inst{20} = size;
3449  let Inst{16} = 0b0;
3450}
3451
3452class MVE_VMUL_fp<string iname, string suffix, bit size, list<dag> pattern=[]>
3453  : MVEFloatArithNeon<iname, suffix, size, (outs MQPR:$Qd),
3454                      (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm", vpred_r, "",
3455                      pattern> {
3456  bits<4> Qd;
3457  bits<4> Qn;
3458
3459  let Inst{28} = 0b1;
3460  let Inst{25-23} = 0b110;
3461  let Inst{22} = Qd{3};
3462  let Inst{21} = 0b0;
3463  let Inst{19-17} = Qn{2-0};
3464  let Inst{15-13} = Qd{2-0};
3465  let Inst{12-8} = 0b01101;
3466  let Inst{7} = Qn{3};
3467  let Inst{4} = 0b1;
3468  let validForTailPredication = 1;
3469}
3470
3471multiclass MVE_VMULT_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
3472                            SDNode unpred_op, Intrinsic pred_int> {
3473  def "" : MVE_VMUL_fp<iname, VTI.Suffix, VTI.Size{0}>;
3474  defvar Inst = !cast<Instruction>(NAME);
3475
3476  let Predicates = [HasMVEFloat] in {
3477    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
3478              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
3479    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3480                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
3481              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3482                             ARMVCCThen, (VTI.Pred VCCR:$mask),
3483                             (VTI.Vec MQPR:$inactive)))>;
3484  }
3485}
3486
3487multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI>
3488  : MVE_VMULT_fp_m<"vmul", 0, VTI, fmul, int_arm_mve_mul_predicated>;
3489
3490defm MVE_VMULf32 : MVE_VMUL_fp_m<MVE_v4f32>;
3491defm MVE_VMULf16 : MVE_VMUL_fp_m<MVE_v8f16>;
3492
3493class MVE_VCMLA<string suffix, bit size>
3494  : MVEFloatArithNeon<"vcmla", suffix, size, (outs MQPR:$Qd),
3495                         (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
3496                         "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", []> {
3497  bits<4> Qd;
3498  bits<4> Qn;
3499  bits<2> rot;
3500
3501  let Inst{28} = 0b1;
3502  let Inst{25} = 0b0;
3503  let Inst{24-23} = rot;
3504  let Inst{22} = Qd{3};
3505  let Inst{21} = 0b1;
3506  let Inst{19-17} = Qn{2-0};
3507  let Inst{15-13} = Qd{2-0};
3508  let Inst{12-8} = 0b01000;
3509  let Inst{7} = Qn{3};
3510  let Inst{4} = 0b0;
3511}
3512
3513multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI, bit size> {
3514  def "" : MVE_VCMLA<VTI.Suffix, size>;
3515  defvar Inst = !cast<Instruction>(NAME);
3516
3517  let Predicates = [HasMVEFloat] in {
3518    def : Pat<(VTI.Vec (int_arm_mve_vcmlaq
3519                            imm:$rot, (VTI.Vec MQPR:$Qd_src),
3520                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3521              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
3522                             (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3523                             imm:$rot))>;
3524
3525    def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated
3526                            imm:$rot, (VTI.Vec MQPR:$Qd_src),
3527                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3528                            (VTI.Pred VCCR:$mask))),
3529              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),
3530                             (VTI.Vec MQPR:$Qm), imm:$rot,
3531                             ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
3532
3533  }
3534}
3535
3536defm MVE_VCMLAf16 : MVE_VCMLA_m<MVE_v8f16, 0b0>;
3537defm MVE_VCMLAf32 : MVE_VCMLA_m<MVE_v4f32, 0b1>;
3538
3539class MVE_VADDSUBFMA_fp<string iname, string suffix, bit size, bit bit_4,
3540                        bit bit_8, bit bit_21, dag iops=(ins),
3541                        vpred_ops vpred=vpred_r, string cstr="",
3542                        list<dag> pattern=[]>
3543  : MVEFloatArithNeon<iname, suffix, size, (outs MQPR:$Qd),
3544                      !con(iops, (ins MQPR:$Qn, MQPR:$Qm)), "$Qd, $Qn, $Qm",
3545                      vpred, cstr, pattern> {
3546  bits<4> Qd;
3547  bits<4> Qn;
3548
3549  let Inst{28} = 0b0;
3550  let Inst{25-23} = 0b110;
3551  let Inst{22} = Qd{3};
3552  let Inst{21} = bit_21;
3553  let Inst{19-17} = Qn{2-0};
3554  let Inst{15-13} = Qd{2-0};
3555  let Inst{11-9} = 0b110;
3556  let Inst{8} = bit_8;
3557  let Inst{7} = Qn{3};
3558  let Inst{4} = bit_4;
3559  let validForTailPredication = 1;
3560}
3561
3562multiclass MVE_VFMA_fp_multi<string iname, bit fms, MVEVectorVTInfo VTI> {
3563  def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size{0}, 0b1, 0b0, fms,
3564                             (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
3565  defvar Inst = !cast<Instruction>(NAME);
3566  defvar pred_int = int_arm_mve_fma_predicated;
3567  defvar m1   = (VTI.Vec MQPR:$m1);
3568  defvar m2   = (VTI.Vec MQPR:$m2);
3569  defvar add  = (VTI.Vec MQPR:$add);
3570  defvar pred = (VTI.Pred VCCR:$pred);
3571
3572  let Predicates = [HasMVEFloat] in {
3573    if fms then {
3574      def : Pat<(VTI.Vec (fma (fneg m1), m2, add)), (Inst $add, $m1, $m2)>;
3575      def : Pat<(VTI.Vec (fma m1, (fneg m2), add)), (Inst $add, $m1, $m2)>;
3576      def : Pat<(VTI.Vec (pred_int (fneg m1), m2, add, pred)),
3577                (Inst $add, $m1, $m2, ARMVCCThen, $pred)>;
3578      def : Pat<(VTI.Vec (pred_int m1, (fneg m2), add, pred)),
3579                (Inst $add, $m1, $m2, ARMVCCThen, $pred)>;
3580    } else {
3581      def : Pat<(VTI.Vec (fma m1, m2, add)), (Inst $add, $m1, $m2)>;
3582      def : Pat<(VTI.Vec (pred_int m1, m2, add, pred)),
3583                (Inst $add, $m1, $m2, ARMVCCThen, $pred)>;
3584    }
3585  }
3586}
3587
3588defm MVE_VFMAf32 : MVE_VFMA_fp_multi<"vfma", 0, MVE_v4f32>;
3589defm MVE_VFMAf16 : MVE_VFMA_fp_multi<"vfma", 0, MVE_v8f16>;
3590defm MVE_VFMSf32 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v4f32>;
3591defm MVE_VFMSf16 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v8f16>;
3592
3593multiclass MVE_VADDSUB_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
3594                            SDNode unpred_op, Intrinsic pred_int> {
3595  def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size{0}, 0, 1, bit_21> {
3596    let validForTailPredication = 1;
3597  }
3598  defvar Inst = !cast<Instruction>(NAME);
3599
3600  let Predicates = [HasMVEFloat] in {
3601    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
3602              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
3603    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3604                            (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
3605              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3606                             ARMVCCThen, (VTI.Pred VCCR:$mask),
3607                             (VTI.Vec MQPR:$inactive)))>;
3608  }
3609}
3610
3611multiclass MVE_VADD_fp_m<MVEVectorVTInfo VTI>
3612  : MVE_VADDSUB_fp_m<"vadd", 0, VTI, fadd, int_arm_mve_add_predicated>;
3613multiclass MVE_VSUB_fp_m<MVEVectorVTInfo VTI>
3614  : MVE_VADDSUB_fp_m<"vsub", 1, VTI, fsub, int_arm_mve_sub_predicated>;
3615
3616defm MVE_VADDf32 : MVE_VADD_fp_m<MVE_v4f32>;
3617defm MVE_VADDf16 : MVE_VADD_fp_m<MVE_v8f16>;
3618
3619defm MVE_VSUBf32 : MVE_VSUB_fp_m<MVE_v4f32>;
3620defm MVE_VSUBf16 : MVE_VSUB_fp_m<MVE_v8f16>;
3621
3622class MVE_VCADD<string suffix, bit size, string cstr="">
3623  : MVEFloatArithNeon<"vcadd", suffix, size, (outs MQPR:$Qd),
3624                         (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
3625                         "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
3626  bits<4> Qd;
3627  bits<4> Qn;
3628  bit rot;
3629
3630  let Inst{28} = 0b1;
3631  let Inst{25} = 0b0;
3632  let Inst{24} = rot;
3633  let Inst{23} = 0b1;
3634  let Inst{22} = Qd{3};
3635  let Inst{21} = 0b0;
3636  let Inst{19-17} = Qn{2-0};
3637  let Inst{15-13} = Qd{2-0};
3638  let Inst{12-8} = 0b01000;
3639  let Inst{7} = Qn{3};
3640  let Inst{4} = 0b0;
3641}
3642
3643multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, bit size, string cstr=""> {
3644  def "" : MVE_VCADD<VTI.Suffix, size, cstr>;
3645  defvar Inst = !cast<Instruction>(NAME);
3646
3647  let Predicates = [HasMVEFloat] in {
3648    def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),
3649                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3650              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3651                             imm:$rot))>;
3652
3653    def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),
3654                            imm:$rot, (VTI.Vec MQPR:$inactive),
3655                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3656                            (VTI.Pred VCCR:$mask))),
3657              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3658                             imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask),
3659                             (VTI.Vec MQPR:$inactive)))>;
3660
3661  }
3662}
3663
3664defm MVE_VCADDf16 : MVE_VCADD_m<MVE_v8f16, 0b0>;
3665defm MVE_VCADDf32 : MVE_VCADD_m<MVE_v4f32, 0b1, "@earlyclobber $Qd">;
3666
3667class MVE_VABD_fp<string suffix, bit size>
3668  : MVE_float<"vabd", suffix, (outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
3669              "$Qd, $Qn, $Qm", vpred_r, ""> {
3670  bits<4> Qd;
3671  bits<4> Qn;
3672
3673  let Inst{28} = 0b1;
3674  let Inst{25-23} = 0b110;
3675  let Inst{22} = Qd{3};
3676  let Inst{21} = 0b1;
3677  let Inst{20} = size;
3678  let Inst{19-17} = Qn{2-0};
3679  let Inst{16} = 0b0;
3680  let Inst{15-13} = Qd{2-0};
3681  let Inst{11-8} = 0b1101;
3682  let Inst{7} = Qn{3};
3683  let Inst{4} = 0b0;
3684  let validForTailPredication = 1;
3685}
3686
3687multiclass MVE_VABDT_fp_m<MVEVectorVTInfo VTI,
3688                            Intrinsic unpred_int, Intrinsic pred_int> {
3689  def "" : MVE_VABD_fp<VTI.Suffix, VTI.Size{0}>;
3690  defvar Inst = !cast<Instruction>(NAME);
3691
3692  let Predicates = [HasMVEFloat] in {
3693    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3694                            (i32 0))),
3695              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
3696    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3697                            (i32 0), (VTI.Pred VCCR:$mask),
3698                            (VTI.Vec MQPR:$inactive))),
3699              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3700                             ARMVCCThen, (VTI.Pred VCCR:$mask),
3701                             (VTI.Vec MQPR:$inactive)))>;
3702  }
3703}
3704
3705multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>
3706  : MVE_VABDT_fp_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
3707
3708defm MVE_VABDf32 : MVE_VABD_fp_m<MVE_v4f32>;
3709defm MVE_VABDf16 : MVE_VABD_fp_m<MVE_v8f16>;
3710
3711class MVE_VCVT_fix<string suffix, bit fsi, bit U, bit op,
3712                   Operand imm_operand_type>
3713  : MVE_float<"vcvt", suffix,
3714              (outs MQPR:$Qd), (ins MQPR:$Qm, imm_operand_type:$imm6),
3715              "$Qd, $Qm, $imm6", vpred_r, "", []> {
3716  bits<4> Qd;
3717  bits<6> imm6;
3718
3719  let Inst{28} = U;
3720  let Inst{25-23} = 0b111;
3721  let Inst{22} = Qd{3};
3722  let Inst{21} = 0b1;
3723  let Inst{19-16} = imm6{3-0};
3724  let Inst{15-13} = Qd{2-0};
3725  let Inst{11-10} = 0b11;
3726  let Inst{9} = fsi;
3727  let Inst{8} = op;
3728  let Inst{7} = 0b0;
3729  let Inst{4} = 0b1;
3730
3731  let DecoderMethod = "DecodeMVEVCVTt1fp";
3732  let validForTailPredication = 1;
3733}
3734
3735class MVE_VCVT_imm_asmop<int Bits> : AsmOperandClass {
3736  let PredicateMethod = "isImmediate<1," # Bits # ">";
3737  let DiagnosticString =
3738      "MVE fixed-point immediate operand must be between 1 and " # Bits;
3739  let Name = "MVEVcvtImm" # Bits;
3740  let RenderMethod = "addImmOperands";
3741}
3742class MVE_VCVT_imm<int Bits>: Operand<i32> {
3743  let ParserMatchClass = MVE_VCVT_imm_asmop<Bits>;
3744  let EncoderMethod = "getNEONVcvtImm32OpValue";
3745  let DecoderMethod = "DecodeVCVTImmOperand";
3746}
3747
3748class MVE_VCVT_fix_f32<string suffix, bit U, bit op>
3749    : MVE_VCVT_fix<suffix, 0b1, U, op, MVE_VCVT_imm<32>> {
3750  let Inst{20} = imm6{4};
3751}
3752class MVE_VCVT_fix_f16<string suffix, bit U, bit op>
3753    : MVE_VCVT_fix<suffix, 0b0, U, op, MVE_VCVT_imm<16>> {
3754  let Inst{20} = 0b1;
3755}
3756
3757multiclass MVE_VCVT_fix_patterns<Instruction Inst, bit U, MVEVectorVTInfo DestVTI,
3758                                 MVEVectorVTInfo SrcVTI> {
3759  let Predicates = [HasMVEFloat] in {
3760    def : Pat<(DestVTI.Vec (int_arm_mve_vcvt_fix
3761                              (i32 U), (SrcVTI.Vec MQPR:$Qm), imm:$scale)),
3762              (DestVTI.Vec (Inst (SrcVTI.Vec MQPR:$Qm), imm:$scale))>;
3763    def : Pat<(DestVTI.Vec (int_arm_mve_vcvt_fix_predicated (i32 U),
3764                              (DestVTI.Vec MQPR:$inactive),
3765                              (SrcVTI.Vec MQPR:$Qm),
3766                              imm:$scale,
3767                              (DestVTI.Pred VCCR:$mask))),
3768              (DestVTI.Vec (Inst (SrcVTI.Vec MQPR:$Qm), imm:$scale,
3769                             ARMVCCThen, (DestVTI.Pred VCCR:$mask),
3770                             (DestVTI.Vec MQPR:$inactive)))>;
3771  }
3772}
3773
3774multiclass MVE_VCVT_fix_f32_m<bit U, bit op,
3775                              MVEVectorVTInfo DestVTI, MVEVectorVTInfo SrcVTI> {
3776  def "" : MVE_VCVT_fix_f32<DestVTI.Suffix#"."#SrcVTI.Suffix, U, op>;
3777  defm : MVE_VCVT_fix_patterns<!cast<Instruction>(NAME), U, DestVTI, SrcVTI>;
3778}
3779
3780multiclass MVE_VCVT_fix_f16_m<bit U, bit op,
3781                              MVEVectorVTInfo DestVTI, MVEVectorVTInfo SrcVTI> {
3782  def "" : MVE_VCVT_fix_f16<DestVTI.Suffix#"."#SrcVTI.Suffix, U, op>;
3783  defm : MVE_VCVT_fix_patterns<!cast<Instruction>(NAME), U, DestVTI, SrcVTI>;
3784}
3785
3786defm MVE_VCVTf16s16_fix : MVE_VCVT_fix_f16_m<0b0, 0b0, MVE_v8f16, MVE_v8s16>;
3787defm MVE_VCVTs16f16_fix : MVE_VCVT_fix_f16_m<0b0, 0b1, MVE_v8s16, MVE_v8f16>;
3788defm MVE_VCVTf16u16_fix : MVE_VCVT_fix_f16_m<0b1, 0b0, MVE_v8f16, MVE_v8u16>;
3789defm MVE_VCVTu16f16_fix : MVE_VCVT_fix_f16_m<0b1, 0b1, MVE_v8u16, MVE_v8f16>;
3790defm MVE_VCVTf32s32_fix : MVE_VCVT_fix_f32_m<0b0, 0b0, MVE_v4f32, MVE_v4s32>;
3791defm MVE_VCVTs32f32_fix : MVE_VCVT_fix_f32_m<0b0, 0b1, MVE_v4s32, MVE_v4f32>;
3792defm MVE_VCVTf32u32_fix : MVE_VCVT_fix_f32_m<0b1, 0b0, MVE_v4f32, MVE_v4u32>;
3793defm MVE_VCVTu32f32_fix : MVE_VCVT_fix_f32_m<0b1, 0b1, MVE_v4u32, MVE_v4f32>;
3794
3795class MVE_VCVT_fp_int_anpm<string suffix, bits<2> size, bit op, string anpm,
3796                bits<2> rm, list<dag> pattern=[]>
3797  : MVE_float<!strconcat("vcvt", anpm), suffix, (outs MQPR:$Qd),
3798              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
3799  bits<4> Qd;
3800
3801  let Inst{28} = 0b1;
3802  let Inst{25-23} = 0b111;
3803  let Inst{22} = Qd{3};
3804  let Inst{21-20} = 0b11;
3805  let Inst{19-18} = size;
3806  let Inst{17-16} = 0b11;
3807  let Inst{15-13} = Qd{2-0};
3808  let Inst{12-10} = 0b000;
3809  let Inst{9-8} = rm;
3810  let Inst{7} = op;
3811  let Inst{4} = 0b0;
3812  let validForTailPredication = 1;
3813}
3814
3815multiclass MVE_VCVT_fp_int_anpm_inner<MVEVectorVTInfo Int, MVEVectorVTInfo Flt,
3816                                      string anpm, bits<2> rm> {
3817  def "": MVE_VCVT_fp_int_anpm<Int.Suffix # "." # Flt.Suffix, Int.Size,
3818                               Int.Unsigned, anpm, rm>;
3819
3820  defvar Inst         = !cast<Instruction>(NAME);
3821  defvar IntrBaseName = "int_arm_mve_vcvt" # anpm;
3822  defvar UnpredIntr   = !cast<Intrinsic>(IntrBaseName);
3823  defvar PredIntr     = !cast<Intrinsic>(IntrBaseName # "_predicated");
3824
3825  let Predicates = [HasMVEFloat] in {
3826    def : Pat<(Int.Vec (UnpredIntr (i32 Int.Unsigned), (Flt.Vec MQPR:$in))),
3827              (Int.Vec (Inst (Flt.Vec MQPR:$in)))>;
3828
3829    def : Pat<(Int.Vec (PredIntr (i32 Int.Unsigned), (Int.Vec MQPR:$inactive),
3830                                 (Flt.Vec MQPR:$in), (Flt.Pred VCCR:$pred))),
3831              (Int.Vec (Inst (Flt.Vec MQPR:$in), ARMVCCThen,
3832                             (Flt.Pred VCCR:$pred), (Int.Vec MQPR:$inactive)))>;
3833  }
3834}
3835
3836multiclass MVE_VCVT_fp_int_anpm_outer<MVEVectorVTInfo Int,
3837                                      MVEVectorVTInfo Flt> {
3838  defm a : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "a", 0b00>;
3839  defm n : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "n", 0b01>;
3840  defm p : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "p", 0b10>;
3841  defm m : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "m", 0b11>;
3842}
3843
3844// This defines instructions such as MVE_VCVTu16f16a, with an explicit
3845// rounding-mode suffix on the mnemonic. The class below will define
3846// the bare MVE_VCVTu16f16 (with implied rounding toward zero).
3847defm MVE_VCVTs16f16 : MVE_VCVT_fp_int_anpm_outer<MVE_v8s16, MVE_v8f16>;
3848defm MVE_VCVTu16f16 : MVE_VCVT_fp_int_anpm_outer<MVE_v8u16, MVE_v8f16>;
3849defm MVE_VCVTs32f32 : MVE_VCVT_fp_int_anpm_outer<MVE_v4s32, MVE_v4f32>;
3850defm MVE_VCVTu32f32 : MVE_VCVT_fp_int_anpm_outer<MVE_v4u32, MVE_v4f32>;
3851
3852class MVE_VCVT_fp_int<string suffix, bits<2> size, bit toint, bit unsigned,
3853                      list<dag> pattern=[]>
3854  : MVE_float<"vcvt", suffix, (outs MQPR:$Qd),
3855              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
3856  bits<4> Qd;
3857
3858  let Inst{28} = 0b1;
3859  let Inst{25-23} = 0b111;
3860  let Inst{22} = Qd{3};
3861  let Inst{21-20} = 0b11;
3862  let Inst{19-18} = size;
3863  let Inst{17-16} = 0b11;
3864  let Inst{15-13} = Qd{2-0};
3865  let Inst{12-9} = 0b0011;
3866  let Inst{8} = toint;
3867  let Inst{7} = unsigned;
3868  let Inst{4} = 0b0;
3869  let validForTailPredication = 1;
3870}
3871
3872multiclass MVE_VCVT_fp_int_m<MVEVectorVTInfo Dest, MVEVectorVTInfo Src,
3873                             SDNode unpred_op> {
3874  defvar Unsigned = !or(!eq(Dest.SuffixLetter,"u"), !eq(Src.SuffixLetter,"u"));
3875  defvar ToInt = !eq(Src.SuffixLetter,"f");
3876
3877  def "" : MVE_VCVT_fp_int<Dest.Suffix # "." # Src.Suffix, Dest.Size,
3878                           ToInt, Unsigned>;
3879  defvar Inst = !cast<Instruction>(NAME);
3880
3881  let Predicates = [HasMVEFloat] in {
3882    def : Pat<(Dest.Vec (unpred_op (Src.Vec MQPR:$src))),
3883              (Dest.Vec (Inst (Src.Vec MQPR:$src)))>;
3884    def : Pat<(Dest.Vec (int_arm_mve_vcvt_fp_int_predicated
3885                             (Src.Vec MQPR:$src), (i32 Unsigned),
3886                             (Src.Pred VCCR:$mask), (Dest.Vec MQPR:$inactive))),
3887              (Dest.Vec (Inst (Src.Vec MQPR:$src), ARMVCCThen,
3888                              (Src.Pred VCCR:$mask),
3889                              (Dest.Vec MQPR:$inactive)))>;
3890  }
3891}
3892// The unsuffixed VCVT for float->int implicitly rounds toward zero,
3893// which I reflect here in the llvm instruction names
3894defm MVE_VCVTs16f16z : MVE_VCVT_fp_int_m<MVE_v8s16, MVE_v8f16, fp_to_sint>;
3895defm MVE_VCVTu16f16z : MVE_VCVT_fp_int_m<MVE_v8u16, MVE_v8f16, fp_to_uint>;
3896defm MVE_VCVTs32f32z : MVE_VCVT_fp_int_m<MVE_v4s32, MVE_v4f32, fp_to_sint>;
3897defm MVE_VCVTu32f32z : MVE_VCVT_fp_int_m<MVE_v4u32, MVE_v4f32, fp_to_uint>;
3898// Whereas VCVT for int->float rounds to nearest
3899defm MVE_VCVTf16s16n : MVE_VCVT_fp_int_m<MVE_v8f16, MVE_v8s16, sint_to_fp>;
3900defm MVE_VCVTf16u16n : MVE_VCVT_fp_int_m<MVE_v8f16, MVE_v8u16, uint_to_fp>;
3901defm MVE_VCVTf32s32n : MVE_VCVT_fp_int_m<MVE_v4f32, MVE_v4s32, sint_to_fp>;
3902defm MVE_VCVTf32u32n : MVE_VCVT_fp_int_m<MVE_v4f32, MVE_v4u32, uint_to_fp>;
3903
3904class MVE_VABSNEG_fp<string iname, string suffix, bits<2> size, bit negate,
3905                   list<dag> pattern=[]>
3906  : MVE_float<iname, suffix, (outs MQPR:$Qd),
3907              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
3908  bits<4> Qd;
3909
3910  let Inst{28} = 0b1;
3911  let Inst{25-23} = 0b111;
3912  let Inst{22} = Qd{3};
3913  let Inst{21-20} = 0b11;
3914  let Inst{19-18} = size;
3915  let Inst{17-16} = 0b01;
3916  let Inst{15-13} = Qd{2-0};
3917  let Inst{11-8} = 0b0111;
3918  let Inst{7} = negate;
3919  let Inst{4} = 0b0;
3920  let validForTailPredication = 1;
3921}
3922
3923multiclass MVE_VABSNEG_fp_m<string iname, SDNode unpred_op, Intrinsic pred_int,
3924                            MVEVectorVTInfo VTI, bit opcode> {
3925  def "" : MVE_VABSNEG_fp<iname, VTI.Suffix, VTI.Size, opcode>;
3926  defvar Inst = !cast<Instruction>(NAME);
3927
3928  let Predicates = [HasMVEInt] in {
3929    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))), (VTI.Vec (Inst $v))>;
3930
3931    def : Pat<(VTI.Vec (pred_int  (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
3932                                  (VTI.Vec MQPR:$inactive))),
3933              (VTI.Vec (Inst $v, ARMVCCThen, $mask, $inactive))>;
3934  }
3935}
3936
3937defm MVE_VABSf16 : MVE_VABSNEG_fp_m<"vabs", fabs, int_arm_mve_abs_predicated,
3938                                    MVE_v8f16, 0>;
3939defm MVE_VABSf32 : MVE_VABSNEG_fp_m<"vabs", fabs, int_arm_mve_abs_predicated,
3940                                    MVE_v4f32, 0>;
3941defm MVE_VNEGf16 : MVE_VABSNEG_fp_m<"vneg", fneg, int_arm_mve_neg_predicated,
3942                                    MVE_v8f16, 1>;
3943defm MVE_VNEGf32 : MVE_VABSNEG_fp_m<"vneg", fneg, int_arm_mve_neg_predicated,
3944                                    MVE_v4f32, 1>;
3945
3946class MVE_VMAXMINNMA<string iname, string suffix, bit size, bit bit_12,
3947                     list<dag> pattern=[]>
3948  : MVE_f<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
3949          NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",
3950          pattern> {
3951  bits<4> Qd;
3952  bits<4> Qm;
3953
3954  let Inst{28} = size;
3955  let Inst{25-23} = 0b100;
3956  let Inst{22} = Qd{3};
3957  let Inst{21-16} = 0b111111;
3958  let Inst{15-13} = Qd{2-0};
3959  let Inst{12} = bit_12;
3960  let Inst{11-6} = 0b111010;
3961  let Inst{5} = Qm{3};
3962  let Inst{4} = 0b0;
3963  let Inst{3-1} = Qm{2-0};
3964  let Inst{0} = 0b1;
3965}
3966
3967multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,
3968                      SDNode unpred_op, Intrinsic pred_int,
3969                      bit bit_12> {
3970  def "" : MVE_VMAXMINNMA<iname, VTI.Suffix, VTI.Size{0}, bit_12>;
3971  defvar Inst = !cast<Instruction>(NAME);
3972
3973  let Predicates = [HasMVEInt] in {
3974    // Unpredicated v(max|min)nma
3975    def : Pat<(VTI.Vec (unpred_op (fabs (VTI.Vec MQPR:$Qd)),
3976                                  (fabs (VTI.Vec MQPR:$Qm)))),
3977              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
3978
3979    // Predicated v(max|min)nma
3980    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
3981                            (VTI.Pred VCCR:$mask))),
3982              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
3983                            ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
3984  }
3985}
3986
3987multiclass MVE_VMAXNMA<MVEVectorVTInfo VTI, bit bit_12>
3988  : MVE_VMAXMINNMA_m<"vmaxnma", VTI, fmaxnum, int_arm_mve_vmaxnma_predicated, bit_12>;
3989
3990defm MVE_VMAXNMAf32 : MVE_VMAXNMA<MVE_v4f32, 0b0>;
3991defm MVE_VMAXNMAf16 : MVE_VMAXNMA<MVE_v8f16, 0b0>;
3992
3993multiclass MVE_VMINNMA<MVEVectorVTInfo VTI, bit bit_12>
3994  : MVE_VMAXMINNMA_m<"vminnma", VTI, fminnum, int_arm_mve_vminnma_predicated, bit_12>;
3995
3996defm MVE_VMINNMAf32 : MVE_VMINNMA<MVE_v4f32, 0b1>;
3997defm MVE_VMINNMAf16 : MVE_VMINNMA<MVE_v8f16, 0b1>;
3998
3999// end of MVE Floating Point instructions
4000
4001// start of MVE compares
4002
4003class MVE_VCMPqq<string suffix, bit bit_28, bits<2> bits_21_20,
4004                 VCMPPredicateOperand predtype, list<dag> pattern=[]>
4005  : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, MQPR:$Qm, predtype:$fc),
4006           NoItinerary, "vcmp", suffix, "$fc, $Qn, $Qm", vpred_n, "", pattern> {
4007  // Base class for comparing two vector registers
4008  bits<3> fc;
4009  bits<4> Qn;
4010  bits<4> Qm;
4011
4012  let Inst{28} = bit_28;
4013  let Inst{25-22} = 0b1000;
4014  let Inst{21-20} = bits_21_20;
4015  let Inst{19-17} = Qn{2-0};
4016  let Inst{16-13} = 0b1000;
4017  let Inst{12} = fc{2};
4018  let Inst{11-8} = 0b1111;
4019  let Inst{7} = fc{0};
4020  let Inst{6} = 0b0;
4021  let Inst{5} = Qm{3};
4022  let Inst{4} = 0b0;
4023  let Inst{3-1} = Qm{2-0};
4024  let Inst{0} = fc{1};
4025
4026  let Constraints = "";
4027
4028  // We need a custom decoder method for these instructions because of
4029  // the output VCCR operand, which isn't encoded in the instruction
4030  // bits anywhere (there is only one choice for it) but has to be
4031  // included in the MC operands so that codegen will be able to track
4032  // its data flow between instructions, spill/reload it when
4033  // necessary, etc. There seems to be no way to get the Tablegen
4034  // decoder to emit an operand that isn't affected by any instruction
4035  // bit.
4036  let DecoderMethod = "DecodeMVEVCMP<false," # predtype.DecoderMethod # ">";
4037  let validForTailPredication = 1;
4038}
4039
4040class MVE_VCMPqqf<string suffix, bit size>
4041    : MVE_VCMPqq<suffix, size, 0b11, pred_basic_fp> {
4042  let Predicates = [HasMVEFloat];
4043}
4044
4045class MVE_VCMPqqi<string suffix, bits<2> size>
4046    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_i> {
4047  let Inst{12} = 0b0;
4048  let Inst{0} = 0b0;
4049}
4050
4051class MVE_VCMPqqu<string suffix, bits<2> size>
4052    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_u> {
4053  let Inst{12} = 0b0;
4054  let Inst{0} = 0b1;
4055}
4056
4057class MVE_VCMPqqs<string suffix, bits<2> size>
4058    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_s> {
4059  let Inst{12} = 0b1;
4060}
4061
4062def MVE_VCMPf32 : MVE_VCMPqqf<"f32", 0b0>;
4063def MVE_VCMPf16 : MVE_VCMPqqf<"f16", 0b1>;
4064
4065def MVE_VCMPi8  : MVE_VCMPqqi<"i8",  0b00>;
4066def MVE_VCMPi16 : MVE_VCMPqqi<"i16", 0b01>;
4067def MVE_VCMPi32 : MVE_VCMPqqi<"i32", 0b10>;
4068
4069def MVE_VCMPu8  : MVE_VCMPqqu<"u8",  0b00>;
4070def MVE_VCMPu16 : MVE_VCMPqqu<"u16", 0b01>;
4071def MVE_VCMPu32 : MVE_VCMPqqu<"u32", 0b10>;
4072
4073def MVE_VCMPs8  : MVE_VCMPqqs<"s8",  0b00>;
4074def MVE_VCMPs16 : MVE_VCMPqqs<"s16", 0b01>;
4075def MVE_VCMPs32 : MVE_VCMPqqs<"s32", 0b10>;
4076
4077class MVE_VCMPqr<string suffix, bit bit_28, bits<2> bits_21_20,
4078                 VCMPPredicateOperand predtype, list<dag> pattern=[]>
4079  : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, GPRwithZR:$Rm, predtype:$fc),
4080           NoItinerary, "vcmp", suffix, "$fc, $Qn, $Rm", vpred_n, "", pattern> {
4081  // Base class for comparing a vector register with a scalar
4082  bits<3> fc;
4083  bits<4> Qn;
4084  bits<4> Rm;
4085
4086  let Inst{28} = bit_28;
4087  let Inst{25-22} = 0b1000;
4088  let Inst{21-20} = bits_21_20;
4089  let Inst{19-17} = Qn{2-0};
4090  let Inst{16-13} = 0b1000;
4091  let Inst{12} = fc{2};
4092  let Inst{11-8} = 0b1111;
4093  let Inst{7} = fc{0};
4094  let Inst{6} = 0b1;
4095  let Inst{5} = fc{1};
4096  let Inst{4} = 0b0;
4097  let Inst{3-0} = Rm{3-0};
4098
4099  let Constraints = "";
4100  // Custom decoder method, for the same reason as MVE_VCMPqq
4101  let DecoderMethod = "DecodeMVEVCMP<true," # predtype.DecoderMethod # ">";
4102  let validForTailPredication = 1;
4103}
4104
4105class MVE_VCMPqrf<string suffix, bit size>
4106    : MVE_VCMPqr<suffix, size, 0b11, pred_basic_fp> {
4107  let Predicates = [HasMVEFloat];
4108}
4109
4110class MVE_VCMPqri<string suffix, bits<2> size>
4111    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_i> {
4112  let Inst{12} = 0b0;
4113  let Inst{5} = 0b0;
4114}
4115
4116class MVE_VCMPqru<string suffix, bits<2> size>
4117    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_u> {
4118  let Inst{12} = 0b0;
4119  let Inst{5} = 0b1;
4120}
4121
4122class MVE_VCMPqrs<string suffix, bits<2> size>
4123    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_s> {
4124  let Inst{12} = 0b1;
4125}
4126
4127def MVE_VCMPf32r : MVE_VCMPqrf<"f32", 0b0>;
4128def MVE_VCMPf16r : MVE_VCMPqrf<"f16", 0b1>;
4129
4130def MVE_VCMPi8r  : MVE_VCMPqri<"i8",  0b00>;
4131def MVE_VCMPi16r : MVE_VCMPqri<"i16", 0b01>;
4132def MVE_VCMPi32r : MVE_VCMPqri<"i32", 0b10>;
4133
4134def MVE_VCMPu8r  : MVE_VCMPqru<"u8",  0b00>;
4135def MVE_VCMPu16r : MVE_VCMPqru<"u16", 0b01>;
4136def MVE_VCMPu32r : MVE_VCMPqru<"u32", 0b10>;
4137
4138def MVE_VCMPs8r  : MVE_VCMPqrs<"s8",  0b00>;
4139def MVE_VCMPs16r : MVE_VCMPqrs<"s16", 0b01>;
4140def MVE_VCMPs32r : MVE_VCMPqrs<"s32", 0b10>;
4141
4142multiclass unpred_vcmp_z<string suffix, PatLeaf fc> {
4143  def i8  : Pat<(v16i1 (ARMvcmpz (v16i8 MQPR:$v1), fc)),
4144                (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc))>;
4145  def i16 : Pat<(v8i1 (ARMvcmpz (v8i16 MQPR:$v1), fc)),
4146                (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc))>;
4147  def i32 : Pat<(v4i1 (ARMvcmpz (v4i32 MQPR:$v1), fc)),
4148                (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc))>;
4149
4150  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmpz (v16i8 MQPR:$v1), fc)))),
4151            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
4152  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8i16 MQPR:$v1), fc)))),
4153            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
4154  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4i32 MQPR:$v1), fc)))),
4155            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
4156}
4157
4158multiclass unpred_vcmp_r<string suffix, PatLeaf fc> {
4159  def i8  : Pat<(v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc)),
4160                (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc))>;
4161  def i16 : Pat<(v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc)),
4162                (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc))>;
4163  def i32 : Pat<(v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc)),
4164                (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc))>;
4165
4166  def i8r  : Pat<(v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 (ARMvdup rGPR:$v2)), fc)),
4167                 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4168  def i16r : Pat<(v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 (ARMvdup rGPR:$v2)), fc)),
4169                 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4170  def i32r : Pat<(v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 (ARMvdup rGPR:$v2)), fc)),
4171                 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4172
4173  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc)))),
4174            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4175  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc)))),
4176            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4177  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc)))),
4178            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4179
4180  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 (ARMvdup rGPR:$v2)), fc)))),
4181            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4182  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 (ARMvdup rGPR:$v2)), fc)))),
4183            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4184  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 (ARMvdup rGPR:$v2)), fc)))),
4185            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4186}
4187
4188multiclass unpred_vcmpf_z<PatLeaf fc> {
4189  def f16 : Pat<(v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)),
4190                (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc))>;
4191  def f32 : Pat<(v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)),
4192                (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc))>;
4193
4194  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)))),
4195            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
4196  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)))),
4197            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
4198}
4199
4200multiclass unpred_vcmpf_r<int fc> {
4201  def : Pat<(v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc)),
4202            (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc))>;
4203  def : Pat<(v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc)),
4204            (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc))>;
4205
4206  def : Pat<(v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 (ARMvdup rGPR:$v2)), fc)),
4207            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4208  def : Pat<(v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 (ARMvdup rGPR:$v2)), fc)),
4209            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4210
4211  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc)))),
4212            (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4213  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc)))),
4214            (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4215
4216  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 (ARMvdup rGPR:$v2)), fc)))),
4217            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4218  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 (ARMvdup rGPR:$v2)), fc)))),
4219            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1))>;
4220}
4221
4222let Predicates = [HasMVEInt] in {
4223  defm MVE_VCEQZ  : unpred_vcmp_z<"i", ARMCCeq>;
4224  defm MVE_VCNEZ  : unpred_vcmp_z<"i", ARMCCne>;
4225  defm MVE_VCGEZ  : unpred_vcmp_z<"s", ARMCCge>;
4226  defm MVE_VCLTZ  : unpred_vcmp_z<"s", ARMCClt>;
4227  defm MVE_VCGTZ  : unpred_vcmp_z<"s", ARMCCgt>;
4228  defm MVE_VCLEZ  : unpred_vcmp_z<"s", ARMCCle>;
4229  defm MVE_VCGTUZ : unpred_vcmp_z<"u", ARMCChi>;
4230  defm MVE_VCGEUZ : unpred_vcmp_z<"u", ARMCChs>;
4231
4232  defm MVE_VCEQ   : unpred_vcmp_r<"i", ARMCCeq>;
4233  defm MVE_VCNE   : unpred_vcmp_r<"i", ARMCCne>;
4234  defm MVE_VCGE   : unpred_vcmp_r<"s", ARMCCge>;
4235  defm MVE_VCLT   : unpred_vcmp_r<"s", ARMCClt>;
4236  defm MVE_VCGT   : unpred_vcmp_r<"s", ARMCCgt>;
4237  defm MVE_VCLE   : unpred_vcmp_r<"s", ARMCCle>;
4238  defm MVE_VCGTU  : unpred_vcmp_r<"u", ARMCChi>;
4239  defm MVE_VCGEU  : unpred_vcmp_r<"u", ARMCChs>;
4240}
4241
4242let Predicates = [HasMVEFloat] in {
4243  defm MVE_VFCEQZ  : unpred_vcmpf_z<ARMCCeq>;
4244  defm MVE_VFCNEZ  : unpred_vcmpf_z<ARMCCne>;
4245  defm MVE_VFCGEZ  : unpred_vcmpf_z<ARMCCge>;
4246  defm MVE_VFCLTZ  : unpred_vcmpf_z<ARMCClt>;
4247  defm MVE_VFCGTZ  : unpred_vcmpf_z<ARMCCgt>;
4248  defm MVE_VFCLEZ  : unpred_vcmpf_z<ARMCCle>;
4249
4250  defm MVE_VFCEQ   : unpred_vcmpf_r<ARMCCeq>;
4251  defm MVE_VFCNE   : unpred_vcmpf_r<ARMCCne>;
4252  defm MVE_VFCGE   : unpred_vcmpf_r<ARMCCge>;
4253  defm MVE_VFCLT   : unpred_vcmpf_r<ARMCClt>;
4254  defm MVE_VFCGT   : unpred_vcmpf_r<ARMCCgt>;
4255  defm MVE_VFCLE   : unpred_vcmpf_r<ARMCCle>;
4256}
4257
4258
4259// Extra "worst case" and/or/xor patterns, going into and out of GRP
4260multiclass two_predops<SDPatternOperator opnode, Instruction insn> {
4261  def v16i1 : Pat<(v16i1 (opnode (v16i1 VCCR:$p1), (v16i1 VCCR:$p2))),
4262                  (v16i1 (COPY_TO_REGCLASS
4263                           (insn (i32 (COPY_TO_REGCLASS (v16i1 VCCR:$p1), rGPR)),
4264                                 (i32 (COPY_TO_REGCLASS (v16i1 VCCR:$p2), rGPR))),
4265                           VCCR))>;
4266  def v8i1  : Pat<(v8i1 (opnode (v8i1 VCCR:$p1), (v8i1 VCCR:$p2))),
4267                  (v8i1 (COPY_TO_REGCLASS
4268                          (insn (i32 (COPY_TO_REGCLASS (v8i1 VCCR:$p1), rGPR)),
4269                                (i32 (COPY_TO_REGCLASS (v8i1 VCCR:$p2), rGPR))),
4270                          VCCR))>;
4271  def v4i1  : Pat<(v4i1 (opnode (v4i1 VCCR:$p1), (v4i1 VCCR:$p2))),
4272                  (v4i1 (COPY_TO_REGCLASS
4273                          (insn (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p1), rGPR)),
4274                                (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p2), rGPR))),
4275                          VCCR))>;
4276}
4277
4278let Predicates = [HasMVEInt] in {
4279  defm POR    : two_predops<or,  t2ORRrr>;
4280  defm PAND   : two_predops<and, t2ANDrr>;
4281  defm PEOR   : two_predops<xor, t2EORrr>;
4282}
4283
4284// Occasionally we need to cast between a i32 and a boolean vector, for
4285// example when moving between rGPR and VPR.P0 as part of predicate vector
4286// shuffles. We also sometimes need to cast between different predicate
4287// vector types (v4i1<>v8i1, etc.) also as part of lowering vector shuffles.
4288def predicate_cast : SDNode<"ARMISD::PREDICATE_CAST", SDTUnaryOp>;
4289
4290let Predicates = [HasMVEInt] in {
4291  foreach VT = [ v4i1, v8i1, v16i1 ] in {
4292    def : Pat<(i32 (predicate_cast (VT VCCR:$src))),
4293              (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>;
4294    def : Pat<(VT  (predicate_cast (i32 VCCR:$src))),
4295              (VT  (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>;
4296
4297    foreach VT2 = [ v4i1, v8i1, v16i1 ] in
4298      def : Pat<(VT  (predicate_cast (VT2 VCCR:$src))),
4299                (VT  (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>;
4300  }
4301
4302  // Here we match the specific SDNode type 'ARMVectorRegCastImpl'
4303  // rather than the more general 'ARMVectorRegCast' which would also
4304  // match some bitconverts. If we use the latter in cases where the
4305  // input and output types are the same, the bitconvert gets elided
4306  // and we end up generating a nonsense match of nothing.
4307
4308  foreach VT = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in
4309    foreach VT2 = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in
4310      def : Pat<(VT (ARMVectorRegCastImpl (VT2 MQPR:$src))), (VT MQPR:$src)>;
4311}
4312
4313// end of MVE compares
4314
4315// start of MVE_qDest_qSrc
4316
4317class MVE_qDest_qSrc<string iname, string suffix, dag oops, dag iops,
4318                     string ops, vpred_ops vpred, string cstr,
4319                     list<dag> pattern=[]>
4320  : MVE_p<oops, iops, NoItinerary, iname, suffix,
4321          ops, vpred, cstr, pattern> {
4322  bits<4> Qd;
4323  bits<4> Qm;
4324
4325  let Inst{25-23} = 0b100;
4326  let Inst{22} = Qd{3};
4327  let Inst{15-13} = Qd{2-0};
4328  let Inst{11-9} = 0b111;
4329  let Inst{6} = 0b0;
4330  let Inst{5} = Qm{3};
4331  let Inst{4} = 0b0;
4332  let Inst{3-1} = Qm{2-0};
4333}
4334
4335class MVE_VQxDMLxDH<string iname, bit exch, bit round, bit subtract,
4336                    string suffix, bits<2> size, string cstr="", list<dag> pattern=[]>
4337  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4338                   (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4339                   vpred_n, "$Qd = $Qd_src"#cstr, pattern> {
4340  bits<4> Qn;
4341
4342  let Inst{28} = subtract;
4343  let Inst{21-20} = size;
4344  let Inst{19-17} = Qn{2-0};
4345  let Inst{16} = 0b0;
4346  let Inst{12} = exch;
4347  let Inst{8} = 0b0;
4348  let Inst{7} = Qn{3};
4349  let Inst{0} = round;
4350}
4351
4352multiclass MVE_VQxDMLxDH_p<string iname, bit exch, bit round, bit subtract,
4353                           MVEVectorVTInfo VTI> {
4354  def "": MVE_VQxDMLxDH<iname, exch, round, subtract, VTI.Suffix, VTI.Size,
4355                        !if(!eq(VTI.LaneBits, 32), ",@earlyclobber $Qd", "")>;
4356  defvar Inst = !cast<Instruction>(NAME);
4357  defvar ConstParams = (? (i32 exch), (i32 round), (i32 subtract));
4358  defvar unpred_intr = int_arm_mve_vqdmlad;
4359  defvar pred_intr = int_arm_mve_vqdmlad_predicated;
4360
4361  def : Pat<(VTI.Vec !con((unpred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4362                                       (VTI.Vec MQPR:$c)), ConstParams)),
4363            (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4364                           (VTI.Vec MQPR:$c)))>;
4365  def : Pat<(VTI.Vec !con((pred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4366                                     (VTI.Vec MQPR:$c)), ConstParams,
4367                          (? (VTI.Pred VCCR:$pred)))),
4368            (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4369                           (VTI.Vec MQPR:$c),
4370                           ARMVCCThen, (VTI.Pred VCCR:$pred)))>;
4371}
4372
4373multiclass MVE_VQxDMLxDH_multi<string iname, bit exch,
4374                               bit round, bit subtract> {
4375  defm s8  : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v16s8>;
4376  defm s16 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v8s16>;
4377  defm s32 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v4s32>;
4378}
4379
4380defm MVE_VQDMLADH   : MVE_VQxDMLxDH_multi<"vqdmladh",   0b0, 0b0, 0b0>;
4381defm MVE_VQDMLADHX  : MVE_VQxDMLxDH_multi<"vqdmladhx",  0b1, 0b0, 0b0>;
4382defm MVE_VQRDMLADH  : MVE_VQxDMLxDH_multi<"vqrdmladh",  0b0, 0b1, 0b0>;
4383defm MVE_VQRDMLADHX : MVE_VQxDMLxDH_multi<"vqrdmladhx", 0b1, 0b1, 0b0>;
4384defm MVE_VQDMLSDH   : MVE_VQxDMLxDH_multi<"vqdmlsdh",   0b0, 0b0, 0b1>;
4385defm MVE_VQDMLSDHX  : MVE_VQxDMLxDH_multi<"vqdmlsdhx",  0b1, 0b0, 0b1>;
4386defm MVE_VQRDMLSDH  : MVE_VQxDMLxDH_multi<"vqrdmlsdh",  0b0, 0b1, 0b1>;
4387defm MVE_VQRDMLSDHX : MVE_VQxDMLxDH_multi<"vqrdmlsdhx", 0b1, 0b1, 0b1>;
4388
4389class MVE_VCMUL<string iname, string suffix, bit size, string cstr="">
4390  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4391                   (ins MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
4392                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
4393  bits<4> Qn;
4394  bits<2> rot;
4395
4396  let Inst{28} = size;
4397  let Inst{21-20} = 0b11;
4398  let Inst{19-17} = Qn{2-0};
4399  let Inst{16} = 0b0;
4400  let Inst{12} = rot{1};
4401  let Inst{8} = 0b0;
4402  let Inst{7} = Qn{3};
4403  let Inst{0} = rot{0};
4404
4405  let Predicates = [HasMVEFloat];
4406}
4407
4408multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,
4409                       bit size, string cstr=""> {
4410  def "" : MVE_VCMUL<iname, VTI.Suffix, size, cstr>;
4411  defvar Inst = !cast<Instruction>(NAME);
4412
4413  let Predicates = [HasMVEFloat] in {
4414    def : Pat<(VTI.Vec (int_arm_mve_vcmulq
4415                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
4416              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4417                             imm:$rot))>;
4418
4419    def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated
4420                            imm:$rot, (VTI.Vec MQPR:$inactive),
4421                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4422                            (VTI.Pred VCCR:$mask))),
4423              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4424                             imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask),
4425                             (VTI.Vec MQPR:$inactive)))>;
4426
4427  }
4428}
4429
4430defm MVE_VCMULf16 : MVE_VCMUL_m<"vcmul", MVE_v8f16, 0b0>;
4431defm MVE_VCMULf32 : MVE_VCMUL_m<"vcmul", MVE_v4f32, 0b1, "@earlyclobber $Qd">;
4432
4433class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,
4434                bit T, string cstr, list<dag> pattern=[]>
4435  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4436                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4437                   vpred_r, cstr, pattern> {
4438  bits<4> Qd;
4439  bits<4> Qn;
4440  bits<4> Qm;
4441
4442  let Inst{28} = bit_28;
4443  let Inst{21-20} = bits_21_20;
4444  let Inst{19-17} = Qn{2-0};
4445  let Inst{16} = 0b1;
4446  let Inst{12} = T;
4447  let Inst{8} = 0b0;
4448  let Inst{7} = Qn{3};
4449  let Inst{0} = 0b0;
4450  let validForTailPredication = 1;
4451  let doubleWidthResult = 1;
4452}
4453
4454multiclass MVE_VMULL_m<MVEVectorVTInfo VTI,
4455                       SDNode unpred_op, Intrinsic pred_int,
4456                       bit Top, string cstr=""> {
4457  def "" : MVE_VMULL<"vmull" # !if(Top, "t", "b"), VTI.Suffix, VTI.Unsigned,
4458                     VTI.Size, Top, cstr>;
4459  defvar Inst = !cast<Instruction>(NAME);
4460
4461  let Predicates = [HasMVEInt] in {
4462    defvar uflag = !if(!eq(VTI.SuffixLetter, "p"), (?), (? (i32 VTI.Unsigned)));
4463
4464    // Unpredicated multiply
4465    def : Pat<(VTI.DblVec !con((unpred_op (VTI.Vec MQPR:$Qm),
4466                                          (VTI.Vec MQPR:$Qn)),
4467                               uflag, (? (i32 Top)))),
4468              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4469
4470    // Predicated multiply
4471    def : Pat<(VTI.DblVec !con((pred_int (VTI.Vec MQPR:$Qm),
4472                                         (VTI.Vec MQPR:$Qn)),
4473                               uflag, (? (i32 Top), (VTI.DblPred VCCR:$mask),
4474                                         (VTI.DblVec MQPR:$inactive)))),
4475              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4476                                ARMVCCThen, (VTI.DblPred VCCR:$mask),
4477                                (VTI.DblVec MQPR:$inactive)))>;
4478  }
4479}
4480
4481// For polynomial multiplies, the size bits take the unused value 0b11, and
4482// the unsigned bit switches to encoding the size.
4483
4484defm MVE_VMULLBs8  : MVE_VMULL_m<MVE_v16s8, int_arm_mve_vmull,
4485                                 int_arm_mve_mull_int_predicated, 0b0>;
4486defm MVE_VMULLTs8  : MVE_VMULL_m<MVE_v16s8, int_arm_mve_vmull,
4487                                 int_arm_mve_mull_int_predicated, 0b1>;
4488defm MVE_VMULLBs16 : MVE_VMULL_m<MVE_v8s16, int_arm_mve_vmull,
4489                                 int_arm_mve_mull_int_predicated, 0b0>;
4490defm MVE_VMULLTs16 : MVE_VMULL_m<MVE_v8s16, int_arm_mve_vmull,
4491                                 int_arm_mve_mull_int_predicated, 0b1>;
4492defm MVE_VMULLBs32 : MVE_VMULL_m<MVE_v4s32, int_arm_mve_vmull,
4493                                 int_arm_mve_mull_int_predicated, 0b0,
4494                                 "@earlyclobber $Qd">;
4495defm MVE_VMULLTs32 : MVE_VMULL_m<MVE_v4s32, int_arm_mve_vmull,
4496                                 int_arm_mve_mull_int_predicated, 0b1,
4497                                 "@earlyclobber $Qd">;
4498
4499defm MVE_VMULLBu8  : MVE_VMULL_m<MVE_v16u8, int_arm_mve_vmull,
4500                                 int_arm_mve_mull_int_predicated, 0b0>;
4501defm MVE_VMULLTu8  : MVE_VMULL_m<MVE_v16u8, int_arm_mve_vmull,
4502                                 int_arm_mve_mull_int_predicated, 0b1>;
4503defm MVE_VMULLBu16 : MVE_VMULL_m<MVE_v8u16, int_arm_mve_vmull,
4504                                 int_arm_mve_mull_int_predicated, 0b0>;
4505defm MVE_VMULLTu16 : MVE_VMULL_m<MVE_v8u16, int_arm_mve_vmull,
4506                                 int_arm_mve_mull_int_predicated, 0b1>;
4507defm MVE_VMULLBu32 : MVE_VMULL_m<MVE_v4u32, int_arm_mve_vmull,
4508                                 int_arm_mve_mull_int_predicated, 0b0,
4509                                 "@earlyclobber $Qd">;
4510defm MVE_VMULLTu32 : MVE_VMULL_m<MVE_v4u32, int_arm_mve_vmull,
4511                                 int_arm_mve_mull_int_predicated, 0b1,
4512                                 "@earlyclobber $Qd">;
4513
4514defm MVE_VMULLBp8  : MVE_VMULL_m<MVE_v16p8, int_arm_mve_vmull_poly,
4515                                 int_arm_mve_mull_poly_predicated, 0b0>;
4516defm MVE_VMULLTp8  : MVE_VMULL_m<MVE_v16p8, int_arm_mve_vmull_poly,
4517                                 int_arm_mve_mull_poly_predicated, 0b1>;
4518defm MVE_VMULLBp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,
4519                                 int_arm_mve_mull_poly_predicated, 0b0>;
4520defm MVE_VMULLTp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,
4521                                 int_arm_mve_mull_poly_predicated, 0b1>;
4522
4523let Predicates = [HasMVEInt] in {
4524  def : Pat<(v2i64 (ARMvmulls (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),
4525            (MVE_VMULLBs32 MQPR:$src1, MQPR:$src2)>;
4526  def : Pat<(v2i64 (ARMvmulls (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),
4527                              (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),
4528            (MVE_VMULLTs32 MQPR:$src1, MQPR:$src2)>;
4529
4530  def : Pat<(mul (sext_inreg (v4i32 MQPR:$src1), v4i16),
4531                 (sext_inreg (v4i32 MQPR:$src2), v4i16)),
4532            (MVE_VMULLBs16 MQPR:$src1, MQPR:$src2)>;
4533  def : Pat<(mul (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))), v4i16),
4534                 (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))), v4i16)),
4535            (MVE_VMULLTs16 MQPR:$src1, MQPR:$src2)>;
4536
4537  def : Pat<(mul (sext_inreg (v8i16 MQPR:$src1), v8i8),
4538                 (sext_inreg (v8i16 MQPR:$src2), v8i8)),
4539            (MVE_VMULLBs8 MQPR:$src1, MQPR:$src2)>;
4540  def : Pat<(mul (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), v8i8),
4541                 (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), v8i8)),
4542            (MVE_VMULLTs8 MQPR:$src1, MQPR:$src2)>;
4543
4544  def : Pat<(v2i64 (ARMvmullu (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),
4545            (MVE_VMULLBu32 MQPR:$src1, MQPR:$src2)>;
4546  def : Pat<(v2i64 (ARMvmullu (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),
4547                              (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),
4548            (MVE_VMULLTu32 MQPR:$src1, MQPR:$src2)>;
4549
4550  def : Pat<(mul (and (v4i32 MQPR:$src1), (v4i32 (ARMvmovImm (i32 0xCFF)))),
4551                 (and (v4i32 MQPR:$src2), (v4i32 (ARMvmovImm (i32 0xCFF))))),
4552            (MVE_VMULLBu16 MQPR:$src1, MQPR:$src2)>;
4553  def : Pat<(mul (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))),
4554                      (v4i32 (ARMvmovImm (i32 0xCFF)))),
4555                 (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))),
4556                      (v4i32 (ARMvmovImm (i32 0xCFF))))),
4557            (MVE_VMULLTu16 MQPR:$src1, MQPR:$src2)>;
4558
4559  def : Pat<(mul (ARMvbicImm (v8i16 MQPR:$src1), (i32 0xAFF)),
4560                 (ARMvbicImm (v8i16 MQPR:$src2), (i32 0xAFF))),
4561            (MVE_VMULLBu8 MQPR:$src1, MQPR:$src2)>;
4562  def : Pat<(mul (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), (i32 0xAFF)),
4563                 (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), (i32 0xAFF))),
4564            (MVE_VMULLTu8 MQPR:$src1, MQPR:$src2)>;
4565}
4566
4567class MVE_VxMULH<string iname, string suffix, bit U, bits<2> size, bit round,
4568                 list<dag> pattern=[]>
4569  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4570                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4571                   vpred_r, "", pattern> {
4572  bits<4> Qn;
4573
4574  let Inst{28} = U;
4575  let Inst{21-20} = size;
4576  let Inst{19-17} = Qn{2-0};
4577  let Inst{16} = 0b1;
4578  let Inst{12} = round;
4579  let Inst{8} = 0b0;
4580  let Inst{7} = Qn{3};
4581  let Inst{0} = 0b1;
4582}
4583
4584multiclass MVE_VxMULH_m<string iname, MVEVectorVTInfo VTI, SDNode unpred_op,
4585                        Intrinsic pred_int, bit round> {
4586  def "" : MVE_VxMULH<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, round>;
4587  defvar Inst = !cast<Instruction>(NAME);
4588
4589  let Predicates = [HasMVEInt] in {
4590    // Unpredicated multiply returning high bits
4591    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4592                            (i32 VTI.Unsigned))),
4593              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4594
4595    // Predicated multiply returning high bits
4596    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4597                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
4598                            (VTI.Vec MQPR:$inactive))),
4599              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4600                             ARMVCCThen, (VTI.Pred VCCR:$mask),
4601                             (VTI.Vec MQPR:$inactive)))>;
4602  }
4603}
4604
4605multiclass MVE_VMULT<string iname, MVEVectorVTInfo VTI, bit round>
4606  : MVE_VxMULH_m<iname, VTI, !if(round, int_arm_mve_vrmulh, int_arm_mve_vmulh),
4607                 !if(round, int_arm_mve_rmulh_predicated,
4608                            int_arm_mve_mulh_predicated),
4609                 round>;
4610
4611defm MVE_VMULHs8   : MVE_VMULT<"vmulh",  MVE_v16s8, 0b0>;
4612defm MVE_VMULHs16  : MVE_VMULT<"vmulh",  MVE_v8s16, 0b0>;
4613defm MVE_VMULHs32  : MVE_VMULT<"vmulh",  MVE_v4s32, 0b0>;
4614defm MVE_VMULHu8   : MVE_VMULT<"vmulh",  MVE_v16u8, 0b0>;
4615defm MVE_VMULHu16  : MVE_VMULT<"vmulh",  MVE_v8u16, 0b0>;
4616defm MVE_VMULHu32  : MVE_VMULT<"vmulh",  MVE_v4u32, 0b0>;
4617
4618defm MVE_VRMULHs8  : MVE_VMULT<"vrmulh", MVE_v16s8, 0b1>;
4619defm MVE_VRMULHs16 : MVE_VMULT<"vrmulh", MVE_v8s16, 0b1>;
4620defm MVE_VRMULHs32 : MVE_VMULT<"vrmulh", MVE_v4s32, 0b1>;
4621defm MVE_VRMULHu8  : MVE_VMULT<"vrmulh", MVE_v16u8, 0b1>;
4622defm MVE_VRMULHu16 : MVE_VMULT<"vrmulh", MVE_v8u16, 0b1>;
4623defm MVE_VRMULHu32 : MVE_VMULT<"vrmulh", MVE_v4u32, 0b1>;
4624
4625class MVE_VxMOVxN<string iname, string suffix, bit bit_28, bit bit_17,
4626                  bits<2> size, bit T, list<dag> pattern=[]>
4627  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4628                   (ins MQPR:$Qd_src, MQPR:$Qm), "$Qd, $Qm",
4629                   vpred_n, "$Qd = $Qd_src", pattern> {
4630
4631  let Inst{28} = bit_28;
4632  let Inst{21-20} = 0b11;
4633  let Inst{19-18} = size;
4634  let Inst{17} = bit_17;
4635  let Inst{16} = 0b1;
4636  let Inst{12} = T;
4637  let Inst{8} = 0b0;
4638  let Inst{7} = !if(!eq(bit_17, 0), 1, 0);
4639  let Inst{0} = 0b1;
4640  let validForTailPredication = 1;
4641  let retainsPreviousHalfElement = 1;
4642}
4643
4644multiclass MVE_VxMOVxN_halves<string iname, string suffix,
4645                              bit bit_28, bit bit_17, bits<2> size> {
4646  def bh : MVE_VxMOVxN<iname # "b", suffix, bit_28, bit_17, size, 0b0>;
4647  def th : MVE_VxMOVxN<iname # "t", suffix, bit_28, bit_17, size, 0b1>;
4648}
4649
4650defm MVE_VMOVNi16   : MVE_VxMOVxN_halves<"vmovn",   "i16", 0b1, 0b0, 0b00>;
4651defm MVE_VMOVNi32   : MVE_VxMOVxN_halves<"vmovn",   "i32", 0b1, 0b0, 0b01>;
4652defm MVE_VQMOVNs16  : MVE_VxMOVxN_halves<"vqmovn",  "s16", 0b0, 0b1, 0b00>;
4653defm MVE_VQMOVNs32  : MVE_VxMOVxN_halves<"vqmovn",  "s32", 0b0, 0b1, 0b01>;
4654defm MVE_VQMOVNu16  : MVE_VxMOVxN_halves<"vqmovn",  "u16", 0b1, 0b1, 0b00>;
4655defm MVE_VQMOVNu32  : MVE_VxMOVxN_halves<"vqmovn",  "u32", 0b1, 0b1, 0b01>;
4656defm MVE_VQMOVUNs16 : MVE_VxMOVxN_halves<"vqmovun", "s16", 0b0, 0b0, 0b00>;
4657defm MVE_VQMOVUNs32 : MVE_VxMOVxN_halves<"vqmovun", "s32", 0b0, 0b0, 0b01>;
4658
4659def MVEvmovn       : SDNode<"ARMISD::VMOVN", SDTARMVEXT>;
4660
4661multiclass MVE_VMOVN_p<Instruction Inst, bit top,
4662                       MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4663  // Match the most obvious MVEvmovn(a,b,t), which overwrites the odd or even
4664  // lanes of a (depending on t) with the even lanes of b.
4665  def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qd_src),
4666                               (VTI.Vec MQPR:$Qm), (i32 top))),
4667            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4668
4669  if !eq(top, 0) then {
4670    // If we see MVEvmovn(a,ARMvrev(b),1), that wants to overwrite the odd
4671    // lanes of a with the odd lanes of b. In other words, the lanes we're
4672    // _keeping_ from a are the even ones. So we can flip it round and say that
4673    // this is the same as overwriting the even lanes of b with the even lanes
4674    // of a, i.e. it's a VMOVNB with the operands reversed.
4675    defvar vrev = !cast<SDNode>("ARMvrev" # InVTI.LaneBits);
4676    def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qm),
4677                                 (VTI.Vec (vrev MQPR:$Qd_src)), (i32 1))),
4678              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4679  }
4680
4681  // Match the IR intrinsic for a predicated VMOVN. This regards the Qm input
4682  // as having wider lanes that we're narrowing, instead of already-narrow
4683  // lanes that we're taking every other one of.
4684  def : Pat<(VTI.Vec (int_arm_mve_vmovn_predicated (VTI.Vec MQPR:$Qd_src),
4685                                  (InVTI.Vec MQPR:$Qm), (i32 top),
4686                                  (InVTI.Pred VCCR:$pred))),
4687            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4688                              (InVTI.Vec MQPR:$Qm),
4689                              ARMVCCThen, (InVTI.Pred VCCR:$pred)))>;
4690}
4691
4692defm : MVE_VMOVN_p<MVE_VMOVNi32bh, 0, MVE_v8i16, MVE_v4i32>;
4693defm : MVE_VMOVN_p<MVE_VMOVNi32th, 1, MVE_v8i16, MVE_v4i32>;
4694defm : MVE_VMOVN_p<MVE_VMOVNi16bh, 0, MVE_v16i8, MVE_v8i16>;
4695defm : MVE_VMOVN_p<MVE_VMOVNi16th, 1, MVE_v16i8, MVE_v8i16>;
4696
4697multiclass MVE_VQMOVN_p<Instruction Inst, bit outU, bit inU, bit top,
4698                        MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4699  def : Pat<(VTI.Vec (int_arm_mve_vqmovn (VTI.Vec MQPR:$Qd_src),
4700                                  (InVTI.Vec MQPR:$Qm),
4701                                  (i32 outU), (i32 inU), (i32 top))),
4702            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4703                              (InVTI.Vec MQPR:$Qm)))>;
4704
4705  def : Pat<(VTI.Vec (int_arm_mve_vqmovn_predicated (VTI.Vec MQPR:$Qd_src),
4706                                  (InVTI.Vec MQPR:$Qm),
4707                                  (i32 outU), (i32 inU), (i32 top),
4708                                  (InVTI.Pred VCCR:$pred))),
4709            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4710                              (InVTI.Vec MQPR:$Qm),
4711                              ARMVCCThen, (InVTI.Pred VCCR:$pred)))>;
4712}
4713
4714defm : MVE_VQMOVN_p<MVE_VQMOVNs32bh,  0, 0, 0, MVE_v8i16, MVE_v4i32>;
4715defm : MVE_VQMOVN_p<MVE_VQMOVNs32th,  0, 0, 1, MVE_v8i16, MVE_v4i32>;
4716defm : MVE_VQMOVN_p<MVE_VQMOVNs16bh,  0, 0, 0, MVE_v16i8, MVE_v8i16>;
4717defm : MVE_VQMOVN_p<MVE_VQMOVNs16th,  0, 0, 1, MVE_v16i8, MVE_v8i16>;
4718defm : MVE_VQMOVN_p<MVE_VQMOVNu32bh,  1, 1, 0, MVE_v8i16, MVE_v4i32>;
4719defm : MVE_VQMOVN_p<MVE_VQMOVNu32th,  1, 1, 1, MVE_v8i16, MVE_v4i32>;
4720defm : MVE_VQMOVN_p<MVE_VQMOVNu16bh,  1, 1, 0, MVE_v16i8, MVE_v8i16>;
4721defm : MVE_VQMOVN_p<MVE_VQMOVNu16th,  1, 1, 1, MVE_v16i8, MVE_v8i16>;
4722defm : MVE_VQMOVN_p<MVE_VQMOVUNs32bh, 1, 0, 0, MVE_v8i16, MVE_v4i32>;
4723defm : MVE_VQMOVN_p<MVE_VQMOVUNs32th, 1, 0, 1, MVE_v8i16, MVE_v4i32>;
4724defm : MVE_VQMOVN_p<MVE_VQMOVUNs16bh, 1, 0, 0, MVE_v16i8, MVE_v8i16>;
4725defm : MVE_VQMOVN_p<MVE_VQMOVUNs16th, 1, 0, 1, MVE_v16i8, MVE_v8i16>;
4726
4727def SDTARMVMOVNQ : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 1>,
4728                                        SDTCisVec<2>, SDTCisVT<3, i32>]>;
4729def MVEvqmovns   : SDNode<"ARMISD::VQMOVNs", SDTARMVMOVNQ>;
4730def MVEvqmovnu   : SDNode<"ARMISD::VQMOVNu", SDTARMVMOVNQ>;
4731
4732let Predicates = [HasMVEInt] in {
4733  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 0))),
4734            (v8i16 (MVE_VQMOVNs32bh (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4735  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 1))),
4736            (v8i16 (MVE_VQMOVNs32th (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4737  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 0))),
4738            (v16i8 (MVE_VQMOVNs16bh (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4739  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 1))),
4740            (v16i8 (MVE_VQMOVNs16th (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4741
4742  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 0))),
4743            (v8i16 (MVE_VQMOVNu32bh (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4744  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 1))),
4745            (v8i16 (MVE_VQMOVNu32th (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4746  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 0))),
4747            (v16i8 (MVE_VQMOVNu16bh (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4748  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 1))),
4749            (v16i8 (MVE_VQMOVNu16th (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4750
4751  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshrsImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 0))),
4752            (v8i16 (MVE_VQSHRNbhs32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4753  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshrsImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 0))),
4754            (v16i8 (MVE_VQSHRNbhs16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4755  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshrsImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 1))),
4756            (v8i16 (MVE_VQSHRNths32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4757  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshrsImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 1))),
4758            (v16i8 (MVE_VQSHRNths16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4759
4760  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshruImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 0))),
4761            (v8i16 (MVE_VQSHRNbhu32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4762  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshruImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 0))),
4763            (v16i8 (MVE_VQSHRNbhu16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4764  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshruImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 1))),
4765            (v8i16 (MVE_VQSHRNthu32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4766  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshruImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 1))),
4767            (v16i8 (MVE_VQSHRNthu16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4768}
4769
4770class MVE_VCVT_ff<string iname, string suffix, bit op, bit T,
4771                  dag iops_extra, vpred_ops vpred, string cstr>
4772  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4773                   !con(iops_extra, (ins MQPR:$Qm)), "$Qd, $Qm",
4774                   vpred, cstr, []> {
4775  let Inst{28} = op;
4776  let Inst{21-16} = 0b111111;
4777  let Inst{12} = T;
4778  let Inst{8-7} = 0b00;
4779  let Inst{0} = 0b1;
4780
4781  let Predicates = [HasMVEFloat];
4782  let retainsPreviousHalfElement = 1;
4783}
4784
4785def SDTARMVCVTL    : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
4786                                         SDTCisVT<2, i32>]>;
4787def MVEvcvtn       : SDNode<"ARMISD::VCVTN", SDTARMVMOVNQ>;
4788def MVEvcvtl       : SDNode<"ARMISD::VCVTL", SDTARMVCVTL>;
4789
4790multiclass MVE_VCVT_f2h_m<string iname, int half> {
4791  def "": MVE_VCVT_ff<iname, "f16.f32", 0b0, half,
4792                      (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
4793  defvar Inst = !cast<Instruction>(NAME);
4794
4795  let Predicates = [HasMVEFloat] in {
4796    def : Pat<(v8f16 (int_arm_mve_vcvt_narrow
4797                         (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),
4798              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;
4799    def : Pat<(v8f16 (int_arm_mve_vcvt_narrow_predicated
4800                         (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half),
4801                         (v4i1 VCCR:$mask))),
4802              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm),
4803                           ARMVCCThen, (v4i1 VCCR:$mask)))>;
4804
4805    def : Pat<(v8f16 (MVEvcvtn (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),
4806              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;
4807  }
4808}
4809
4810multiclass MVE_VCVT_h2f_m<string iname, int half> {
4811  def "": MVE_VCVT_ff<iname, "f32.f16", 0b1, half, (ins), vpred_r, "">;
4812  defvar Inst = !cast<Instruction>(NAME);
4813
4814  let Predicates = [HasMVEFloat] in {
4815    def : Pat<(v4f32 (int_arm_mve_vcvt_widen (v8f16 MQPR:$Qm), (i32 half))),
4816              (v4f32 (Inst (v8f16 MQPR:$Qm)))>;
4817    def : Pat<(v4f32 (int_arm_mve_vcvt_widen_predicated
4818                         (v4f32 MQPR:$inactive), (v8f16 MQPR:$Qm), (i32 half),
4819                         (v4i1 VCCR:$mask))),
4820              (v4f32 (Inst (v8f16 MQPR:$Qm), ARMVCCThen,
4821                           (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive)))>;
4822
4823    def : Pat<(v4f32 (MVEvcvtl (v8f16 MQPR:$Qm), (i32 half))),
4824              (v4f32 (Inst (v8f16 MQPR:$Qm)))>;
4825  }
4826}
4827
4828defm MVE_VCVTf16f32bh : MVE_VCVT_f2h_m<"vcvtb", 0b0>;
4829defm MVE_VCVTf16f32th : MVE_VCVT_f2h_m<"vcvtt", 0b1>;
4830defm MVE_VCVTf32f16bh : MVE_VCVT_h2f_m<"vcvtb", 0b0>;
4831defm MVE_VCVTf32f16th : MVE_VCVT_h2f_m<"vcvtt", 0b1>;
4832
4833class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
4834                 string cstr="">
4835  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4836                   (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
4837                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
4838  bits<4> Qn;
4839  bit rot;
4840
4841  let Inst{28} = halve;
4842  let Inst{21-20} = size;
4843  let Inst{19-17} = Qn{2-0};
4844  let Inst{16} = 0b0;
4845  let Inst{12} = rot;
4846  let Inst{8} = 0b1;
4847  let Inst{7} = Qn{3};
4848  let Inst{0} = 0b0;
4849}
4850
4851multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,
4852                        bit halve, string cstr=""> {
4853  def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;
4854  defvar Inst = !cast<Instruction>(NAME);
4855
4856  let Predicates = [HasMVEInt] in {
4857    def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,
4858                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
4859              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4860                             imm:$rot))>;
4861
4862    def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,
4863                            imm:$rot, (VTI.Vec MQPR:$inactive),
4864                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4865                            (VTI.Pred VCCR:$mask))),
4866              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4867                             imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask),
4868                             (VTI.Vec MQPR:$inactive)))>;
4869
4870  }
4871}
4872
4873defm MVE_VCADDi8   : MVE_VxCADD_m<"vcadd", MVE_v16i8, 0b1>;
4874defm MVE_VCADDi16  : MVE_VxCADD_m<"vcadd", MVE_v8i16, 0b1>;
4875defm MVE_VCADDi32  : MVE_VxCADD_m<"vcadd", MVE_v4i32, 0b1, "@earlyclobber $Qd">;
4876
4877defm MVE_VHCADDs8  : MVE_VxCADD_m<"vhcadd", MVE_v16s8, 0b0>;
4878defm MVE_VHCADDs16 : MVE_VxCADD_m<"vhcadd", MVE_v8s16, 0b0>;
4879defm MVE_VHCADDs32 : MVE_VxCADD_m<"vhcadd", MVE_v4s32, 0b0, "@earlyclobber $Qd">;
4880
4881class MVE_VADCSBC<string iname, bit I, bit subtract,
4882                  dag carryin, list<dag> pattern=[]>
4883  : MVE_qDest_qSrc<iname, "i32", (outs MQPR:$Qd, cl_FPSCR_NZCV:$carryout),
4884                   !con((ins MQPR:$Qn, MQPR:$Qm), carryin),
4885                   "$Qd, $Qn, $Qm", vpred_r, "", pattern> {
4886  bits<4> Qn;
4887
4888  let Inst{28} = subtract;
4889  let Inst{21-20} = 0b11;
4890  let Inst{19-17} = Qn{2-0};
4891  let Inst{16} = 0b0;
4892  let Inst{12} = I;
4893  let Inst{8} = 0b1;
4894  let Inst{7} = Qn{3};
4895  let Inst{0} = 0b0;
4896
4897  // Custom decoder method in order to add the FPSCR operand(s), which
4898  // Tablegen won't do right
4899  let DecoderMethod = "DecodeMVEVADCInstruction";
4900}
4901
4902def MVE_VADC  : MVE_VADCSBC<"vadc",  0b0, 0b0, (ins cl_FPSCR_NZCV:$carryin)>;
4903def MVE_VADCI : MVE_VADCSBC<"vadci", 0b1, 0b0, (ins)>;
4904
4905def MVE_VSBC  : MVE_VADCSBC<"vsbc",  0b0, 0b1, (ins cl_FPSCR_NZCV:$carryin)>;
4906def MVE_VSBCI : MVE_VADCSBC<"vsbci", 0b1, 0b1, (ins)>;
4907
4908class MVE_VQDMULL<string iname, string suffix, bit size, bit T,
4909                  string cstr="", list<dag> pattern=[]>
4910  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4911                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4912                   vpred_r, cstr, pattern> {
4913  bits<4> Qn;
4914
4915  let Inst{28} = size;
4916  let Inst{21-20} = 0b11;
4917  let Inst{19-17} = Qn{2-0};
4918  let Inst{16} = 0b0;
4919  let Inst{12} = T;
4920  let Inst{8} = 0b1;
4921  let Inst{7} = Qn{3};
4922  let Inst{0} = 0b1;
4923  let validForTailPredication = 1;
4924  let doubleWidthResult = 1;
4925}
4926
4927multiclass MVE_VQDMULL_m<string iname, MVEVectorVTInfo VTI, bit size, bit T,
4928                         string cstr> {
4929  def "" : MVE_VQDMULL<iname, VTI.Suffix, size, T, cstr>;
4930  defvar Inst = !cast<Instruction>(NAME);
4931
4932  let Predicates = [HasMVEInt] in {
4933    // Unpredicated saturating multiply
4934    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
4935                                               (VTI.Vec MQPR:$Qn), (i32 T))),
4936              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4937    // Predicated saturating multiply
4938    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
4939                                    (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4940                                    (i32 T), (VTI.DblPred VCCR:$mask),
4941                                    (VTI.DblVec MQPR:$inactive))),
4942              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4943                                ARMVCCThen, (VTI.DblPred VCCR:$mask),
4944                                (VTI.DblVec MQPR:$inactive)))>;
4945  }
4946}
4947
4948multiclass MVE_VQDMULL_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
4949  defm bh : MVE_VQDMULL_m<"vqdmullb", VTI, size, 0b0, cstr>;
4950  defm th : MVE_VQDMULL_m<"vqdmullt", VTI, size, 0b1, cstr>;
4951}
4952
4953defm MVE_VQDMULLs16 : MVE_VQDMULL_halves<MVE_v8s16, 0b0>;
4954defm MVE_VQDMULLs32 : MVE_VQDMULL_halves<MVE_v4s32, 0b1, "@earlyclobber $Qd">;
4955
4956// end of mve_qDest_qSrc
4957
4958// start of mve_qDest_rSrc
4959
4960class MVE_qr_base<dag oops, dag iops, InstrItinClass itin, string iname,
4961                  string suffix, string ops, vpred_ops vpred, string cstr,
4962                  list<dag> pattern=[]>
4963   : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
4964  bits<4> Qd;
4965  bits<4> Qn;
4966  bits<4> Rm;
4967
4968  let Inst{25-23} = 0b100;
4969  let Inst{22} = Qd{3};
4970  let Inst{19-17} = Qn{2-0};
4971  let Inst{15-13} = Qd{2-0};
4972  let Inst{11-9} = 0b111;
4973  let Inst{7} = Qn{3};
4974  let Inst{6} = 0b1;
4975  let Inst{4} = 0b0;
4976  let Inst{3-0} = Rm{3-0};
4977}
4978
4979class MVE_qDest_rSrc<string iname, string suffix, string cstr="", list<dag> pattern=[]>
4980  : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qn, rGPR:$Rm),
4981          NoItinerary, iname, suffix, "$Qd, $Qn, $Rm", vpred_r, cstr,
4982           pattern>;
4983
4984class MVE_qDestSrc_rSrc<string iname, string suffix, list<dag> pattern=[]>
4985  : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qn, rGPR:$Rm),
4986          NoItinerary, iname, suffix, "$Qd, $Qn, $Rm", vpred_n, "$Qd = $Qd_src",
4987           pattern>;
4988
4989class MVE_qDest_single_rSrc<string iname, string suffix, list<dag> pattern=[]>
4990  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, rGPR:$Rm), NoItinerary, iname,
4991          suffix, "$Qd, $Rm", vpred_n, "$Qd = $Qd_src", pattern> {
4992  bits<4> Qd;
4993  bits<4> Rm;
4994
4995  let Inst{22} = Qd{3};
4996  let Inst{15-13} = Qd{2-0};
4997  let Inst{3-0} = Rm{3-0};
4998}
4999
5000// Patterns for vector-scalar instructions with integer operands
5001multiclass MVE_vec_scalar_int_pat_m<Instruction inst, MVEVectorVTInfo VTI,
5002                                    SDNode unpred_op, SDNode pred_op,
5003                                    bit unpred_has_sign = 0,
5004                                    bit pred_has_sign = 0> {
5005  defvar UnpredSign = !if(unpred_has_sign, (? (i32 VTI.Unsigned)), (?));
5006  defvar PredSign = !if(pred_has_sign, (? (i32 VTI.Unsigned)), (?));
5007
5008  let Predicates = [HasMVEInt] in {
5009    // Unpredicated version
5010    def : Pat<(VTI.Vec !con((unpred_op (VTI.Vec MQPR:$Qm),
5011                                       (VTI.Vec (ARMvdup rGPR:$val))),
5012                            UnpredSign)),
5013              (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5014    // Predicated version
5015    def : Pat<(VTI.Vec !con((pred_op (VTI.Vec MQPR:$Qm),
5016                                     (VTI.Vec (ARMvdup rGPR:$val))),
5017                            PredSign,
5018                            (pred_op (VTI.Pred VCCR:$mask),
5019                                     (VTI.Vec MQPR:$inactive)))),
5020              (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5021                             ARMVCCThen, (VTI.Pred VCCR:$mask),
5022                             (VTI.Vec MQPR:$inactive)))>;
5023  }
5024}
5025
5026// Patterns for vector-scalar instructions with FP operands
5027multiclass MVE_vec_scalar_fp_pat_m<SDNode unpred_op, Intrinsic pred_int,
5028                                   Instruction instr_f16,
5029                                   Instruction instr_f32> {
5030  let Predicates = [HasMVEFloat] in {
5031    // Unpredicated F16
5032    def : Pat<(v8f16 (unpred_op (v8f16 MQPR:$Qm), (v8f16 (ARMvdup rGPR:$val)))),
5033              (v8f16 (instr_f16 (v8f16 MQPR:$Qm), (i32 rGPR:$val)))>;
5034    // Unpredicated F32
5035    def : Pat<(v4f32 (unpred_op (v4f32 MQPR:$Qm), (v4f32 (ARMvdup rGPR:$val)))),
5036              (v4f32 (instr_f32 (v4f32 MQPR:$Qm), (i32 rGPR:$val)))>;
5037    // Predicated F16
5038    def : Pat<(v8f16 (pred_int (v8f16 MQPR:$Qm), (v8f16 (ARMvdup rGPR:$val)),
5039                               (v8i1 VCCR:$mask), (v8f16 MQPR:$inactive))),
5040              (v8f16 (instr_f16 (v8f16 MQPR:$Qm), (i32 rGPR:$val),
5041                                ARMVCCThen, (v8i1 VCCR:$mask),
5042                                (v8f16 MQPR:$inactive)))>;
5043    // Predicated F32
5044    def : Pat<(v4f32 (pred_int (v4f32 MQPR:$Qm), (v4f32 (ARMvdup rGPR:$val)),
5045                               (v4i1 VCCR:$mask), (v4f32 MQPR:$inactive))),
5046              (v4f32 (instr_f32 (v4f32 MQPR:$Qm), (i32 rGPR:$val),
5047                                ARMVCCThen, (v4i1 VCCR:$mask),
5048                                (v4f32 MQPR:$inactive)))>;
5049  }
5050}
5051
5052class MVE_VADDSUB_qr<string iname, string suffix, bits<2> size,
5053                     bit bit_5, bit bit_12, bit bit_16, bit bit_28>
5054  : MVE_qDest_rSrc<iname, suffix, ""> {
5055
5056  let Inst{28} = bit_28;
5057  let Inst{21-20} = size;
5058  let Inst{16} = bit_16;
5059  let Inst{12} = bit_12;
5060  let Inst{8} = 0b1;
5061  let Inst{5} = bit_5;
5062  let validForTailPredication = 1;
5063}
5064
5065// Vector-scalar add/sub
5066multiclass MVE_VADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5067                            SDNode unpred_op, Intrinsic pred_int> {
5068  def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b0, subtract, 0b1, 0b0>;
5069  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
5070                                  unpred_op, pred_int>;
5071}
5072
5073multiclass MVE_VADD_qr_m<MVEVectorVTInfo VTI>
5074  : MVE_VADDSUB_qr_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
5075
5076multiclass MVE_VSUB_qr_m<MVEVectorVTInfo VTI>
5077  : MVE_VADDSUB_qr_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
5078
5079defm MVE_VADD_qr_i8  : MVE_VADD_qr_m<MVE_v16i8>;
5080defm MVE_VADD_qr_i16 : MVE_VADD_qr_m<MVE_v8i16>;
5081defm MVE_VADD_qr_i32 : MVE_VADD_qr_m<MVE_v4i32>;
5082
5083defm MVE_VSUB_qr_i8  : MVE_VSUB_qr_m<MVE_v16i8>;
5084defm MVE_VSUB_qr_i16 : MVE_VSUB_qr_m<MVE_v8i16>;
5085defm MVE_VSUB_qr_i32 : MVE_VSUB_qr_m<MVE_v4i32>;
5086
5087// Vector-scalar saturating add/sub
5088multiclass MVE_VQADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5089                             SDNode unpred_op_s, SDNode unpred_op_u,
5090                             Intrinsic pred_int> {
5091  def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b1, subtract,
5092                          0b0, VTI.Unsigned>;
5093  defvar unpred_op = !if(VTI.Unsigned, unpred_op_u, unpred_op_s);
5094  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
5095                                  unpred_op, pred_int, 0, 1>;
5096}
5097
5098multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI>
5099  : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, saddsat, uaddsat,
5100                     int_arm_mve_qadd_predicated>;
5101
5102multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI>
5103  : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, ssubsat, usubsat,
5104                     int_arm_mve_qsub_predicated>;
5105
5106defm MVE_VQADD_qr_s8  : MVE_VQADD_qr_m<MVE_v16s8>;
5107defm MVE_VQADD_qr_s16 : MVE_VQADD_qr_m<MVE_v8s16>;
5108defm MVE_VQADD_qr_s32 : MVE_VQADD_qr_m<MVE_v4s32>;
5109defm MVE_VQADD_qr_u8  : MVE_VQADD_qr_m<MVE_v16u8>;
5110defm MVE_VQADD_qr_u16 : MVE_VQADD_qr_m<MVE_v8u16>;
5111defm MVE_VQADD_qr_u32 : MVE_VQADD_qr_m<MVE_v4u32>;
5112
5113defm MVE_VQSUB_qr_s8  : MVE_VQSUB_qr_m<MVE_v16s8>;
5114defm MVE_VQSUB_qr_s16 : MVE_VQSUB_qr_m<MVE_v8s16>;
5115defm MVE_VQSUB_qr_s32 : MVE_VQSUB_qr_m<MVE_v4s32>;
5116defm MVE_VQSUB_qr_u8  : MVE_VQSUB_qr_m<MVE_v16u8>;
5117defm MVE_VQSUB_qr_u16 : MVE_VQSUB_qr_m<MVE_v8u16>;
5118defm MVE_VQSUB_qr_u32 : MVE_VQSUB_qr_m<MVE_v4u32>;
5119
5120class MVE_VQDMULL_qr<string iname, string suffix, bit size,
5121                     bit T, string cstr="", list<dag> pattern=[]>
5122  : MVE_qDest_rSrc<iname, suffix, cstr, pattern> {
5123
5124  let Inst{28} = size;
5125  let Inst{21-20} = 0b11;
5126  let Inst{16} = 0b0;
5127  let Inst{12} = T;
5128  let Inst{8} = 0b1;
5129  let Inst{5} = 0b1;
5130  let validForTailPredication = 1;
5131  let doubleWidthResult = 1;
5132}
5133
5134multiclass MVE_VQDMULL_qr_m<string iname, MVEVectorVTInfo VTI, bit size,
5135                            bit T, string cstr> {
5136  def "" : MVE_VQDMULL_qr<iname, VTI.Suffix, size, T, cstr>;
5137  defvar Inst = !cast<Instruction>(NAME);
5138
5139  let Predicates = [HasMVEInt] in {
5140    // Unpredicated saturating multiply
5141    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5142                                               (VTI.Vec (ARMvdup rGPR:$val)),
5143                                               (i32 T))),
5144              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5145    // Predicated saturating multiply
5146    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5147                                    (VTI.Vec MQPR:$Qm),
5148                                    (VTI.Vec (ARMvdup rGPR:$val)),
5149                                    (i32 T),
5150                                    (VTI.DblPred VCCR:$mask),
5151                                    (VTI.DblVec MQPR:$inactive))),
5152              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5153                             ARMVCCThen, (VTI.DblPred VCCR:$mask),
5154                             (VTI.DblVec MQPR:$inactive)))>;
5155  }
5156}
5157
5158multiclass MVE_VQDMULL_qr_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5159  defm bh : MVE_VQDMULL_qr_m<"vqdmullb", VTI, size, 0b0, cstr>;
5160  defm th : MVE_VQDMULL_qr_m<"vqdmullt", VTI, size, 0b1, cstr>;
5161}
5162
5163defm MVE_VQDMULL_qr_s16 : MVE_VQDMULL_qr_halves<MVE_v8s16, 0b0>;
5164defm MVE_VQDMULL_qr_s32 : MVE_VQDMULL_qr_halves<MVE_v4s32, 0b1, "@earlyclobber $Qd">;
5165
5166class MVE_VxADDSUB_qr<string iname, string suffix,
5167                      bit bit_28, bits<2> bits_21_20, bit subtract,
5168                      list<dag> pattern=[]>
5169  : MVE_qDest_rSrc<iname, suffix, "", pattern> {
5170
5171  let Inst{28} = bit_28;
5172  let Inst{21-20} = bits_21_20;
5173  let Inst{16} = 0b0;
5174  let Inst{12} = subtract;
5175  let Inst{8} = 0b1;
5176  let Inst{5} = 0b0;
5177  let validForTailPredication = 1;
5178}
5179
5180multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5181                             Intrinsic unpred_int, Intrinsic pred_int> {
5182  def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract>;
5183  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME),
5184                                  VTI, unpred_int, pred_int, 1, 1>;
5185}
5186
5187multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI> :
5188  MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd,
5189                                       int_arm_mve_hadd_predicated>;
5190
5191multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI> :
5192  MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub,
5193                                       int_arm_mve_hsub_predicated>;
5194
5195defm MVE_VHADD_qr_s8  : MVE_VHADD_qr_m<MVE_v16s8>;
5196defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16>;
5197defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32>;
5198defm MVE_VHADD_qr_u8  : MVE_VHADD_qr_m<MVE_v16u8>;
5199defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16>;
5200defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32>;
5201
5202defm MVE_VHSUB_qr_s8  : MVE_VHSUB_qr_m<MVE_v16s8>;
5203defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16>;
5204defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32>;
5205defm MVE_VHSUB_qr_u8  : MVE_VHSUB_qr_m<MVE_v16u8>;
5206defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16>;
5207defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32>;
5208
5209let Predicates = [HasMVEFloat] in {
5210  def MVE_VADD_qr_f32 : MVE_VxADDSUB_qr<"vadd",  "f32", 0b0, 0b11, 0b0>;
5211  def MVE_VADD_qr_f16 : MVE_VxADDSUB_qr<"vadd",  "f16", 0b1, 0b11, 0b0>;
5212
5213  def MVE_VSUB_qr_f32 : MVE_VxADDSUB_qr<"vsub",  "f32", 0b0, 0b11, 0b1>;
5214  def MVE_VSUB_qr_f16 : MVE_VxADDSUB_qr<"vsub",  "f16", 0b1, 0b11, 0b1>;
5215}
5216
5217defm : MVE_vec_scalar_fp_pat_m<fadd, int_arm_mve_add_predicated,
5218                               MVE_VADD_qr_f16, MVE_VADD_qr_f32>;
5219defm : MVE_vec_scalar_fp_pat_m<fsub, int_arm_mve_sub_predicated,
5220                               MVE_VSUB_qr_f16, MVE_VSUB_qr_f32>;
5221
5222class MVE_VxSHL_qr<string iname, string suffix, bit U, bits<2> size,
5223                   bit bit_7, bit bit_17, list<dag> pattern=[]>
5224  : MVE_qDest_single_rSrc<iname, suffix, pattern> {
5225
5226  let Inst{28} = U;
5227  let Inst{25-23} = 0b100;
5228  let Inst{21-20} = 0b11;
5229  let Inst{19-18} = size;
5230  let Inst{17} = bit_17;
5231  let Inst{16} = 0b1;
5232  let Inst{12-8} = 0b11110;
5233  let Inst{7} = bit_7;
5234  let Inst{6-4} = 0b110;
5235  let validForTailPredication = 1;
5236}
5237
5238multiclass MVE_VxSHL_qr_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
5239  def "" : MVE_VxSHL_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
5240  defvar Inst = !cast<Instruction>(NAME);
5241
5242  def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar
5243                         (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5244                         (i32 q), (i32 r), (i32 VTI.Unsigned))),
5245            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh)))>;
5246
5247  def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar_predicated
5248                         (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5249                         (i32 q), (i32 r), (i32 VTI.Unsigned),
5250                         (VTI.Pred VCCR:$mask))),
5251            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5252                           ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
5253}
5254
5255multiclass MVE_VxSHL_qr_types<string iname, bit bit_7, bit bit_17> {
5256  defm s8  : MVE_VxSHL_qr_p<iname, MVE_v16s8, bit_7, bit_17>;
5257  defm s16 : MVE_VxSHL_qr_p<iname, MVE_v8s16, bit_7, bit_17>;
5258  defm s32 : MVE_VxSHL_qr_p<iname, MVE_v4s32, bit_7, bit_17>;
5259  defm u8  : MVE_VxSHL_qr_p<iname, MVE_v16u8, bit_7, bit_17>;
5260  defm u16 : MVE_VxSHL_qr_p<iname, MVE_v8u16, bit_7, bit_17>;
5261  defm u32 : MVE_VxSHL_qr_p<iname, MVE_v4u32, bit_7, bit_17>;
5262}
5263
5264defm MVE_VSHL_qr   : MVE_VxSHL_qr_types<"vshl",   0b0, 0b0>;
5265defm MVE_VRSHL_qr  : MVE_VxSHL_qr_types<"vrshl",  0b0, 0b1>;
5266defm MVE_VQSHL_qr  : MVE_VxSHL_qr_types<"vqshl",  0b1, 0b0>;
5267defm MVE_VQRSHL_qr : MVE_VxSHL_qr_types<"vqrshl", 0b1, 0b1>;
5268
5269let Predicates = [HasMVEInt] in {
5270  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 (ARMvdup rGPR:$Rm)))),
5271            (v4i32 (MVE_VSHL_qru32 (v4i32 MQPR:$Qm), rGPR:$Rm))>;
5272  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 (ARMvdup rGPR:$Rm)))),
5273            (v8i16 (MVE_VSHL_qru16 (v8i16 MQPR:$Qm), rGPR:$Rm))>;
5274  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 (ARMvdup rGPR:$Rm)))),
5275            (v16i8 (MVE_VSHL_qru8 (v16i8 MQPR:$Qm), rGPR:$Rm))>;
5276
5277  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 (ARMvdup rGPR:$Rm)))),
5278            (v4i32 (MVE_VSHL_qrs32 (v4i32 MQPR:$Qm), rGPR:$Rm))>;
5279  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 (ARMvdup rGPR:$Rm)))),
5280            (v8i16 (MVE_VSHL_qrs16 (v8i16 MQPR:$Qm), rGPR:$Rm))>;
5281  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 (ARMvdup rGPR:$Rm)))),
5282            (v16i8 (MVE_VSHL_qrs8 (v16i8 MQPR:$Qm), rGPR:$Rm))>;
5283}
5284
5285class MVE_VBRSR<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
5286  : MVE_qDest_rSrc<iname, suffix, "", pattern> {
5287
5288  let Inst{28} = 0b1;
5289  let Inst{21-20} = size;
5290  let Inst{16} = 0b1;
5291  let Inst{12} = 0b1;
5292  let Inst{8} = 0b0;
5293  let Inst{5} = 0b1;
5294  let validForTailPredication = 1;
5295}
5296
5297def MVE_VBRSR8  : MVE_VBRSR<"vbrsr", "8", 0b00>;
5298def MVE_VBRSR16 : MVE_VBRSR<"vbrsr", "16", 0b01>;
5299def MVE_VBRSR32 : MVE_VBRSR<"vbrsr", "32", 0b10>;
5300
5301multiclass MVE_VBRSR_pat_m<MVEVectorVTInfo VTI, Instruction Inst> {
5302  // Unpredicated
5303  def : Pat<(VTI.Vec (int_arm_mve_vbrsr (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm))),
5304            (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm)))>;
5305  // Predicated
5306  def : Pat<(VTI.Vec (int_arm_mve_vbrsr_predicated
5307                          (VTI.Vec MQPR:$inactive),
5308                          (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5309                          (VTI.Pred VCCR:$mask))),
5310            (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5311                          ARMVCCThen, (VTI.Pred VCCR:$mask),
5312                          (VTI.Vec MQPR:$inactive)))>;
5313}
5314
5315let Predicates = [HasMVEInt] in {
5316  def : Pat<(v16i8 ( bitreverse (v16i8 MQPR:$val1))),
5317            (v16i8 ( MVE_VBRSR8 (v16i8 MQPR:$val1), (t2MOVi (i32 8)) ))>;
5318
5319  def : Pat<(v4i32 ( bitreverse (v4i32 MQPR:$val1))),
5320            (v4i32 ( MVE_VBRSR32 (v4i32 MQPR:$val1), (t2MOVi (i32 32)) ))>;
5321
5322  def : Pat<(v8i16 ( bitreverse (v8i16 MQPR:$val1))),
5323            (v8i16 ( MVE_VBRSR16 (v8i16 MQPR:$val1), (t2MOVi (i32 16)) ))>;
5324
5325  defm : MVE_VBRSR_pat_m<MVE_v16i8, MVE_VBRSR8>;
5326  defm : MVE_VBRSR_pat_m<MVE_v8i16, MVE_VBRSR16>;
5327  defm : MVE_VBRSR_pat_m<MVE_v4i32, MVE_VBRSR32>;
5328}
5329
5330let Predicates = [HasMVEFloat] in {
5331  defm : MVE_VBRSR_pat_m<MVE_v8f16, MVE_VBRSR16>;
5332  defm : MVE_VBRSR_pat_m<MVE_v4f32, MVE_VBRSR32>;
5333}
5334
5335class MVE_VMUL_qr_int<string iname, string suffix, bits<2> size>
5336  : MVE_qDest_rSrc<iname, suffix, ""> {
5337
5338  let Inst{28} = 0b0;
5339  let Inst{21-20} = size;
5340  let Inst{16} = 0b1;
5341  let Inst{12} = 0b1;
5342  let Inst{8} = 0b0;
5343  let Inst{5} = 0b1;
5344  let validForTailPredication = 1;
5345}
5346
5347multiclass MVE_VMUL_qr_int_m<MVEVectorVTInfo VTI> {
5348  def "" : MVE_VMUL_qr_int<"vmul", VTI.Suffix, VTI.Size>;
5349  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
5350                                  mul, int_arm_mve_mul_predicated>;
5351}
5352
5353defm MVE_VMUL_qr_i8  : MVE_VMUL_qr_int_m<MVE_v16i8>;
5354defm MVE_VMUL_qr_i16 : MVE_VMUL_qr_int_m<MVE_v8i16>;
5355defm MVE_VMUL_qr_i32 : MVE_VMUL_qr_int_m<MVE_v4i32>;
5356
5357class MVE_VxxMUL_qr<string iname, string suffix,
5358                    bit bit_28, bits<2> bits_21_20, list<dag> pattern=[]>
5359  : MVE_qDest_rSrc<iname, suffix, "", pattern> {
5360
5361  let Inst{28} = bit_28;
5362  let Inst{21-20} = bits_21_20;
5363  let Inst{16} = 0b1;
5364  let Inst{12} = 0b0;
5365  let Inst{8} = 0b0;
5366  let Inst{5} = 0b1;
5367}
5368
5369multiclass MVE_VxxMUL_qr_m<string iname, MVEVectorVTInfo VTI, bit bit_28,
5370                           Intrinsic int_unpred, Intrinsic int_pred> {
5371  def "" : MVE_VxxMUL_qr<iname, VTI.Suffix, bit_28, VTI.Size>;
5372  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI,
5373                                  int_unpred, int_pred>;
5374}
5375
5376multiclass MVE_VQDMULH_qr_m<MVEVectorVTInfo VTI> :
5377  MVE_VxxMUL_qr_m<"vqdmulh", VTI, 0b0,
5378                  int_arm_mve_vqdmulh, int_arm_mve_qdmulh_predicated>;
5379
5380multiclass MVE_VQRDMULH_qr_m<MVEVectorVTInfo VTI> :
5381  MVE_VxxMUL_qr_m<"vqrdmulh", VTI, 0b1,
5382                  int_arm_mve_vqrdmulh, int_arm_mve_qrdmulh_predicated>;
5383
5384defm MVE_VQDMULH_qr_s8    : MVE_VQDMULH_qr_m<MVE_v16s8>;
5385defm MVE_VQDMULH_qr_s16   : MVE_VQDMULH_qr_m<MVE_v8s16>;
5386defm MVE_VQDMULH_qr_s32   : MVE_VQDMULH_qr_m<MVE_v4s32>;
5387
5388defm MVE_VQRDMULH_qr_s8   : MVE_VQRDMULH_qr_m<MVE_v16s8>;
5389defm MVE_VQRDMULH_qr_s16  : MVE_VQRDMULH_qr_m<MVE_v8s16>;
5390defm MVE_VQRDMULH_qr_s32  : MVE_VQRDMULH_qr_m<MVE_v4s32>;
5391
5392let Predicates = [HasMVEFloat], validForTailPredication = 1 in {
5393  def MVE_VMUL_qr_f16   : MVE_VxxMUL_qr<"vmul", "f16", 0b1, 0b11>;
5394  def MVE_VMUL_qr_f32   : MVE_VxxMUL_qr<"vmul", "f32", 0b0, 0b11>;
5395}
5396
5397defm : MVE_vec_scalar_fp_pat_m<fmul, int_arm_mve_mul_predicated,
5398                               MVE_VMUL_qr_f16, MVE_VMUL_qr_f32>;
5399
5400class MVE_VFMAMLA_qr<string iname, string suffix,
5401                     bit bit_28, bits<2> bits_21_20, bit S,
5402                     list<dag> pattern=[]>
5403  : MVE_qDestSrc_rSrc<iname, suffix, pattern> {
5404
5405  let Inst{28} = bit_28;
5406  let Inst{21-20} = bits_21_20;
5407  let Inst{16} = 0b1;
5408  let Inst{12} = S;
5409  let Inst{8} = 0b0;
5410  let Inst{5} = 0b0;
5411  let validForTailPredication = 1;
5412  let hasSideEffects = 0;
5413}
5414
5415multiclass MVE_VMLA_qr_multi<string iname, MVEVectorVTInfo VTI,
5416                             bit scalar_addend> {
5417  def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
5418                         scalar_addend>;
5419  defvar Inst = !cast<Instruction>(NAME);
5420  defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # iname # "_n_predicated");
5421  defvar v1   = (VTI.Vec MQPR:$v1);
5422  defvar v2   = (VTI.Vec MQPR:$v2);
5423  defvar vs   = (VTI.Vec (ARMvdup rGPR:$s));
5424  defvar s    = (i32 rGPR:$s);
5425  defvar pred = (VTI.Pred VCCR:$pred);
5426
5427  // The signed and unsigned variants of this instruction have different
5428  // encodings, but they're functionally identical. For the sake of
5429  // determinism, we generate only the unsigned variant.
5430  if VTI.Unsigned then let Predicates = [HasMVEInt] in {
5431    if scalar_addend then {
5432      def : Pat<(VTI.Vec (add (mul v1, v2), vs)),
5433                (VTI.Vec (Inst v1, v2, s))>;
5434    } else {
5435      def : Pat<(VTI.Vec (add (mul v2, vs), v1)),
5436                (VTI.Vec (Inst v1, v2, s))>;
5437    }
5438
5439    def : Pat<(VTI.Vec (pred_int v1, v2, s, pred)),
5440              (VTI.Vec (Inst v1, v2, s, ARMVCCThen, pred))>;
5441  }
5442}
5443
5444defm MVE_VMLA_qr_s8   : MVE_VMLA_qr_multi<"vmla", MVE_v16s8, 0b0>;
5445defm MVE_VMLA_qr_s16  : MVE_VMLA_qr_multi<"vmla", MVE_v8s16, 0b0>;
5446defm MVE_VMLA_qr_s32  : MVE_VMLA_qr_multi<"vmla", MVE_v4s32, 0b0>;
5447defm MVE_VMLA_qr_u8   : MVE_VMLA_qr_multi<"vmla", MVE_v16u8, 0b0>;
5448defm MVE_VMLA_qr_u16  : MVE_VMLA_qr_multi<"vmla", MVE_v8u16, 0b0>;
5449defm MVE_VMLA_qr_u32  : MVE_VMLA_qr_multi<"vmla", MVE_v4u32, 0b0>;
5450
5451defm MVE_VMLAS_qr_s8  : MVE_VMLA_qr_multi<"vmlas", MVE_v16s8, 0b1>;
5452defm MVE_VMLAS_qr_s16 : MVE_VMLA_qr_multi<"vmlas", MVE_v8s16, 0b1>;
5453defm MVE_VMLAS_qr_s32 : MVE_VMLA_qr_multi<"vmlas", MVE_v4s32, 0b1>;
5454defm MVE_VMLAS_qr_u8  : MVE_VMLA_qr_multi<"vmlas", MVE_v16u8, 0b1>;
5455defm MVE_VMLAS_qr_u16 : MVE_VMLA_qr_multi<"vmlas", MVE_v8u16, 0b1>;
5456defm MVE_VMLAS_qr_u32 : MVE_VMLA_qr_multi<"vmlas", MVE_v4u32, 0b1>;
5457
5458multiclass MVE_VFMA_qr_multi<string iname, MVEVectorVTInfo VTI,
5459                             bit scalar_addend> {
5460  def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, scalar_addend>;
5461  defvar Inst = !cast<Instruction>(NAME);
5462  defvar pred_int = int_arm_mve_fma_predicated;
5463  defvar v1   = (VTI.Vec MQPR:$v1);
5464  defvar v2   = (VTI.Vec MQPR:$v2);
5465  defvar vs   = (VTI.Vec (ARMvdup (i32 rGPR:$s)));
5466  defvar is   = (i32 rGPR:$s);
5467  defvar pred = (VTI.Pred VCCR:$pred);
5468
5469  let Predicates = [HasMVEFloat] in {
5470    if scalar_addend then {
5471      def : Pat<(VTI.Vec (fma v1, v2, vs)),
5472                (VTI.Vec (Inst v1, v2, is))>;
5473      def : Pat<(VTI.Vec (pred_int v1, v2, vs, pred)),
5474                (VTI.Vec (Inst v1, v2, is, ARMVCCThen, pred))>;
5475    } else {
5476      def : Pat<(VTI.Vec (fma v1, vs, v2)),
5477                (VTI.Vec (Inst v2, v1, is))>;
5478      def : Pat<(VTI.Vec (fma vs, v1, v2)),
5479                (VTI.Vec (Inst v2, v1, is))>;
5480      def : Pat<(VTI.Vec (pred_int v1, vs, v2, pred)),
5481                (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred))>;
5482      def : Pat<(VTI.Vec (pred_int vs, v1, v2, pred)),
5483                (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred))>;
5484    }
5485  }
5486}
5487
5488let Predicates = [HasMVEFloat] in {
5489  defm MVE_VFMA_qr_f16  : MVE_VFMA_qr_multi<"vfma",  MVE_v8f16, 0>;
5490  defm MVE_VFMA_qr_f32  : MVE_VFMA_qr_multi<"vfma",  MVE_v4f32, 0>;
5491  defm MVE_VFMA_qr_Sf16 : MVE_VFMA_qr_multi<"vfmas", MVE_v8f16, 1>;
5492  defm MVE_VFMA_qr_Sf32 : MVE_VFMA_qr_multi<"vfmas", MVE_v4f32, 1>;
5493}
5494
5495class MVE_VQDMLAH_qr<string iname, string suffix, bit U, bits<2> size,
5496                     bit bit_5, bit bit_12, list<dag> pattern=[]>
5497  : MVE_qDestSrc_rSrc<iname, suffix, pattern> {
5498
5499  let Inst{28} = U;
5500  let Inst{21-20} = size;
5501  let Inst{16} = 0b0;
5502  let Inst{12} = bit_12;
5503  let Inst{8} = 0b0;
5504  let Inst{5} = bit_5;
5505}
5506
5507multiclass MVE_VQDMLAH_qr_multi<string iname, MVEVectorVTInfo VTI,
5508                                bit bit_5, bit bit_12> {
5509  def "": MVE_VQDMLAH_qr<iname, VTI.Suffix, 0b0, VTI.Size, bit_5, bit_12>;
5510  defvar Inst = !cast<Instruction>(NAME);
5511  defvar unpred_int = !cast<Intrinsic>("int_arm_mve_" # iname);
5512  defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # iname # "_predicated");
5513
5514  let Predicates = [HasMVEInt] in {
5515    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5516                                   (i32 rGPR:$s))),
5517              (VTI.Vec (Inst       (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5518                                   (i32 rGPR:$s)))>;
5519    def : Pat<(VTI.Vec (pred_int   (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5520                                   (i32 rGPR:$s), (VTI.Pred VCCR:$pred))),
5521              (VTI.Vec (Inst       (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5522                                   (i32 rGPR:$s), ARMVCCThen,
5523                                   (VTI.Pred VCCR:$pred)))>;
5524  }
5525}
5526
5527multiclass MVE_VQDMLAH_qr_types<string iname, bit bit_5, bit bit_12> {
5528  defm s8  : MVE_VQDMLAH_qr_multi<iname, MVE_v16s8, bit_5, bit_12>;
5529  defm s16 : MVE_VQDMLAH_qr_multi<iname, MVE_v8s16, bit_5, bit_12>;
5530  defm s32 : MVE_VQDMLAH_qr_multi<iname, MVE_v4s32, bit_5, bit_12>;
5531}
5532
5533defm MVE_VQDMLAH_qr   : MVE_VQDMLAH_qr_types<"vqdmlah",   0b1, 0b0>;
5534defm MVE_VQRDMLAH_qr  : MVE_VQDMLAH_qr_types<"vqrdmlah",  0b0, 0b0>;
5535defm MVE_VQDMLASH_qr  : MVE_VQDMLAH_qr_types<"vqdmlash",  0b1, 0b1>;
5536defm MVE_VQRDMLASH_qr : MVE_VQDMLAH_qr_types<"vqrdmlash", 0b0, 0b1>;
5537
5538class MVE_VxDUP<string iname, string suffix, bits<2> size, bit bit_12,
5539              list<dag> pattern=[]>
5540  : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
5541          (ins tGPREven:$Rn_src, MVE_VIDUP_imm:$imm), NoItinerary,
5542          iname, suffix, "$Qd, $Rn, $imm", vpred_r, "$Rn = $Rn_src",
5543          pattern> {
5544  bits<4> Qd;
5545  bits<4> Rn;
5546  bits<2> imm;
5547
5548  let Inst{28} = 0b0;
5549  let Inst{25-23} = 0b100;
5550  let Inst{22} = Qd{3};
5551  let Inst{21-20} = size;
5552  let Inst{19-17} = Rn{3-1};
5553  let Inst{16} = 0b1;
5554  let Inst{15-13} = Qd{2-0};
5555  let Inst{12} = bit_12;
5556  let Inst{11-8} = 0b1111;
5557  let Inst{7} = imm{1};
5558  let Inst{6-1} = 0b110111;
5559  let Inst{0} = imm{0};
5560  let validForTailPredication = 1;
5561  let hasSideEffects = 0;
5562}
5563
5564def MVE_VIDUPu8  : MVE_VxDUP<"vidup", "u8",  0b00, 0b0>;
5565def MVE_VIDUPu16 : MVE_VxDUP<"vidup", "u16", 0b01, 0b0>;
5566def MVE_VIDUPu32 : MVE_VxDUP<"vidup", "u32", 0b10, 0b0>;
5567
5568def MVE_VDDUPu8  : MVE_VxDUP<"vddup", "u8",  0b00, 0b1>;
5569def MVE_VDDUPu16 : MVE_VxDUP<"vddup", "u16", 0b01, 0b1>;
5570def MVE_VDDUPu32 : MVE_VxDUP<"vddup", "u32", 0b10, 0b1>;
5571
5572class MVE_VxWDUP<string iname, string suffix, bits<2> size, bit bit_12,
5573                 list<dag> pattern=[]>
5574  : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
5575          (ins tGPREven:$Rn_src, tGPROdd:$Rm, MVE_VIDUP_imm:$imm), NoItinerary,
5576          iname, suffix, "$Qd, $Rn, $Rm, $imm", vpred_r, "$Rn = $Rn_src",
5577          pattern> {
5578  bits<4> Qd;
5579  bits<4> Rm;
5580  bits<4> Rn;
5581  bits<2> imm;
5582
5583  let Inst{28} = 0b0;
5584  let Inst{25-23} = 0b100;
5585  let Inst{22} = Qd{3};
5586  let Inst{21-20} = size;
5587  let Inst{19-17} = Rn{3-1};
5588  let Inst{16} = 0b1;
5589  let Inst{15-13} = Qd{2-0};
5590  let Inst{12} = bit_12;
5591  let Inst{11-8} = 0b1111;
5592  let Inst{7} = imm{1};
5593  let Inst{6-4} = 0b110;
5594  let Inst{3-1} = Rm{3-1};
5595  let Inst{0} = imm{0};
5596  let validForTailPredication = 1;
5597  let hasSideEffects = 0;
5598}
5599
5600def MVE_VIWDUPu8  : MVE_VxWDUP<"viwdup", "u8",  0b00, 0b0>;
5601def MVE_VIWDUPu16 : MVE_VxWDUP<"viwdup", "u16", 0b01, 0b0>;
5602def MVE_VIWDUPu32 : MVE_VxWDUP<"viwdup", "u32", 0b10, 0b0>;
5603
5604def MVE_VDWDUPu8  : MVE_VxWDUP<"vdwdup", "u8",  0b00, 0b1>;
5605def MVE_VDWDUPu16 : MVE_VxWDUP<"vdwdup", "u16", 0b01, 0b1>;
5606def MVE_VDWDUPu32 : MVE_VxWDUP<"vdwdup", "u32", 0b10, 0b1>;
5607
5608let hasSideEffects = 1 in
5609class MVE_VCTPInst<string suffix, bits<2> size, list<dag> pattern=[]>
5610  : MVE_p<(outs VCCR:$P0), (ins rGPR:$Rn), NoItinerary, "vctp", suffix,
5611          "$Rn", vpred_n, "", pattern> {
5612  bits<4> Rn;
5613
5614  let Inst{28-27} = 0b10;
5615  let Inst{26-22} = 0b00000;
5616  let Inst{21-20} = size;
5617  let Inst{19-16} = Rn{3-0};
5618  let Inst{15-11} = 0b11101;
5619  let Inst{10-0}  = 0b00000000001;
5620  let Unpredictable{10-0} = 0b11111111111;
5621
5622  let Constraints = "";
5623  let DecoderMethod = "DecodeMveVCTP";
5624  let validForTailPredication = 1;
5625}
5626
5627multiclass MVE_VCTP<MVEVectorVTInfo VTI, Intrinsic intr> {
5628  def "": MVE_VCTPInst<VTI.BitsSuffix, VTI.Size>;
5629  defvar Inst = !cast<Instruction>(NAME);
5630
5631  let Predicates = [HasMVEInt] in {
5632    def : Pat<(intr rGPR:$Rn), (VTI.Pred (Inst rGPR:$Rn))>;
5633    def : Pat<(and (intr rGPR:$Rn), (VTI.Pred VCCR:$mask)),
5634              (VTI.Pred (Inst rGPR:$Rn, ARMVCCThen, VCCR:$mask))>;
5635  }
5636}
5637
5638defm MVE_VCTP8  : MVE_VCTP<MVE_v16i8, int_arm_mve_vctp8>;
5639defm MVE_VCTP16 : MVE_VCTP<MVE_v8i16, int_arm_mve_vctp16>;
5640defm MVE_VCTP32 : MVE_VCTP<MVE_v4i32, int_arm_mve_vctp32>;
5641defm MVE_VCTP64 : MVE_VCTP<MVE_v2i64, int_arm_mve_vctp64>;
5642
5643// end of mve_qDest_rSrc
5644
5645// start of coproc mov
5646
5647class MVE_VMOV_64bit<dag oops, dag iops, bit to_qreg, string ops, string cstr>
5648  : MVE_VMOV_lane_base<oops, !con(iops, (ins MVEPairVectorIndex2:$idx,
5649                                             MVEPairVectorIndex0:$idx2)),
5650                       NoItinerary, "vmov", "", ops, cstr, []> {
5651  bits<5> Rt;
5652  bits<5> Rt2;
5653  bits<4> Qd;
5654  bit idx;
5655  bit idx2;
5656
5657  let Inst{31-23} = 0b111011000;
5658  let Inst{22} = Qd{3};
5659  let Inst{21} = 0b0;
5660  let Inst{20} = to_qreg;
5661  let Inst{19-16} = Rt2{3-0};
5662  let Inst{15-13} = Qd{2-0};
5663  let Inst{12-5} = 0b01111000;
5664  let Inst{4} = idx2;
5665  let Inst{3-0} = Rt{3-0};
5666
5667  let hasSideEffects = 0;
5668}
5669
5670// The assembly syntax for these instructions mentions the vector
5671// register name twice, e.g.
5672//
5673//    vmov q2[2], q2[0], r0, r1
5674//    vmov r0, r1, q2[2], q2[0]
5675//
5676// which needs a bit of juggling with MC operand handling.
5677//
5678// For the move _into_ a vector register, the MC operand list also has
5679// to mention the register name twice: once as the output, and once as
5680// an extra input to represent where the unchanged half of the output
5681// register comes from (when this instruction is used in code
5682// generation). So we arrange that the first mention of the vector reg
5683// in the instruction is considered by the AsmMatcher to be the output
5684// ($Qd), and the second one is the input ($QdSrc). Binding them
5685// together with the existing 'tie' constraint is enough to enforce at
5686// register allocation time that they have to be the same register.
5687//
5688// For the move _from_ a vector register, there's no way to get round
5689// the fact that both instances of that register name have to be
5690// inputs. They have to be the same register again, but this time, we
5691// can't use a tie constraint, because that has to be between an
5692// output and an input operand. So this time, we have to arrange that
5693// the q-reg appears just once in the MC operand list, in spite of
5694// being mentioned twice in the asm syntax - which needs a custom
5695// AsmMatchConverter.
5696
5697def MVE_VMOV_q_rr : MVE_VMOV_64bit<(outs MQPR:$Qd),
5698                                   (ins MQPR:$QdSrc, rGPR:$Rt, rGPR:$Rt2),
5699                                   0b1, "$Qd$idx, $QdSrc$idx2, $Rt, $Rt2",
5700                                   "$Qd = $QdSrc"> {
5701  let DecoderMethod = "DecodeMVEVMOVDRegtoQ";
5702}
5703
5704def MVE_VMOV_rr_q : MVE_VMOV_64bit<(outs rGPR:$Rt, rGPR:$Rt2), (ins MQPR:$Qd),
5705                                   0b0, "$Rt, $Rt2, $Qd$idx, $Qd$idx2", ""> {
5706  let DecoderMethod = "DecodeMVEVMOVQtoDReg";
5707  let AsmMatchConverter = "cvtMVEVMOVQtoDReg";
5708}
5709
5710// end of coproc mov
5711
5712// start of MVE interleaving load/store
5713
5714// Base class for the family of interleaving/deinterleaving
5715// load/stores with names like VLD20.8 and VST43.32.
5716class MVE_vldst24_base<bit writeback, bit fourregs, bits<2> stage, bits<2> size,
5717                       bit load, dag Oops, dag loadIops, dag wbIops,
5718                       string iname, string ops,
5719                       string cstr, list<dag> pattern=[]>
5720  : MVE_MI<Oops, !con(loadIops, wbIops), NoItinerary, iname, ops, cstr, pattern> {
5721  bits<4> VQd;
5722  bits<4> Rn;
5723
5724  let Inst{31-22} = 0b1111110010;
5725  let Inst{21} = writeback;
5726  let Inst{20} = load;
5727  let Inst{19-16} = Rn;
5728  let Inst{15-13} = VQd{2-0};
5729  let Inst{12-9} = 0b1111;
5730  let Inst{8-7} = size;
5731  let Inst{6-5} = stage;
5732  let Inst{4-1} = 0b0000;
5733  let Inst{0} = fourregs;
5734
5735  let mayLoad = load;
5736  let mayStore = !eq(load,0);
5737  let hasSideEffects = 0;
5738}
5739
5740// A parameter class used to encapsulate all the ways the writeback
5741// variants of VLD20 and friends differ from the non-writeback ones.
5742class MVE_vldst24_writeback<bit b, dag Oo, dag Io,
5743                            string sy="", string c="", string n=""> {
5744  bit writeback = b;
5745  dag Oops = Oo;
5746  dag Iops = Io;
5747  string syntax = sy;
5748  string cstr = c;
5749  string id_suffix = n;
5750}
5751
5752// Another parameter class that encapsulates the differences between VLD2x
5753// and VLD4x.
5754class MVE_vldst24_nvecs<int n, list<int> s, bit b, RegisterOperand vl> {
5755  int nvecs = n;
5756  list<int> stages = s;
5757  bit bit0 = b;
5758  RegisterOperand VecList = vl;
5759}
5760
5761// A third parameter class that distinguishes VLDnn.8 from .16 from .32.
5762class MVE_vldst24_lanesize<int i, bits<2> b> {
5763  int lanesize = i;
5764  bits<2> sizebits = b;
5765}
5766
5767// A base class for each direction of transfer: one for load, one for
5768// store. I can't make these a fourth independent parametric tuple
5769// class, because they have to take the nvecs tuple class as a
5770// parameter, in order to find the right VecList operand type.
5771
5772class MVE_vld24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,
5773                     MVE_vldst24_writeback wb, string iname,
5774                     list<dag> pattern=[]>
5775  : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 1,
5776                     !con((outs n.VecList:$VQd), wb.Oops),
5777                     (ins n.VecList:$VQdSrc), wb.Iops,
5778                     iname, "$VQd, $Rn" # wb.syntax,
5779                     wb.cstr # ",$VQdSrc = $VQd", pattern>;
5780
5781class MVE_vst24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,
5782                     MVE_vldst24_writeback wb, string iname,
5783                     list<dag> pattern=[]>
5784  : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 0,
5785                     wb.Oops, (ins n.VecList:$VQd), wb.Iops,
5786                     iname, "$VQd, $Rn" # wb.syntax,
5787                     wb.cstr, pattern>;
5788
5789// Actually define all the interleaving loads and stores, by a series
5790// of nested foreaches over number of vectors (VLD2/VLD4); stage
5791// within one of those series (VLDx0/VLDx1/VLDx2/VLDx3); size of
5792// vector lane; writeback or no writeback.
5793foreach n = [MVE_vldst24_nvecs<2, [0,1],     0, VecList2Q>,
5794             MVE_vldst24_nvecs<4, [0,1,2,3], 1, VecList4Q>] in
5795foreach stage = n.stages in
5796foreach s = [MVE_vldst24_lanesize< 8, 0b00>,
5797             MVE_vldst24_lanesize<16, 0b01>,
5798             MVE_vldst24_lanesize<32, 0b10>] in
5799foreach wb = [MVE_vldst24_writeback<
5800                1, (outs rGPR:$wb), (ins t2_nosp_addr_offset_none:$Rn),
5801                "!", "$Rn.base = $wb", "_wb">,
5802              MVE_vldst24_writeback<0, (outs), (ins t2_addr_offset_none:$Rn)>] in {
5803
5804  // For each case within all of those foreaches, define the actual
5805  // instructions. The def names are made by gluing together pieces
5806  // from all the parameter classes, and will end up being things like
5807  // MVE_VLD20_8 and MVE_VST43_16_wb.
5808
5809  def "MVE_VLD" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix
5810    : MVE_vld24_base<n, stage, s.sizebits, wb,
5811                     "vld" # n.nvecs # stage # "." # s.lanesize>;
5812
5813  def "MVE_VST" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix
5814    : MVE_vst24_base<n, stage, s.sizebits, wb,
5815                     "vst" # n.nvecs # stage # "." # s.lanesize>;
5816}
5817
5818def SDTARMVST2    : SDTypeProfile<1, 5, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>,
5819                                         SDTCisSameAs<3, 4>, SDTCisVT<5, i32>]>;
5820def SDTARMVST4    : SDTypeProfile<1, 7, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>,
5821                                         SDTCisSameAs<3, 4>, SDTCisSameAs<3, 5>,
5822                                         SDTCisSameAs<3, 6>, SDTCisVT<7, i32>]>;
5823def MVEVST2UPD       : SDNode<"ARMISD::VST2_UPD", SDTARMVST2, [SDNPHasChain]>;
5824def MVEVST4UPD       : SDNode<"ARMISD::VST4_UPD", SDTARMVST4, [SDNPHasChain]>;
5825
5826multiclass MVE_vst24_patterns<int lanesize, ValueType VT> {
5827  foreach stage = [0,1] in
5828    def : Pat<(int_arm_mve_vst2q i32:$addr,
5829                (VT MQPR:$v0), (VT MQPR:$v1), (i32 stage)),
5830              (!cast<Instruction>("MVE_VST2"#stage#"_"#lanesize)
5831                (REG_SEQUENCE QQPR, VT:$v0, qsub_0, VT:$v1, qsub_1),
5832                t2_addr_offset_none:$addr)>;
5833  foreach stage = [0,1] in
5834    def : Pat<(i32 (MVEVST2UPD i32:$addr, (i32 32),
5835                (VT MQPR:$v0), (VT MQPR:$v1), (i32 stage))),
5836              (i32 (!cast<Instruction>("MVE_VST2"#stage#"_"#lanesize#_wb)
5837                (REG_SEQUENCE QQPR, VT:$v0, qsub_0, VT:$v1, qsub_1),
5838                t2_addr_offset_none:$addr))>;
5839
5840  foreach stage = [0,1,2,3] in
5841    def : Pat<(int_arm_mve_vst4q i32:$addr,
5842                (VT MQPR:$v0), (VT MQPR:$v1),
5843                (VT MQPR:$v2), (VT MQPR:$v3), (i32 stage)),
5844              (!cast<Instruction>("MVE_VST4"#stage#"_"#lanesize)
5845                (REG_SEQUENCE QQQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1,
5846                                      VT:$v2, qsub_2, VT:$v3, qsub_3),
5847                t2_addr_offset_none:$addr)>;
5848  foreach stage = [0,1,2,3] in
5849    def : Pat<(i32 (MVEVST4UPD i32:$addr, (i32 64),
5850                (VT MQPR:$v0), (VT MQPR:$v1),
5851                (VT MQPR:$v2), (VT MQPR:$v3), (i32 stage))),
5852              (i32 (!cast<Instruction>("MVE_VST4"#stage#"_"#lanesize#_wb)
5853                (REG_SEQUENCE QQQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1,
5854                                      VT:$v2, qsub_2, VT:$v3, qsub_3),
5855                t2_addr_offset_none:$addr))>;
5856}
5857defm : MVE_vst24_patterns<8, v16i8>;
5858defm : MVE_vst24_patterns<16, v8i16>;
5859defm : MVE_vst24_patterns<32, v4i32>;
5860defm : MVE_vst24_patterns<16, v8f16>;
5861defm : MVE_vst24_patterns<32, v4f32>;
5862
5863// end of MVE interleaving load/store
5864
5865// start of MVE predicable load/store
5866
5867// A parameter class for the direction of transfer.
5868class MVE_ldst_direction<bit b, dag Oo, dag Io, string c=""> {
5869  bit load = b;
5870  dag Oops = Oo;
5871  dag Iops = Io;
5872  string cstr = c;
5873}
5874def MVE_ld: MVE_ldst_direction<1, (outs MQPR:$Qd), (ins), ",@earlyclobber $Qd">;
5875def MVE_st: MVE_ldst_direction<0, (outs), (ins MQPR:$Qd)>;
5876
5877// A parameter class for the size of memory access in a load.
5878class MVE_memsz<bits<2> e, int s, AddrMode m, string mn, list<string> types> {
5879  bits<2> encoding = e;         // opcode bit(s) for encoding
5880  int shift = s;                // shift applied to immediate load offset
5881  AddrMode AM = m;
5882
5883  // For instruction aliases: define the complete list of type
5884  // suffixes at this size, and the canonical ones for loads and
5885  // stores.
5886  string MnemonicLetter = mn;
5887  int TypeBits = !shl(8, s);
5888  string CanonLoadSuffix = ".u" # TypeBits;
5889  string CanonStoreSuffix = "." # TypeBits;
5890  list<string> suffixes = !foreach(letter, types, "." # letter # TypeBits);
5891}
5892
5893// Instances of MVE_memsz.
5894//
5895// (memD doesn't need an AddrMode, because those are only for
5896// contiguous loads, and memD is only used by gather/scatters.)
5897def MVE_memB: MVE_memsz<0b00, 0, AddrModeT2_i7,   "b", ["", "u", "s"]>;
5898def MVE_memH: MVE_memsz<0b01, 1, AddrModeT2_i7s2, "h", ["", "u", "s", "f"]>;
5899def MVE_memW: MVE_memsz<0b10, 2, AddrModeT2_i7s4, "w", ["", "u", "s", "f"]>;
5900def MVE_memD: MVE_memsz<0b11, 3, ?,               "d", ["", "u", "s", "f"]>;
5901
5902// This is the base class for all the MVE loads and stores other than
5903// the interleaving ones. All the non-interleaving loads/stores share
5904// the characteristic that they operate on just one vector register,
5905// so they are VPT-predicable.
5906//
5907// The predication operand is vpred_n, for both loads and stores. For
5908// store instructions, the reason is obvious: if there is no output
5909// register, there can't be a need for an input parameter giving the
5910// output register's previous value. Load instructions also don't need
5911// that input parameter, because unlike MVE data processing
5912// instructions, predicated loads are defined to set the inactive
5913// lanes of the output register to zero, instead of preserving their
5914// input values.
5915class MVE_VLDRSTR_base<MVE_ldst_direction dir, bit U, bit P, bit W, bit opc,
5916                       dag oops, dag iops, string asm, string suffix,
5917                       string ops, string cstr, list<dag> pattern=[]>
5918 : MVE_p<oops, iops, NoItinerary, asm, suffix, ops, vpred_n, cstr, pattern> {
5919  bits<3> Qd;
5920
5921  let Inst{28} = U;
5922  let Inst{25} = 0b0;
5923  let Inst{24} = P;
5924  let Inst{22} = 0b0;
5925  let Inst{21} = W;
5926  let Inst{20} = dir.load;
5927  let Inst{15-13} = Qd{2-0};
5928  let Inst{12} = opc;
5929  let Inst{11-9} = 0b111;
5930
5931  let mayLoad = dir.load;
5932  let mayStore = !eq(dir.load,0);
5933  let hasSideEffects = 0;
5934  let validForTailPredication = 1;
5935}
5936
5937// Contiguous load and store instructions. These come in two main
5938// categories: same-size loads/stores in which 128 bits of vector
5939// register is transferred to or from 128 bits of memory in the most
5940// obvious way, and widening loads / narrowing stores, in which the
5941// size of memory accessed is less than the size of a vector register,
5942// so the load instructions sign- or zero-extend each memory value
5943// into a wider vector lane, and the store instructions truncate
5944// correspondingly.
5945//
5946// The instruction mnemonics for these two classes look reasonably
5947// similar, but the actual encodings are different enough to need two
5948// separate base classes.
5949
5950// Contiguous, same size
5951class MVE_VLDRSTR_cs<MVE_ldst_direction dir, MVE_memsz memsz, bit P, bit W,
5952                     dag oops, dag iops, string asm, string suffix,
5953                     IndexMode im, string ops, string cstr>
5954  : MVE_VLDRSTR_base<dir, 0, P, W, 1, oops, iops, asm, suffix, ops, cstr> {
5955  bits<12> addr;
5956  let Inst{23} = addr{7};
5957  let Inst{19-16} = addr{11-8};
5958  let Inst{8-7} = memsz.encoding;
5959  let Inst{6-0} = addr{6-0};
5960}
5961
5962// Contiguous, widening/narrowing
5963class MVE_VLDRSTR_cw<MVE_ldst_direction dir, MVE_memsz memsz, bit U,
5964                     bit P, bit W, bits<2> size, dag oops, dag iops,
5965                     string asm, string suffix, IndexMode im,
5966                     string ops, string cstr>
5967  : MVE_VLDRSTR_base<dir, U, P, W, 0, oops, iops, asm, suffix, ops, cstr> {
5968  bits<11> addr;
5969  let Inst{23} = addr{7};
5970  let Inst{19} = memsz.encoding{0}; // enough to tell 16- from 32-bit
5971  let Inst{18-16} = addr{10-8};
5972  let Inst{8-7} = size;
5973  let Inst{6-0} = addr{6-0};
5974
5975  let IM = im;
5976}
5977
5978// Multiclass wrapper on each of the _cw and _cs base classes, to
5979// generate three writeback modes (none, preindex, postindex).
5980
5981multiclass MVE_VLDRSTR_cw_m<MVE_ldst_direction dir, MVE_memsz memsz,
5982                            string asm, string suffix, bit U, bits<2> size> {
5983  let AM = memsz.AM in {
5984    def "" : MVE_VLDRSTR_cw<
5985        dir, memsz, U, 1, 0, size,
5986        dir.Oops, !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),
5987        asm, suffix, IndexModeNone, "$Qd, $addr", "">;
5988
5989    def _pre : MVE_VLDRSTR_cw<
5990        dir, memsz, U, 1, 1, size,
5991        !con((outs tGPR:$wb), dir.Oops),
5992        !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),
5993        asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {
5994      let DecoderMethod = "DecodeMVE_MEM_1_pre<"#memsz.shift#">";
5995    }
5996
5997    def _post : MVE_VLDRSTR_cw<
5998        dir, memsz, U, 0, 1, size,
5999        !con((outs tGPR:$wb), dir.Oops),
6000        !con(dir.Iops, (ins t_addr_offset_none:$Rn,
6001                            t2am_imm7_offset<memsz.shift>:$addr)),
6002        asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {
6003      bits<4> Rn;
6004      let Inst{18-16} = Rn{2-0};
6005    }
6006  }
6007}
6008
6009multiclass MVE_VLDRSTR_cs_m<MVE_ldst_direction dir, MVE_memsz memsz,
6010                            string asm, string suffix> {
6011  let AM = memsz.AM in {
6012    def "" : MVE_VLDRSTR_cs<
6013        dir, memsz, 1, 0,
6014        dir.Oops, !con(dir.Iops, (ins t2addrmode_imm7<memsz.shift>:$addr)),
6015        asm, suffix, IndexModeNone, "$Qd, $addr", "">;
6016
6017    def _pre : MVE_VLDRSTR_cs<
6018        dir, memsz, 1, 1,
6019        !con((outs rGPR:$wb), dir.Oops),
6020        !con(dir.Iops, (ins t2addrmode_imm7_pre<memsz.shift>:$addr)),
6021        asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {
6022      let DecoderMethod = "DecodeMVE_MEM_2_pre<"#memsz.shift#">";
6023    }
6024
6025    def _post : MVE_VLDRSTR_cs<
6026        dir, memsz, 0, 1,
6027        !con((outs rGPR:$wb), dir.Oops),
6028        // We need an !if here to select the base register class,
6029        // because it's legal to write back to SP in a load of this
6030        // type, but not in a store.
6031        !con(dir.Iops, (ins !if(dir.load, t2_addr_offset_none,
6032                                          t2_nosp_addr_offset_none):$Rn,
6033                            t2am_imm7_offset<memsz.shift>:$addr)),
6034        asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {
6035      bits<4> Rn;
6036      let Inst{19-16} = Rn{3-0};
6037    }
6038  }
6039}
6040
6041// Now actually declare all the contiguous load/stores, via those
6042// multiclasses. The instruction ids coming out of this are the bare
6043// names shown in the defm, with _pre or _post appended for writeback,
6044// e.g. MVE_VLDRBS16, MVE_VSTRB16_pre, MVE_VSTRHU16_post.
6045
6046defm MVE_VLDRBS16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s16", 0, 0b01>;
6047defm MVE_VLDRBS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s32", 0, 0b10>;
6048defm MVE_VLDRBU16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u16", 1, 0b01>;
6049defm MVE_VLDRBU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u32", 1, 0b10>;
6050defm MVE_VLDRHS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "s32", 0, 0b10>;
6051defm MVE_VLDRHU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "u32", 1, 0b10>;
6052
6053defm MVE_VLDRBU8:  MVE_VLDRSTR_cs_m<MVE_ld, MVE_memB, "vldrb", "u8">;
6054defm MVE_VLDRHU16: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memH, "vldrh", "u16">;
6055defm MVE_VLDRWU32: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memW, "vldrw", "u32">;
6056
6057defm MVE_VSTRB16:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "16",  0, 0b01>;
6058defm MVE_VSTRB32:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "32",  0, 0b10>;
6059defm MVE_VSTRH32:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memH, "vstrh", "32",  0, 0b10>;
6060
6061defm MVE_VSTRBU8 : MVE_VLDRSTR_cs_m<MVE_st, MVE_memB, "vstrb", "8">;
6062defm MVE_VSTRHU16: MVE_VLDRSTR_cs_m<MVE_st, MVE_memH, "vstrh", "16">;
6063defm MVE_VSTRWU32: MVE_VLDRSTR_cs_m<MVE_st, MVE_memW, "vstrw", "32">;
6064
6065// Gather loads / scatter stores whose address operand is of the form
6066// [Rn,Qm], i.e. a single GPR as the common base address, plus a
6067// vector of offset from it. ('Load/store this sequence of elements of
6068// the same array.')
6069//
6070// Like the contiguous family, these loads and stores can widen the
6071// loaded values / truncate the stored ones, or they can just
6072// load/store the same size of memory and vector lane. But unlike the
6073// contiguous family, there's no particular difference in encoding
6074// between those two cases.
6075//
6076// This family also comes with the option to scale the offset values
6077// in Qm by the size of the loaded memory (i.e. to treat them as array
6078// indices), or not to scale them (to treat them as plain byte offsets
6079// in memory, so that perhaps the loaded values are unaligned). The
6080// scaled instructions' address operand in assembly looks like
6081// [Rn,Qm,UXTW #2] or similar.
6082
6083// Base class.
6084class MVE_VLDRSTR_rq<MVE_ldst_direction dir, MVE_memsz memsz, bit U,
6085                     bits<2> size, bit os, string asm, string suffix, int shift>
6086  : MVE_VLDRSTR_base<dir, U, 0b0, 0b0, 0, dir.Oops,
6087                     !con(dir.Iops, (ins mve_addr_rq_shift<shift>:$addr)),
6088                     asm, suffix, "$Qd, $addr", dir.cstr> {
6089  bits<7> addr;
6090  let Inst{23} = 0b1;
6091  let Inst{19-16} = addr{6-3};
6092  let Inst{8-7} = size;
6093  let Inst{6} = memsz.encoding{1};
6094  let Inst{5} = 0;
6095  let Inst{4} = memsz.encoding{0};
6096  let Inst{3-1} = addr{2-0};
6097  let Inst{0} = os;
6098}
6099
6100// Multiclass that defines the scaled and unscaled versions of an
6101// instruction, when the memory size is wider than a byte. The scaled
6102// version gets the default name like MVE_VLDRBU16_rq; the unscaled /
6103// potentially unaligned version gets a "_u" suffix, e.g.
6104// MVE_VLDRBU16_rq_u.
6105multiclass MVE_VLDRSTR_rq_w<MVE_ldst_direction dir, MVE_memsz memsz,
6106                            string asm, string suffix, bit U, bits<2> size> {
6107  def _u : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;
6108  def "" : MVE_VLDRSTR_rq<dir, memsz, U, size, 1, asm, suffix, memsz.shift>;
6109}
6110
6111// Subclass of MVE_VLDRSTR_rq with the same API as that multiclass,
6112// for use when the memory size is one byte, so there's no 'scaled'
6113// version of the instruction at all. (This is encoded as if it were
6114// unscaled, but named in the default way with no _u suffix.)
6115class MVE_VLDRSTR_rq_b<MVE_ldst_direction dir, MVE_memsz memsz,
6116                       string asm, string suffix, bit U, bits<2> size>
6117  : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;
6118
6119// Multiclasses wrapping that to add ISel patterns for intrinsics.
6120multiclass MVE_VLDR_rq_w<MVE_memsz memsz, list<MVEVectorVTInfo> VTIs> {
6121  defm "": MVE_VLDRSTR_rq_w<MVE_ld, memsz, "vldr" # memsz.MnemonicLetter,
6122                            VTIs[0].Suffix, VTIs[0].Unsigned, VTIs[0].Size>;
6123  defvar Inst = !cast<Instruction>(NAME);
6124  defvar InstU = !cast<Instruction>(NAME # "_u");
6125
6126  foreach VTI = VTIs in
6127  foreach UnsignedFlag = !if(!eq(VTI.Size, memsz.encoding),
6128                             [0,1], [VTI.Unsigned]) in {
6129    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag)),
6130              (VTI.Vec (InstU GPR:$base, MQPR:$offsets))>;
6131    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag)),
6132              (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6133    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag, (VTI.Pred VCCR:$pred))),
6134              (VTI.Vec (InstU GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred))>;
6135    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag, (VTI.Pred VCCR:$pred))),
6136              (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred))>;
6137  }
6138}
6139multiclass MVE_VLDR_rq_b<list<MVEVectorVTInfo> VTIs> {
6140  def "": MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb",
6141                           VTIs[0].Suffix, VTIs[0].Unsigned, VTIs[0].Size>;
6142  defvar Inst = !cast<Instruction>(NAME);
6143
6144  foreach VTI = VTIs in {
6145    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned)),
6146              (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6147    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned, (VTI.Pred VCCR:$pred))),
6148              (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred))>;
6149  }
6150}
6151multiclass MVE_VSTR_rq_w<MVE_memsz memsz, list<MVEVectorVTInfo> VTIs> {
6152  defm "": MVE_VLDRSTR_rq_w<MVE_st, memsz, "vstr" # memsz.MnemonicLetter,
6153                            VTIs[0].BitsSuffix, 0, VTIs[0].Size>;
6154  defvar Inst = !cast<Instruction>(NAME);
6155  defvar InstU = !cast<Instruction>(NAME # "_u");
6156
6157  foreach VTI = VTIs in {
6158    def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0),
6159              (InstU MQPR:$data, GPR:$base, MQPR:$offsets)>;
6160    def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift),
6161              (Inst MQPR:$data, GPR:$base, MQPR:$offsets)>;
6162    def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0, (VTI.Pred VCCR:$pred)),
6163              (InstU MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred)>;
6164    def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift, (VTI.Pred VCCR:$pred)),
6165              (Inst MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred)>;
6166  }
6167}
6168multiclass MVE_VSTR_rq_b<list<MVEVectorVTInfo> VTIs> {
6169  def "": MVE_VLDRSTR_rq_b<MVE_st, MVE_memB, "vstrb",
6170                           VTIs[0].BitsSuffix, 0, VTIs[0].Size>;
6171  defvar Inst = !cast<Instruction>(NAME);
6172
6173  foreach VTI = VTIs in {
6174    def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0),
6175              (Inst MQPR:$data, GPR:$base, MQPR:$offsets)>;
6176    def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0, (VTI.Pred VCCR:$pred)),
6177              (Inst MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred)>;
6178  }
6179}
6180
6181// Actually define all the loads and stores in this family.
6182
6183defm MVE_VLDRBU8_rq : MVE_VLDR_rq_b<[MVE_v16u8,MVE_v16s8]>;
6184defm MVE_VLDRBU16_rq: MVE_VLDR_rq_b<[MVE_v8u16]>;
6185defm MVE_VLDRBS16_rq: MVE_VLDR_rq_b<[MVE_v8s16]>;
6186defm MVE_VLDRBU32_rq: MVE_VLDR_rq_b<[MVE_v4u32]>;
6187defm MVE_VLDRBS32_rq: MVE_VLDR_rq_b<[MVE_v4s32]>;
6188
6189defm MVE_VLDRHU16_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v8u16,MVE_v8s16,MVE_v8f16]>;
6190defm MVE_VLDRHU32_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v4u32]>;
6191defm MVE_VLDRHS32_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v4s32]>;
6192defm MVE_VLDRWU32_rq: MVE_VLDR_rq_w<MVE_memW, [MVE_v4u32,MVE_v4s32,MVE_v4f32]>;
6193defm MVE_VLDRDU64_rq: MVE_VLDR_rq_w<MVE_memD, [MVE_v2u64,MVE_v2s64]>;
6194
6195defm MVE_VSTRB8_rq  : MVE_VSTR_rq_b<[MVE_v16i8]>;
6196defm MVE_VSTRB16_rq : MVE_VSTR_rq_b<[MVE_v8i16]>;
6197defm MVE_VSTRB32_rq : MVE_VSTR_rq_b<[MVE_v4i32]>;
6198
6199defm MVE_VSTRH16_rq : MVE_VSTR_rq_w<MVE_memH, [MVE_v8i16,MVE_v8f16]>;
6200defm MVE_VSTRH32_rq : MVE_VSTR_rq_w<MVE_memH, [MVE_v4i32]>;
6201defm MVE_VSTRW32_rq : MVE_VSTR_rq_w<MVE_memW, [MVE_v4i32,MVE_v4f32]>;
6202defm MVE_VSTRD64_rq : MVE_VSTR_rq_w<MVE_memD, [MVE_v2i64]>;
6203
6204// Gather loads / scatter stores whose address operand is of the form
6205// [Qm,#imm], i.e. a vector containing a full base address for each
6206// loaded item, plus an immediate offset applied consistently to all
6207// of them. ('Load/store the same field from this vector of pointers
6208// to a structure type.')
6209//
6210// This family requires the vector lane size to be at least 32 bits
6211// (so there's room for an address in each lane at all). It has no
6212// widening/narrowing variants. But it does support preindex
6213// writeback, in which the address vector is updated to hold the
6214// addresses actually loaded from.
6215
6216// Base class.
6217class MVE_VLDRSTR_qi<MVE_ldst_direction dir, MVE_memsz memsz, bit W, dag wbops,
6218                     string asm, string wbAsm, string suffix, string cstr = "">
6219  : MVE_VLDRSTR_base<dir, 1, 1, W, 1, !con(wbops, dir.Oops),
6220                     !con(dir.Iops, (ins mve_addr_q_shift<memsz.shift>:$addr)),
6221                     asm, suffix, "$Qd, $addr" # wbAsm, cstr # dir.cstr> {
6222  bits<11> addr;
6223  let Inst{23} = addr{7};
6224  let Inst{19-17} = addr{10-8};
6225  let Inst{16} = 0;
6226  let Inst{8} = memsz.encoding{0}; // enough to distinguish 32- from 64-bit
6227  let Inst{7} = 0;
6228  let Inst{6-0} = addr{6-0};
6229}
6230
6231// Multiclass that generates the non-writeback and writeback variants.
6232multiclass MVE_VLDRSTR_qi_m<MVE_ldst_direction dir, MVE_memsz memsz,
6233                            string asm, string suffix> {
6234  def ""   : MVE_VLDRSTR_qi<dir, memsz, 0, (outs),          asm, "",  suffix>;
6235  def _pre : MVE_VLDRSTR_qi<dir, memsz, 1, (outs MQPR:$wb), asm, "!", suffix,
6236                            "$addr.base = $wb"> {
6237    let DecoderMethod="DecodeMVE_MEM_3_pre<"#memsz.shift#">";
6238  }
6239}
6240
6241// Multiclasses wrapping that one, adding selection patterns for the
6242// non-writeback loads and all the stores. (The writeback loads must
6243// deliver multiple output values, so they have to be selected by C++
6244// code.)
6245multiclass MVE_VLDR_qi<MVE_memsz memsz, MVEVectorVTInfo AVTI,
6246                       list<MVEVectorVTInfo> DVTIs> {
6247  defm "" : MVE_VLDRSTR_qi_m<MVE_ld, memsz, "vldr" # memsz.MnemonicLetter,
6248                             "u" # memsz.TypeBits>;
6249  defvar Inst = !cast<Instruction>(NAME);
6250
6251  foreach DVTI = DVTIs in {
6252    def : Pat<(DVTI.Vec (int_arm_mve_vldr_gather_base
6253                 (AVTI.Vec MQPR:$addr), (i32 imm:$offset))),
6254              (DVTI.Vec (Inst (AVTI.Vec MQPR:$addr), (i32 imm:$offset)))>;
6255    def : Pat<(DVTI.Vec (int_arm_mve_vldr_gather_base_predicated
6256                 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (AVTI.Pred VCCR:$pred))),
6257              (DVTI.Vec (Inst (AVTI.Vec MQPR:$addr), (i32 imm:$offset),
6258                        ARMVCCThen, VCCR:$pred))>;
6259  }
6260}
6261multiclass MVE_VSTR_qi<MVE_memsz memsz, MVEVectorVTInfo AVTI,
6262                       list<MVEVectorVTInfo> DVTIs> {
6263  defm "" : MVE_VLDRSTR_qi_m<MVE_st, memsz, "vstr" # memsz.MnemonicLetter,
6264                             !cast<string>(memsz.TypeBits)>;
6265  defvar Inst = !cast<Instruction>(NAME);
6266  defvar InstPre = !cast<Instruction>(NAME # "_pre");
6267
6268  foreach DVTI = DVTIs in {
6269    def : Pat<(int_arm_mve_vstr_scatter_base
6270                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data)),
6271              (Inst (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6272                    (i32 imm:$offset))>;
6273    def : Pat<(int_arm_mve_vstr_scatter_base_predicated
6274                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data), (AVTI.Pred VCCR:$pred)),
6275              (Inst (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6276                    (i32 imm:$offset), ARMVCCThen, VCCR:$pred)>;
6277    def : Pat<(AVTI.Vec (int_arm_mve_vstr_scatter_base_wb
6278                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data))),
6279              (AVTI.Vec (InstPre (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6280                                 (i32 imm:$offset)))>;
6281    def : Pat<(AVTI.Vec (int_arm_mve_vstr_scatter_base_wb_predicated
6282                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data), (AVTI.Pred VCCR:$pred))),
6283              (AVTI.Vec (InstPre (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6284                                 (i32 imm:$offset), ARMVCCThen, VCCR:$pred))>;
6285  }
6286}
6287
6288// Actual instruction definitions.
6289defm MVE_VLDRWU32_qi: MVE_VLDR_qi<MVE_memW, MVE_v4i32, [MVE_v4i32,MVE_v4f32]>;
6290defm MVE_VLDRDU64_qi: MVE_VLDR_qi<MVE_memD, MVE_v2i64, [MVE_v2i64,MVE_v2f64]>;
6291defm MVE_VSTRW32_qi:  MVE_VSTR_qi<MVE_memW, MVE_v4i32, [MVE_v4i32,MVE_v4f32]>;
6292defm MVE_VSTRD64_qi:  MVE_VSTR_qi<MVE_memD, MVE_v2i64, [MVE_v2i64,MVE_v2f64]>;
6293
6294// Define aliases for all the instructions where memory size and
6295// vector lane size are the same. These are mnemonic aliases, so they
6296// apply consistently across all of the above families - contiguous
6297// loads, and both the rq and qi types of gather/scatter.
6298//
6299// Rationale: As long as you're loading (for example) 16-bit memory
6300// values into 16-bit vector lanes, you can think of them as signed or
6301// unsigned integers, fp16 or just raw 16-bit blobs and it makes no
6302// difference. So we permit all of vldrh.16, vldrh.u16, vldrh.s16,
6303// vldrh.f16 and treat them all as equivalent to the canonical
6304// spelling (which happens to be .u16 for loads, and just .16 for
6305// stores).
6306
6307foreach vpt_cond = ["", "t", "e"] in
6308foreach memsz = [MVE_memB, MVE_memH, MVE_memW, MVE_memD] in
6309foreach suffix = memsz.suffixes in {
6310  // Define an alias with every suffix in the list, except for the one
6311  // used by the real Instruction record (i.e. the one that all the
6312  // rest are aliases *for*).
6313
6314  if !ne(suffix, memsz.CanonLoadSuffix) then {
6315    def : MnemonicAlias<
6316      "vldr" # memsz.MnemonicLetter # vpt_cond # suffix,
6317      "vldr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonLoadSuffix>;
6318  }
6319
6320  if !ne(suffix, memsz.CanonStoreSuffix) then {
6321    def : MnemonicAlias<
6322      "vstr" # memsz.MnemonicLetter # vpt_cond # suffix,
6323      "vstr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonStoreSuffix>;
6324  }
6325}
6326
6327// end of MVE predicable load/store
6328
6329class MVE_VPT<string suffix, bits<2> size, dag iops, string asm, list<dag> pattern=[]>
6330  : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm, "", pattern> {
6331  bits<3> fc;
6332  bits<4> Mk;
6333  bits<3> Qn;
6334
6335  let Inst{31-23} = 0b111111100;
6336  let Inst{22} = Mk{3};
6337  let Inst{21-20} = size;
6338  let Inst{19-17} = Qn{2-0};
6339  let Inst{16} = 0b1;
6340  let Inst{15-13} = Mk{2-0};
6341  let Inst{12} = fc{2};
6342  let Inst{11-8} = 0b1111;
6343  let Inst{7} = fc{0};
6344  let Inst{4} = 0b0;
6345
6346  let Defs = [VPR];
6347}
6348
6349class MVE_VPTt1<string suffix, bits<2> size, dag iops>
6350  : MVE_VPT<suffix, size, iops, "$fc, $Qn, $Qm"> {
6351  bits<4> Qm;
6352  bits<4> Mk;
6353
6354  let Inst{6} = 0b0;
6355  let Inst{5} = Qm{3};
6356  let Inst{3-1} = Qm{2-0};
6357  let Inst{0} = fc{1};
6358}
6359
6360class MVE_VPTt1i<string suffix, bits<2> size>
6361 : MVE_VPTt1<suffix, size,
6362           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_i:$fc)> {
6363  let Inst{12} = 0b0;
6364  let Inst{0} = 0b0;
6365}
6366
6367def MVE_VPTv4i32 : MVE_VPTt1i<"i32", 0b10>;
6368def MVE_VPTv8i16 : MVE_VPTt1i<"i16", 0b01>;
6369def MVE_VPTv16i8 : MVE_VPTt1i<"i8", 0b00>;
6370
6371class MVE_VPTt1u<string suffix, bits<2> size>
6372 : MVE_VPTt1<suffix, size,
6373           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_u:$fc)> {
6374  let Inst{12} = 0b0;
6375  let Inst{0} = 0b1;
6376}
6377
6378def MVE_VPTv4u32 : MVE_VPTt1u<"u32", 0b10>;
6379def MVE_VPTv8u16 : MVE_VPTt1u<"u16", 0b01>;
6380def MVE_VPTv16u8 : MVE_VPTt1u<"u8", 0b00>;
6381
6382class MVE_VPTt1s<string suffix, bits<2> size>
6383 : MVE_VPTt1<suffix, size,
6384           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_s:$fc)> {
6385  let Inst{12} = 0b1;
6386}
6387
6388def MVE_VPTv4s32 : MVE_VPTt1s<"s32", 0b10>;
6389def MVE_VPTv8s16 : MVE_VPTt1s<"s16", 0b01>;
6390def MVE_VPTv16s8 : MVE_VPTt1s<"s8", 0b00>;
6391
6392class MVE_VPTt2<string suffix, bits<2> size, dag iops>
6393  : MVE_VPT<suffix, size, iops,
6394          "$fc, $Qn, $Rm"> {
6395  bits<4> Rm;
6396  bits<3> fc;
6397  bits<4> Mk;
6398
6399  let Inst{6} = 0b1;
6400  let Inst{5} = fc{1};
6401  let Inst{3-0} = Rm{3-0};
6402}
6403
6404class MVE_VPTt2i<string suffix, bits<2> size>
6405  : MVE_VPTt2<suffix, size,
6406            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_i:$fc)> {
6407  let Inst{12} = 0b0;
6408  let Inst{5} = 0b0;
6409}
6410
6411def MVE_VPTv4i32r : MVE_VPTt2i<"i32", 0b10>;
6412def MVE_VPTv8i16r : MVE_VPTt2i<"i16", 0b01>;
6413def MVE_VPTv16i8r : MVE_VPTt2i<"i8", 0b00>;
6414
6415class MVE_VPTt2u<string suffix, bits<2> size>
6416  : MVE_VPTt2<suffix, size,
6417            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_u:$fc)> {
6418  let Inst{12} = 0b0;
6419  let Inst{5} = 0b1;
6420}
6421
6422def MVE_VPTv4u32r : MVE_VPTt2u<"u32", 0b10>;
6423def MVE_VPTv8u16r : MVE_VPTt2u<"u16", 0b01>;
6424def MVE_VPTv16u8r : MVE_VPTt2u<"u8", 0b00>;
6425
6426class MVE_VPTt2s<string suffix, bits<2> size>
6427  : MVE_VPTt2<suffix, size,
6428            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_s:$fc)> {
6429  let Inst{12} = 0b1;
6430}
6431
6432def MVE_VPTv4s32r : MVE_VPTt2s<"s32", 0b10>;
6433def MVE_VPTv8s16r : MVE_VPTt2s<"s16", 0b01>;
6434def MVE_VPTv16s8r : MVE_VPTt2s<"s8", 0b00>;
6435
6436
6437class MVE_VPTf<string suffix, bit size, dag iops, string asm, list<dag> pattern=[]>
6438  : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm,
6439            "", pattern> {
6440  bits<3> fc;
6441  bits<4> Mk;
6442  bits<3> Qn;
6443
6444  let Inst{31-29} = 0b111;
6445  let Inst{28} = size;
6446  let Inst{27-23} = 0b11100;
6447  let Inst{22} = Mk{3};
6448  let Inst{21-20} = 0b11;
6449  let Inst{19-17} = Qn{2-0};
6450  let Inst{16} = 0b1;
6451  let Inst{15-13} = Mk{2-0};
6452  let Inst{12} = fc{2};
6453  let Inst{11-8} = 0b1111;
6454  let Inst{7} = fc{0};
6455  let Inst{4} = 0b0;
6456
6457  let Defs = [VPR];
6458  let Predicates = [HasMVEFloat];
6459}
6460
6461class MVE_VPTft1<string suffix, bit size>
6462  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_fp:$fc),
6463          "$fc, $Qn, $Qm"> {
6464  bits<3> fc;
6465  bits<4> Qm;
6466
6467  let Inst{6} = 0b0;
6468  let Inst{5} = Qm{3};
6469  let Inst{3-1} = Qm{2-0};
6470  let Inst{0} = fc{1};
6471}
6472
6473def MVE_VPTv4f32         : MVE_VPTft1<"f32", 0b0>;
6474def MVE_VPTv8f16         : MVE_VPTft1<"f16", 0b1>;
6475
6476class MVE_VPTft2<string suffix, bit size>
6477  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_fp:$fc),
6478          "$fc, $Qn, $Rm"> {
6479  bits<3> fc;
6480  bits<4> Rm;
6481
6482  let Inst{6} = 0b1;
6483  let Inst{5} = fc{1};
6484  let Inst{3-0} = Rm{3-0};
6485}
6486
6487def MVE_VPTv4f32r        : MVE_VPTft2<"f32", 0b0>;
6488def MVE_VPTv8f16r        : MVE_VPTft2<"f16", 0b1>;
6489
6490def MVE_VPST : MVE_MI<(outs ), (ins vpt_mask:$Mk), NoItinerary,
6491       !strconcat("vpst", "${Mk}"), "", "", []> {
6492  bits<4> Mk;
6493
6494  let Inst{31-23} = 0b111111100;
6495  let Inst{22} = Mk{3};
6496  let Inst{21-16} = 0b110001;
6497  let Inst{15-13} = Mk{2-0};
6498  let Inst{12-0} = 0b0111101001101;
6499  let Unpredictable{12} = 0b1;
6500  let Unpredictable{7} = 0b1;
6501  let Unpredictable{5} = 0b1;
6502
6503  let Uses = [VPR];
6504  let validForTailPredication = 1;
6505}
6506
6507def MVE_VPSEL : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
6508                      "vpsel", "", "$Qd, $Qn, $Qm", vpred_n, "", []> {
6509  bits<4> Qn;
6510  bits<4> Qd;
6511  bits<4> Qm;
6512
6513  let Inst{28} = 0b1;
6514  let Inst{25-23} = 0b100;
6515  let Inst{22} = Qd{3};
6516  let Inst{21-20} = 0b11;
6517  let Inst{19-17} = Qn{2-0};
6518  let Inst{16} = 0b1;
6519  let Inst{15-13} = Qd{2-0};
6520  let Inst{12-9} = 0b0111;
6521  let Inst{8} = 0b1;
6522  let Inst{7} = Qn{3};
6523  let Inst{6} = 0b0;
6524  let Inst{5} = Qm{3};
6525  let Inst{4} = 0b0;
6526  let Inst{3-1} = Qm{2-0};
6527  let Inst{0} = 0b1;
6528}
6529
6530foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32",
6531                  "i8", "i16", "i32",       "f16", "f32"] in
6532def : MVEInstAlias<"vpsel${vp}." # suffix # "\t$Qd, $Qn, $Qm",
6533                   (MVE_VPSEL MQPR:$Qd, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
6534
6535let Predicates = [HasMVEInt] in {
6536  def : Pat<(v16i8 (vselect (v16i1 VCCR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),
6537            (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred))>;
6538  def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred), (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),
6539            (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred))>;
6540  def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),
6541            (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred))>;
6542
6543  def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),
6544            (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred))>;
6545  def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),
6546            (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred))>;
6547
6548  def : Pat<(v16i8 (vselect (v16i8 MQPR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),
6549            (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6550                              (MVE_VCMPi8 (v16i8 MQPR:$pred), (MVE_VMOVimmi8 0), ARMCCne)))>;
6551  def : Pat<(v8i16 (vselect (v8i16 MQPR:$pred), (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),
6552            (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6553                              (MVE_VCMPi16 (v8i16 MQPR:$pred), (MVE_VMOVimmi16 0), ARMCCne)))>;
6554  def : Pat<(v4i32 (vselect (v4i32 MQPR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),
6555            (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6556                              (MVE_VCMPi32 (v4i32 MQPR:$pred), (MVE_VMOVimmi32 0), ARMCCne)))>;
6557
6558  def : Pat<(v8f16 (vselect (v8i16 MQPR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),
6559            (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6560                              (MVE_VCMPi16 (v8i16 MQPR:$pred), (MVE_VMOVimmi16 0), ARMCCne)))>;
6561  def : Pat<(v4f32 (vselect (v4i32 MQPR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),
6562            (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6563                              (MVE_VCMPi32 (v4i32 MQPR:$pred), (MVE_VMOVimmi32 0), ARMCCne)))>;
6564
6565  // Pred <-> Int
6566  def : Pat<(v16i8 (zext  (v16i1 VCCR:$pred))),
6567            (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred))>;
6568  def : Pat<(v8i16 (zext  (v8i1  VCCR:$pred))),
6569            (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred))>;
6570  def : Pat<(v4i32 (zext  (v4i1  VCCR:$pred))),
6571            (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred))>;
6572
6573  def : Pat<(v16i8 (sext  (v16i1 VCCR:$pred))),
6574            (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred))>;
6575  def : Pat<(v8i16 (sext  (v8i1  VCCR:$pred))),
6576            (v8i16 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred))>;
6577  def : Pat<(v4i32 (sext  (v4i1  VCCR:$pred))),
6578            (v4i32 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred))>;
6579
6580  def : Pat<(v16i8 (anyext  (v16i1 VCCR:$pred))),
6581            (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred))>;
6582  def : Pat<(v8i16 (anyext  (v8i1  VCCR:$pred))),
6583            (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred))>;
6584  def : Pat<(v4i32 (anyext  (v4i1  VCCR:$pred))),
6585            (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred))>;
6586
6587  def : Pat<(v16i1 (trunc (v16i8 MQPR:$v1))),
6588            (v16i1 (MVE_VCMPi32r (v16i8 MQPR:$v1), ZR, ARMCCne))>;
6589  def : Pat<(v8i1 (trunc (v8i16  MQPR:$v1))),
6590            (v8i1 (MVE_VCMPi32r (v8i16 MQPR:$v1), ZR, ARMCCne))>;
6591  def : Pat<(v4i1 (trunc (v4i32  MQPR:$v1))),
6592            (v4i1 (MVE_VCMPi32r (v4i32 MQPR:$v1), ZR, ARMCCne))>;
6593}
6594
6595let Predicates = [HasMVEFloat] in {
6596  // Pred <-> Float
6597  // 112 is 1.0 in float
6598  def : Pat<(v4f32 (uint_to_fp (v4i1 VCCR:$pred))),
6599            (v4f32 (MVE_VPSEL (v4f32 (MVE_VMOVimmf32 112)), (v4f32 (MVE_VMOVimmi32 0)), ARMVCCNone, VCCR:$pred))>;
6600  // 2620 in 1.0 in half
6601  def : Pat<(v8f16 (uint_to_fp (v8i1 VCCR:$pred))),
6602            (v8f16 (MVE_VPSEL (v8f16 (MVE_VMOVimmi16 2620)), (v8f16 (MVE_VMOVimmi16 0)), ARMVCCNone, VCCR:$pred))>;
6603  // 240 is -1.0 in float
6604  def : Pat<(v4f32 (sint_to_fp (v4i1 VCCR:$pred))),
6605            (v4f32 (MVE_VPSEL (v4f32 (MVE_VMOVimmf32 240)), (v4f32 (MVE_VMOVimmi32 0)), ARMVCCNone, VCCR:$pred))>;
6606  // 2748 is -1.0 in half
6607  def : Pat<(v8f16 (sint_to_fp (v8i1 VCCR:$pred))),
6608            (v8f16 (MVE_VPSEL (v8f16 (MVE_VMOVimmi16 2748)), (v8f16 (MVE_VMOVimmi16 0)), ARMVCCNone, VCCR:$pred))>;
6609
6610  def : Pat<(v4i1 (fp_to_uint (v4f32 MQPR:$v1))),
6611            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, ARMCCne))>;
6612  def : Pat<(v8i1 (fp_to_uint (v8f16 MQPR:$v1))),
6613            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, ARMCCne))>;
6614  def : Pat<(v4i1 (fp_to_sint (v4f32 MQPR:$v1))),
6615            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, ARMCCne))>;
6616  def : Pat<(v8i1 (fp_to_sint (v8f16 MQPR:$v1))),
6617            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, ARMCCne))>;
6618}
6619
6620def MVE_VPNOT : MVE_p<(outs VCCR:$P0), (ins VCCR:$P0_in), NoItinerary,
6621                      "vpnot", "", "", vpred_n, "", []> {
6622  let Inst{31-0} = 0b11111110001100010000111101001101;
6623  let Unpredictable{19-17} = 0b111;
6624  let Unpredictable{12} = 0b1;
6625  let Unpredictable{7} = 0b1;
6626  let Unpredictable{5} = 0b1;
6627
6628  let Constraints = "";
6629  let DecoderMethod = "DecodeMVEVPNOT";
6630}
6631
6632let Predicates = [HasMVEInt] in {
6633  def : Pat<(v4i1 (xor (v4i1 VCCR:$pred), (v4i1 (predicate_cast (i32 65535))))),
6634            (v4i1 (MVE_VPNOT (v4i1 VCCR:$pred)))>;
6635  def : Pat<(v8i1 (xor (v8i1 VCCR:$pred), (v8i1 (predicate_cast (i32 65535))))),
6636            (v8i1 (MVE_VPNOT (v8i1 VCCR:$pred)))>;
6637  def : Pat<(v16i1 (xor (v16i1 VCCR:$pred), (v16i1 (predicate_cast (i32 65535))))),
6638            (v16i1 (MVE_VPNOT (v16i1 VCCR:$pred)))>;
6639}
6640
6641
6642class MVE_loltp_start<dag iops, string asm, string ops, bits<2> size>
6643  : t2LOL<(outs GPRlr:$LR), iops, asm, ops> {
6644  bits<4> Rn;
6645  let Predicates = [HasMVEInt];
6646  let Inst{22} = 0b0;
6647  let Inst{21-20} = size;
6648  let Inst{19-16} = Rn{3-0};
6649  let Inst{12} = 0b0;
6650}
6651
6652class MVE_DLSTP<string asm, bits<2> size>
6653  : MVE_loltp_start<(ins rGPR:$Rn), asm, "$LR, $Rn", size> {
6654  let Inst{13} = 0b1;
6655  let Inst{11-1} = 0b00000000000;
6656  let Unpredictable{10-1} = 0b1111111111;
6657}
6658
6659class MVE_WLSTP<string asm, bits<2> size>
6660  : MVE_loltp_start<(ins rGPR:$Rn, wlslabel_u11:$label),
6661                    asm, "$LR, $Rn, $label", size> {
6662  bits<11> label;
6663  let Inst{13} = 0b0;
6664  let Inst{11} = label{0};
6665  let Inst{10-1} = label{10-1};
6666  let isBranch = 1;
6667  let isTerminator = 1;
6668}
6669
6670def MVE_DLSTP_8  : MVE_DLSTP<"dlstp.8",  0b00>;
6671def MVE_DLSTP_16 : MVE_DLSTP<"dlstp.16", 0b01>;
6672def MVE_DLSTP_32 : MVE_DLSTP<"dlstp.32", 0b10>;
6673def MVE_DLSTP_64 : MVE_DLSTP<"dlstp.64", 0b11>;
6674
6675def MVE_WLSTP_8  : MVE_WLSTP<"wlstp.8",  0b00>;
6676def MVE_WLSTP_16 : MVE_WLSTP<"wlstp.16", 0b01>;
6677def MVE_WLSTP_32 : MVE_WLSTP<"wlstp.32", 0b10>;
6678def MVE_WLSTP_64 : MVE_WLSTP<"wlstp.64", 0b11>;
6679
6680class MVE_loltp_end<dag oops, dag iops, string asm, string ops>
6681  : t2LOL<oops, iops, asm, ops> {
6682  let Predicates = [HasMVEInt];
6683  let Inst{22-21} = 0b00;
6684  let Inst{19-16} = 0b1111;
6685  let Inst{12} = 0b0;
6686}
6687
6688def MVE_LETP : MVE_loltp_end<(outs GPRlr:$LRout),
6689                             (ins GPRlr:$LRin, lelabel_u11:$label),
6690                             "letp", "$LRin, $label"> {
6691  bits<11> label;
6692  let Inst{20} = 0b1;
6693  let Inst{13} = 0b0;
6694  let Inst{11} = label{0};
6695  let Inst{10-1} = label{10-1};
6696  let isBranch = 1;
6697  let isTerminator = 1;
6698}
6699
6700def MVE_LCTP : MVE_loltp_end<(outs), (ins pred:$p), "lctp${p}", ""> {
6701  let Inst{20} = 0b0;
6702  let Inst{13} = 0b1;
6703  let Inst{11-1} = 0b00000000000;
6704  let Unpredictable{21-20} = 0b11;
6705  let Unpredictable{11-1} = 0b11111111111;
6706}
6707
6708
6709//===----------------------------------------------------------------------===//
6710// Patterns
6711//===----------------------------------------------------------------------===//
6712
6713// PatFrags for loads and stores. Often trying to keep semi-consistent names.
6714
6715def aligned32_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
6716                                  (pre_store node:$val, node:$ptr, node:$offset), [{
6717  return cast<StoreSDNode>(N)->getAlignment() >= 4;
6718}]>;
6719def aligned32_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
6720                                   (post_store node:$val, node:$ptr, node:$offset), [{
6721  return cast<StoreSDNode>(N)->getAlignment() >= 4;
6722}]>;
6723def aligned16_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
6724                                  (pre_store node:$val, node:$ptr, node:$offset), [{
6725  return cast<StoreSDNode>(N)->getAlignment() >= 2;
6726}]>;
6727def aligned16_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
6728                                   (post_store node:$val, node:$ptr, node:$offset), [{
6729  return cast<StoreSDNode>(N)->getAlignment() >= 2;
6730}]>;
6731
6732
6733def aligned_maskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6734                                    (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{
6735  auto *Ld = cast<MaskedLoadSDNode>(N);
6736  return Ld->getMemoryVT().getScalarType() == MVT::i8;
6737}]>;
6738def aligned_sextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6739                                        (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{
6740  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
6741}]>;
6742def aligned_zextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6743                                        (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{
6744  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
6745}]>;
6746def aligned_extmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6747                                       (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{
6748  auto *Ld = cast<MaskedLoadSDNode>(N);
6749  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
6750  return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD;
6751}]>;
6752def aligned_maskedloadvi16: PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6753                                    (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{
6754  auto *Ld = cast<MaskedLoadSDNode>(N);
6755  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
6756  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && Ld->getAlignment() >= 2;
6757}]>;
6758def aligned_sextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6759                                         (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{
6760  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
6761}]>;
6762def aligned_zextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6763                                         (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{
6764  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
6765}]>;
6766def aligned_extmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6767                                        (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{
6768  auto *Ld = cast<MaskedLoadSDNode>(N);
6769  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
6770  return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD;
6771}]>;
6772def aligned_maskedloadvi32: PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
6773                                    (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{
6774  auto *Ld = cast<MaskedLoadSDNode>(N);
6775  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
6776  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && Ld->getAlignment() >= 4;
6777}]>;
6778
6779def aligned_maskedstvi8 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
6780                                  (masked_st node:$val, node:$ptr, undef, node:$pred), [{
6781  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
6782}]>;
6783def aligned_maskedstvi16 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
6784                                   (masked_st node:$val, node:$ptr, undef, node:$pred), [{
6785  auto *St = cast<MaskedStoreSDNode>(N);
6786  EVT ScalarVT = St->getMemoryVT().getScalarType();
6787  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
6788}]>;
6789def aligned_maskedstvi32 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
6790                                   (masked_st node:$val, node:$ptr, undef, node:$pred), [{
6791  auto *St = cast<MaskedStoreSDNode>(N);
6792  EVT ScalarVT = St->getMemoryVT().getScalarType();
6793  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4;
6794}]>;
6795
6796def pre_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask),
6797                              (masked_st node:$val, node:$base, node:$offset, node:$mask), [{
6798  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
6799  return AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
6800}]>;
6801def post_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask),
6802                               (masked_st node:$val, node:$base, node:$offset, node:$mask), [{
6803  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
6804  return AM == ISD::POST_INC || AM == ISD::POST_DEC;
6805}]>;
6806def aligned_pre_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
6807                                         (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
6808  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
6809}]>;
6810def aligned_post_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
6811                                          (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
6812  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
6813}]>;
6814def aligned_pre_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
6815                                          (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
6816  auto *St = cast<MaskedStoreSDNode>(N);
6817  EVT ScalarVT = St->getMemoryVT().getScalarType();
6818  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
6819}]>;
6820def aligned_post_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
6821                                           (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
6822  auto *St = cast<MaskedStoreSDNode>(N);
6823  EVT ScalarVT = St->getMemoryVT().getScalarType();
6824  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
6825}]>;
6826def aligned_pre_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
6827                                          (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
6828  auto *St = cast<MaskedStoreSDNode>(N);
6829  EVT ScalarVT = St->getMemoryVT().getScalarType();
6830  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4;
6831}]>;
6832def aligned_post_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
6833                                           (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
6834  auto *St = cast<MaskedStoreSDNode>(N);
6835  EVT ScalarVT = St->getMemoryVT().getScalarType();
6836  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4;
6837}]>;
6838
6839
6840// PatFrags for "Aligned" extending / truncating
6841
6842def aligned_extloadvi8  : PatFrag<(ops node:$ptr), (extloadvi8 node:$ptr)>;
6843def aligned_sextloadvi8 : PatFrag<(ops node:$ptr), (sextloadvi8 node:$ptr)>;
6844def aligned_zextloadvi8 : PatFrag<(ops node:$ptr), (zextloadvi8 node:$ptr)>;
6845
6846def aligned_truncstvi8 : PatFrag<(ops node:$val, node:$ptr),
6847                                 (truncstorevi8 node:$val, node:$ptr)>;
6848def aligned_post_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),
6849                                      (post_truncstvi8 node:$val, node:$base, node:$offset)>;
6850def aligned_pre_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),
6851                                     (pre_truncstvi8 node:$val, node:$base, node:$offset)>;
6852
6853let MinAlignment = 2 in {
6854  def aligned_extloadvi16  : PatFrag<(ops node:$ptr), (extloadvi16 node:$ptr)>;
6855  def aligned_sextloadvi16 : PatFrag<(ops node:$ptr), (sextloadvi16 node:$ptr)>;
6856  def aligned_zextloadvi16 : PatFrag<(ops node:$ptr), (zextloadvi16 node:$ptr)>;
6857
6858  def aligned_truncstvi16 : PatFrag<(ops node:$val, node:$ptr),
6859                                    (truncstorevi16 node:$val, node:$ptr)>;
6860  def aligned_post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
6861                                         (post_truncstvi16 node:$val, node:$base, node:$offset)>;
6862  def aligned_pre_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
6863                                        (pre_truncstvi16 node:$val, node:$base, node:$offset)>;
6864}
6865
6866def truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$pred),
6867                            (masked_st node:$val, node:$base, undef, node:$pred), [{
6868  return cast<MaskedStoreSDNode>(N)->isTruncatingStore();
6869}]>;
6870def aligned_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$pred),
6871                                       (truncmaskedst node:$val, node:$base, node:$pred), [{
6872  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
6873}]>;
6874def aligned_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$pred),
6875                                        (truncmaskedst node:$val, node:$base, node:$pred), [{
6876  auto *St = cast<MaskedStoreSDNode>(N);
6877  EVT ScalarVT = St->getMemoryVT().getScalarType();
6878  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
6879}]>;
6880def pre_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),
6881                                (masked_st node:$val, node:$base, node:$offset, node:$pred), [{
6882  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
6883  return cast<MaskedStoreSDNode>(N)->isTruncatingStore() && (AM == ISD::PRE_INC || AM == ISD::PRE_DEC);
6884}]>;
6885def aligned_pre_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),
6886                                           (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{
6887  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
6888}]>;
6889def aligned_pre_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),
6890                                            (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{
6891  auto *St = cast<MaskedStoreSDNode>(N);
6892  EVT ScalarVT = St->getMemoryVT().getScalarType();
6893  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
6894}]>;
6895def post_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),
6896                                 (masked_st node:$val, node:$base, node:$offset, node:$postd), [{
6897  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
6898  return cast<MaskedStoreSDNode>(N)->isTruncatingStore() && (AM == ISD::POST_INC || AM == ISD::POST_DEC);
6899}]>;
6900def aligned_post_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),
6901                                            (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{
6902  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
6903}]>;
6904def aligned_post_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),
6905                                             (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{
6906  auto *St = cast<MaskedStoreSDNode>(N);
6907  EVT ScalarVT = St->getMemoryVT().getScalarType();
6908  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
6909}]>;
6910
6911// Load/store patterns
6912
6913class MVE_vector_store_typed<ValueType Ty, Instruction RegImmInst,
6914                             PatFrag StoreKind, int shift>
6915  : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr),
6916        (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr)>;
6917
6918class MVE_vector_maskedstore_typed<ValueType Ty, Instruction RegImmInst,
6919                                   PatFrag StoreKind, int shift>
6920  : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, VCCR:$pred),
6921        (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred)>;
6922
6923multiclass MVE_vector_store<Instruction RegImmInst, PatFrag StoreKind,
6924                            int shift> {
6925  def : MVE_vector_store_typed<v16i8, RegImmInst, StoreKind, shift>;
6926  def : MVE_vector_store_typed<v8i16, RegImmInst, StoreKind, shift>;
6927  def : MVE_vector_store_typed<v8f16, RegImmInst, StoreKind, shift>;
6928  def : MVE_vector_store_typed<v4i32, RegImmInst, StoreKind, shift>;
6929  def : MVE_vector_store_typed<v4f32, RegImmInst, StoreKind, shift>;
6930  def : MVE_vector_store_typed<v2i64, RegImmInst, StoreKind, shift>;
6931  def : MVE_vector_store_typed<v2f64, RegImmInst, StoreKind, shift>;
6932}
6933
6934class MVE_vector_load_typed<ValueType Ty, Instruction RegImmInst,
6935                            PatFrag LoadKind, int shift>
6936  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr)),
6937        (Ty (RegImmInst t2addrmode_imm7<shift>:$addr))>;
6938
6939class MVE_vector_maskedload_typed<ValueType Ty, Instruction RegImmInst,
6940                                  PatFrag LoadKind, int shift>
6941  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr, VCCR:$pred, (Ty NEONimmAllZerosV))),
6942        (Ty (RegImmInst t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred))>;
6943
6944multiclass MVE_vector_load<Instruction RegImmInst, PatFrag LoadKind,
6945                           int shift> {
6946  def : MVE_vector_load_typed<v16i8, RegImmInst, LoadKind, shift>;
6947  def : MVE_vector_load_typed<v8i16, RegImmInst, LoadKind, shift>;
6948  def : MVE_vector_load_typed<v8f16, RegImmInst, LoadKind, shift>;
6949  def : MVE_vector_load_typed<v4i32, RegImmInst, LoadKind, shift>;
6950  def : MVE_vector_load_typed<v4f32, RegImmInst, LoadKind, shift>;
6951  def : MVE_vector_load_typed<v2i64, RegImmInst, LoadKind, shift>;
6952  def : MVE_vector_load_typed<v2f64, RegImmInst, LoadKind, shift>;
6953}
6954
6955class MVE_vector_offset_store_typed<ValueType Ty, Instruction Opcode,
6956                                    PatFrag StoreKind, int shift>
6957  : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<shift>:$addr),
6958        (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<shift>:$addr)>;
6959
6960class MVE_vector_offset_maskedstore_typed<ValueType Ty, Instruction Opcode,
6961                                          PatFrag StoreKind, int shift>
6962  : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<shift>:$addr, VCCR:$pred),
6963        (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<shift>:$addr, ARMVCCThen, VCCR:$pred)>;
6964
6965multiclass MVE_vector_offset_store<Instruction RegImmInst, PatFrag StoreKind,
6966                                   int shift> {
6967  def : MVE_vector_offset_store_typed<v16i8, RegImmInst, StoreKind, shift>;
6968  def : MVE_vector_offset_store_typed<v8i16, RegImmInst, StoreKind, shift>;
6969  def : MVE_vector_offset_store_typed<v8f16, RegImmInst, StoreKind, shift>;
6970  def : MVE_vector_offset_store_typed<v4i32, RegImmInst, StoreKind, shift>;
6971  def : MVE_vector_offset_store_typed<v4f32, RegImmInst, StoreKind, shift>;
6972  def : MVE_vector_offset_store_typed<v2i64, RegImmInst, StoreKind, shift>;
6973  def : MVE_vector_offset_store_typed<v2f64, RegImmInst, StoreKind, shift>;
6974}
6975
6976
6977let Predicates = [HasMVEInt, IsLE] in {
6978  // Stores
6979  defm : MVE_vector_store<MVE_VSTRBU8, byte_alignedstore, 0>;
6980  defm : MVE_vector_store<MVE_VSTRHU16, hword_alignedstore, 1>;
6981  defm : MVE_vector_store<MVE_VSTRWU32, alignedstore32, 2>;
6982
6983  // Loads
6984  defm : MVE_vector_load<MVE_VLDRBU8, byte_alignedload, 0>;
6985  defm : MVE_vector_load<MVE_VLDRHU16, hword_alignedload, 1>;
6986  defm : MVE_vector_load<MVE_VLDRWU32, alignedload32, 2>;
6987
6988  // Pre/post inc stores
6989  defm : MVE_vector_offset_store<MVE_VSTRBU8_pre, pre_store, 0>;
6990  defm : MVE_vector_offset_store<MVE_VSTRBU8_post, post_store, 0>;
6991  defm : MVE_vector_offset_store<MVE_VSTRHU16_pre, aligned16_pre_store, 1>;
6992  defm : MVE_vector_offset_store<MVE_VSTRHU16_post, aligned16_post_store, 1>;
6993  defm : MVE_vector_offset_store<MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
6994  defm : MVE_vector_offset_store<MVE_VSTRWU32_post, aligned32_post_store, 2>;
6995}
6996
6997let Predicates = [HasMVEInt, IsBE] in {
6998  // Aligned Stores
6999  def : MVE_vector_store_typed<v16i8, MVE_VSTRBU8, store, 0>;
7000  def : MVE_vector_store_typed<v8i16, MVE_VSTRHU16, alignedstore16, 1>;
7001  def : MVE_vector_store_typed<v8f16, MVE_VSTRHU16, alignedstore16, 1>;
7002  def : MVE_vector_store_typed<v4i32, MVE_VSTRWU32, alignedstore32, 2>;
7003  def : MVE_vector_store_typed<v4f32, MVE_VSTRWU32, alignedstore32, 2>;
7004
7005  // Aligned Loads
7006  def : MVE_vector_load_typed<v16i8, MVE_VLDRBU8, load, 0>;
7007  def : MVE_vector_load_typed<v8i16, MVE_VLDRHU16, alignedload16, 1>;
7008  def : MVE_vector_load_typed<v8f16, MVE_VLDRHU16, alignedload16, 1>;
7009  def : MVE_vector_load_typed<v4i32, MVE_VLDRWU32, alignedload32, 2>;
7010  def : MVE_vector_load_typed<v4f32, MVE_VLDRWU32, alignedload32, 2>;
7011
7012  // Other unaligned loads/stores need to go though a VREV
7013  def : Pat<(v2f64 (load t2addrmode_imm7<0>:$addr)),
7014            (v2f64 (MVE_VREV64_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7015  def : Pat<(v2i64 (load t2addrmode_imm7<0>:$addr)),
7016            (v2i64 (MVE_VREV64_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7017  def : Pat<(v4i32 (load t2addrmode_imm7<0>:$addr)),
7018            (v4i32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7019  def : Pat<(v4f32 (load t2addrmode_imm7<0>:$addr)),
7020            (v4f32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7021  def : Pat<(v8i16 (load t2addrmode_imm7<0>:$addr)),
7022            (v8i16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7023  def : Pat<(v8f16 (load t2addrmode_imm7<0>:$addr)),
7024            (v8f16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7025  def : Pat<(store (v2f64 MQPR:$val), t2addrmode_imm7<0>:$addr),
7026            (MVE_VSTRBU8 (MVE_VREV64_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7027  def : Pat<(store (v2i64 MQPR:$val), t2addrmode_imm7<0>:$addr),
7028            (MVE_VSTRBU8 (MVE_VREV64_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7029  def : Pat<(store (v4i32 MQPR:$val), t2addrmode_imm7<0>:$addr),
7030            (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7031  def : Pat<(store (v4f32 MQPR:$val), t2addrmode_imm7<0>:$addr),
7032            (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7033  def : Pat<(store (v8i16 MQPR:$val), t2addrmode_imm7<0>:$addr),
7034            (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7035  def : Pat<(store (v8f16 MQPR:$val), t2addrmode_imm7<0>:$addr),
7036            (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7037
7038  // Pre/Post inc stores
7039  def : MVE_vector_offset_store_typed<v16i8, MVE_VSTRBU8_pre, pre_store, 0>;
7040  def : MVE_vector_offset_store_typed<v16i8, MVE_VSTRBU8_post, post_store, 0>;
7041  def : MVE_vector_offset_store_typed<v8i16, MVE_VSTRHU16_pre, aligned16_pre_store, 1>;
7042  def : MVE_vector_offset_store_typed<v8i16, MVE_VSTRHU16_post, aligned16_post_store, 1>;
7043  def : MVE_vector_offset_store_typed<v8f16, MVE_VSTRHU16_pre, aligned16_pre_store, 1>;
7044  def : MVE_vector_offset_store_typed<v8f16, MVE_VSTRHU16_post, aligned16_post_store, 1>;
7045  def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
7046  def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_post, aligned32_post_store, 2>;
7047  def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
7048  def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_post, aligned32_post_store, 2>;
7049}
7050
7051let Predicates = [HasMVEInt] in {
7052  // Aligned masked store, shared between LE and BE
7053  def : MVE_vector_maskedstore_typed<v16i8, MVE_VSTRBU8, aligned_maskedstvi8, 0>;
7054  def : MVE_vector_maskedstore_typed<v8i16, MVE_VSTRHU16, aligned_maskedstvi16, 1>;
7055  def : MVE_vector_maskedstore_typed<v8f16, MVE_VSTRHU16, aligned_maskedstvi16, 1>;
7056  def : MVE_vector_maskedstore_typed<v4i32, MVE_VSTRWU32, aligned_maskedstvi32, 2>;
7057  def : MVE_vector_maskedstore_typed<v4f32, MVE_VSTRWU32, aligned_maskedstvi32, 2>;
7058
7059  // Pre/Post inc masked stores
7060  def : MVE_vector_offset_maskedstore_typed<v16i8, MVE_VSTRBU8_pre, aligned_pre_maskedstorevi8, 0>;
7061  def : MVE_vector_offset_maskedstore_typed<v16i8, MVE_VSTRBU8_post, aligned_post_maskedstorevi8, 0>;
7062  def : MVE_vector_offset_maskedstore_typed<v8i16, MVE_VSTRHU16_pre, aligned_pre_maskedstorevi16, 1>;
7063  def : MVE_vector_offset_maskedstore_typed<v8i16, MVE_VSTRHU16_post, aligned_post_maskedstorevi16, 1>;
7064  def : MVE_vector_offset_maskedstore_typed<v8f16, MVE_VSTRHU16_pre, aligned_pre_maskedstorevi16, 1>;
7065  def : MVE_vector_offset_maskedstore_typed<v8f16, MVE_VSTRHU16_post, aligned_post_maskedstorevi16, 1>;
7066  def : MVE_vector_offset_maskedstore_typed<v4i32, MVE_VSTRWU32_pre, aligned_pre_maskedstorevi32, 2>;
7067  def : MVE_vector_offset_maskedstore_typed<v4i32, MVE_VSTRWU32_post, aligned_post_maskedstorevi32, 2>;
7068  def : MVE_vector_offset_maskedstore_typed<v4f32, MVE_VSTRWU32_pre, aligned_pre_maskedstorevi32, 2>;
7069  def : MVE_vector_offset_maskedstore_typed<v4f32, MVE_VSTRWU32_post, aligned_post_maskedstorevi32, 2>;
7070
7071  // Aligned masked loads
7072  def : MVE_vector_maskedload_typed<v16i8, MVE_VLDRBU8, aligned_maskedloadvi8, 0>;
7073  def : MVE_vector_maskedload_typed<v8i16, MVE_VLDRHU16, aligned_maskedloadvi16, 1>;
7074  def : MVE_vector_maskedload_typed<v8f16, MVE_VLDRHU16, aligned_maskedloadvi16, 1>;
7075  def : MVE_vector_maskedload_typed<v4i32, MVE_VLDRWU32, aligned_maskedloadvi32, 2>;
7076  def : MVE_vector_maskedload_typed<v4f32, MVE_VLDRWU32, aligned_maskedloadvi32, 2>;
7077}
7078
7079// Widening/Narrowing Loads/Stores
7080
7081multiclass MVEExtLoadStore<Instruction LoadSInst, Instruction LoadUInst, string StoreInst,
7082                         string Amble, ValueType VT, int Shift> {
7083  // Trunc stores
7084  def : Pat<(!cast<PatFrag>("aligned_truncst"#Amble) (VT MQPR:$val), taddrmode_imm7<Shift>:$addr),
7085            (!cast<Instruction>(StoreInst) MQPR:$val, taddrmode_imm7<Shift>:$addr)>;
7086  def : Pat<(!cast<PatFrag>("aligned_post_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr),
7087            (!cast<Instruction>(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr)>;
7088  def : Pat<(!cast<PatFrag>("aligned_pre_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr),
7089            (!cast<Instruction>(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr)>;
7090
7091  // Masked trunc stores
7092  def : Pat<(!cast<PatFrag>("aligned_truncmaskedst"#Amble) (VT MQPR:$val), taddrmode_imm7<Shift>:$addr, VCCR:$pred),
7093            (!cast<Instruction>(StoreInst) MQPR:$val, taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred)>;
7094  def : Pat<(!cast<PatFrag>("aligned_post_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, VCCR:$pred),
7095            (!cast<Instruction>(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, ARMVCCThen, VCCR:$pred)>;
7096  def : Pat<(!cast<PatFrag>("aligned_pre_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, VCCR:$pred),
7097            (!cast<Instruction>(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, ARMVCCThen, VCCR:$pred)>;
7098
7099  // Ext loads
7100  def : Pat<(VT (!cast<PatFrag>("aligned_extload"#Amble) taddrmode_imm7<Shift>:$addr)),
7101            (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;
7102  def : Pat<(VT (!cast<PatFrag>("aligned_sextload"#Amble) taddrmode_imm7<Shift>:$addr)),
7103            (VT (LoadSInst taddrmode_imm7<Shift>:$addr))>;
7104  def : Pat<(VT (!cast<PatFrag>("aligned_zextload"#Amble) taddrmode_imm7<Shift>:$addr)),
7105            (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;
7106
7107  // Masked ext loads
7108  def : Pat<(VT (!cast<PatFrag>("aligned_extmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT NEONimmAllZerosV))),
7109            (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred))>;
7110  def : Pat<(VT (!cast<PatFrag>("aligned_sextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT NEONimmAllZerosV))),
7111            (VT (LoadSInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred))>;
7112  def : Pat<(VT (!cast<PatFrag>("aligned_zextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT NEONimmAllZerosV))),
7113            (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred))>;
7114}
7115
7116let Predicates = [HasMVEInt] in {
7117  defm : MVEExtLoadStore<MVE_VLDRBS16, MVE_VLDRBU16, "MVE_VSTRB16", "vi8", v8i16, 0>;
7118  defm : MVEExtLoadStore<MVE_VLDRBS32, MVE_VLDRBU32, "MVE_VSTRB32", "vi8", v4i32, 0>;
7119  defm : MVEExtLoadStore<MVE_VLDRHS32, MVE_VLDRHU32, "MVE_VSTRH32", "vi16", v4i32, 1>;
7120}
7121
7122
7123// Bit convert patterns
7124
7125let Predicates = [HasMVEInt] in {
7126  def : Pat<(v2f64 (bitconvert (v2i64 MQPR:$src))), (v2f64 MQPR:$src)>;
7127  def : Pat<(v2i64 (bitconvert (v2f64 MQPR:$src))), (v2i64 MQPR:$src)>;
7128
7129  def : Pat<(v4i32 (bitconvert (v4f32 MQPR:$src))), (v4i32 MQPR:$src)>;
7130  def : Pat<(v4f32 (bitconvert (v4i32 MQPR:$src))), (v4f32 MQPR:$src)>;
7131
7132  def : Pat<(v8i16 (bitconvert (v8f16 MQPR:$src))), (v8i16  MQPR:$src)>;
7133  def : Pat<(v8f16 (bitconvert (v8i16 MQPR:$src))), (v8f16  MQPR:$src)>;
7134}
7135
7136let Predicates = [IsLE,HasMVEInt] in {
7137  def : Pat<(v2f64 (bitconvert (v4f32 MQPR:$src))), (v2f64 MQPR:$src)>;
7138  def : Pat<(v2f64 (bitconvert (v4i32 MQPR:$src))), (v2f64 MQPR:$src)>;
7139  def : Pat<(v2f64 (bitconvert (v8f16 MQPR:$src))), (v2f64 MQPR:$src)>;
7140  def : Pat<(v2f64 (bitconvert (v8i16 MQPR:$src))), (v2f64 MQPR:$src)>;
7141  def : Pat<(v2f64 (bitconvert (v16i8 MQPR:$src))), (v2f64 MQPR:$src)>;
7142
7143  def : Pat<(v2i64 (bitconvert (v4f32 MQPR:$src))), (v2i64 MQPR:$src)>;
7144  def : Pat<(v2i64 (bitconvert (v4i32 MQPR:$src))), (v2i64 MQPR:$src)>;
7145  def : Pat<(v2i64 (bitconvert (v8f16 MQPR:$src))), (v2i64 MQPR:$src)>;
7146  def : Pat<(v2i64 (bitconvert (v8i16 MQPR:$src))), (v2i64 MQPR:$src)>;
7147  def : Pat<(v2i64 (bitconvert (v16i8 MQPR:$src))), (v2i64 MQPR:$src)>;
7148
7149  def : Pat<(v4f32 (bitconvert (v2f64 MQPR:$src))), (v4f32 MQPR:$src)>;
7150  def : Pat<(v4f32 (bitconvert (v2i64 MQPR:$src))), (v4f32 MQPR:$src)>;
7151  def : Pat<(v4f32 (bitconvert (v8f16 MQPR:$src))), (v4f32 MQPR:$src)>;
7152  def : Pat<(v4f32 (bitconvert (v8i16 MQPR:$src))), (v4f32 MQPR:$src)>;
7153  def : Pat<(v4f32 (bitconvert (v16i8 MQPR:$src))), (v4f32 MQPR:$src)>;
7154
7155  def : Pat<(v4i32 (bitconvert (v2f64 MQPR:$src))), (v4i32 MQPR:$src)>;
7156  def : Pat<(v4i32 (bitconvert (v2i64 MQPR:$src))), (v4i32 MQPR:$src)>;
7157  def : Pat<(v4i32 (bitconvert (v8f16 MQPR:$src))), (v4i32 MQPR:$src)>;
7158  def : Pat<(v4i32 (bitconvert (v8i16 MQPR:$src))), (v4i32 MQPR:$src)>;
7159  def : Pat<(v4i32 (bitconvert (v16i8 MQPR:$src))), (v4i32 MQPR:$src)>;
7160
7161  def : Pat<(v8f16 (bitconvert (v2f64 MQPR:$src))), (v8f16 MQPR:$src)>;
7162  def : Pat<(v8f16 (bitconvert (v2i64 MQPR:$src))), (v8f16 MQPR:$src)>;
7163  def : Pat<(v8f16 (bitconvert (v4f32 MQPR:$src))), (v8f16 MQPR:$src)>;
7164  def : Pat<(v8f16 (bitconvert (v4i32 MQPR:$src))), (v8f16 MQPR:$src)>;
7165  def : Pat<(v8f16 (bitconvert (v16i8 MQPR:$src))), (v8f16 MQPR:$src)>;
7166
7167  def : Pat<(v8i16 (bitconvert (v2f64 MQPR:$src))), (v8i16 MQPR:$src)>;
7168  def : Pat<(v8i16 (bitconvert (v2i64 MQPR:$src))), (v8i16 MQPR:$src)>;
7169  def : Pat<(v8i16 (bitconvert (v4f32 MQPR:$src))), (v8i16 MQPR:$src)>;
7170  def : Pat<(v8i16 (bitconvert (v4i32 MQPR:$src))), (v8i16 MQPR:$src)>;
7171  def : Pat<(v8i16 (bitconvert (v16i8 MQPR:$src))), (v8i16 MQPR:$src)>;
7172
7173  def : Pat<(v16i8 (bitconvert (v2f64 MQPR:$src))), (v16i8 MQPR:$src)>;
7174  def : Pat<(v16i8 (bitconvert (v2i64 MQPR:$src))), (v16i8 MQPR:$src)>;
7175  def : Pat<(v16i8 (bitconvert (v4f32 MQPR:$src))), (v16i8 MQPR:$src)>;
7176  def : Pat<(v16i8 (bitconvert (v4i32 MQPR:$src))), (v16i8 MQPR:$src)>;
7177  def : Pat<(v16i8 (bitconvert (v8f16 MQPR:$src))), (v16i8 MQPR:$src)>;
7178  def : Pat<(v16i8 (bitconvert (v8i16 MQPR:$src))), (v16i8 MQPR:$src)>;
7179}
7180
7181let Predicates = [IsBE,HasMVEInt] in {
7182  def : Pat<(v2f64 (bitconvert (v4f32 MQPR:$src))), (v2f64 (MVE_VREV64_32 MQPR:$src))>;
7183  def : Pat<(v2f64 (bitconvert (v4i32 MQPR:$src))), (v2f64 (MVE_VREV64_32 MQPR:$src))>;
7184  def : Pat<(v2f64 (bitconvert (v8f16 MQPR:$src))), (v2f64 (MVE_VREV64_16 MQPR:$src))>;
7185  def : Pat<(v2f64 (bitconvert (v8i16 MQPR:$src))), (v2f64 (MVE_VREV64_16 MQPR:$src))>;
7186  def : Pat<(v2f64 (bitconvert (v16i8 MQPR:$src))), (v2f64 (MVE_VREV64_8 MQPR:$src))>;
7187
7188  def : Pat<(v2i64 (bitconvert (v4f32 MQPR:$src))), (v2i64 (MVE_VREV64_32 MQPR:$src))>;
7189  def : Pat<(v2i64 (bitconvert (v4i32 MQPR:$src))), (v2i64 (MVE_VREV64_32 MQPR:$src))>;
7190  def : Pat<(v2i64 (bitconvert (v8f16 MQPR:$src))), (v2i64 (MVE_VREV64_16 MQPR:$src))>;
7191  def : Pat<(v2i64 (bitconvert (v8i16 MQPR:$src))), (v2i64 (MVE_VREV64_16 MQPR:$src))>;
7192  def : Pat<(v2i64 (bitconvert (v16i8 MQPR:$src))), (v2i64 (MVE_VREV64_8 MQPR:$src))>;
7193
7194  def : Pat<(v4f32 (bitconvert (v2f64 MQPR:$src))), (v4f32 (MVE_VREV64_32 MQPR:$src))>;
7195  def : Pat<(v4f32 (bitconvert (v2i64 MQPR:$src))), (v4f32 (MVE_VREV64_32 MQPR:$src))>;
7196  def : Pat<(v4f32 (bitconvert (v8f16 MQPR:$src))), (v4f32 (MVE_VREV32_16 MQPR:$src))>;
7197  def : Pat<(v4f32 (bitconvert (v8i16 MQPR:$src))), (v4f32 (MVE_VREV32_16 MQPR:$src))>;
7198  def : Pat<(v4f32 (bitconvert (v16i8 MQPR:$src))), (v4f32 (MVE_VREV32_8 MQPR:$src))>;
7199
7200  def : Pat<(v4i32 (bitconvert (v2f64 MQPR:$src))), (v4i32 (MVE_VREV64_32 MQPR:$src))>;
7201  def : Pat<(v4i32 (bitconvert (v2i64 MQPR:$src))), (v4i32 (MVE_VREV64_32 MQPR:$src))>;
7202  def : Pat<(v4i32 (bitconvert (v8f16 MQPR:$src))), (v4i32 (MVE_VREV32_16 MQPR:$src))>;
7203  def : Pat<(v4i32 (bitconvert (v8i16 MQPR:$src))), (v4i32 (MVE_VREV32_16 MQPR:$src))>;
7204  def : Pat<(v4i32 (bitconvert (v16i8 MQPR:$src))), (v4i32 (MVE_VREV32_8 MQPR:$src))>;
7205
7206  def : Pat<(v8f16 (bitconvert (v2f64 MQPR:$src))), (v8f16 (MVE_VREV64_16 MQPR:$src))>;
7207  def : Pat<(v8f16 (bitconvert (v2i64 MQPR:$src))), (v8f16 (MVE_VREV64_16 MQPR:$src))>;
7208  def : Pat<(v8f16 (bitconvert (v4f32 MQPR:$src))), (v8f16 (MVE_VREV32_16 MQPR:$src))>;
7209  def : Pat<(v8f16 (bitconvert (v4i32 MQPR:$src))), (v8f16 (MVE_VREV32_16 MQPR:$src))>;
7210  def : Pat<(v8f16 (bitconvert (v16i8 MQPR:$src))), (v8f16 (MVE_VREV16_8 MQPR:$src))>;
7211
7212  def : Pat<(v8i16 (bitconvert (v2f64 MQPR:$src))), (v8i16 (MVE_VREV64_16 MQPR:$src))>;
7213  def : Pat<(v8i16 (bitconvert (v2i64 MQPR:$src))), (v8i16 (MVE_VREV64_16 MQPR:$src))>;
7214  def : Pat<(v8i16 (bitconvert (v4f32 MQPR:$src))), (v8i16 (MVE_VREV32_16 MQPR:$src))>;
7215  def : Pat<(v8i16 (bitconvert (v4i32 MQPR:$src))), (v8i16 (MVE_VREV32_16 MQPR:$src))>;
7216  def : Pat<(v8i16 (bitconvert (v16i8 MQPR:$src))), (v8i16 (MVE_VREV16_8 MQPR:$src))>;
7217
7218  def : Pat<(v16i8 (bitconvert (v2f64 MQPR:$src))), (v16i8 (MVE_VREV64_8 MQPR:$src))>;
7219  def : Pat<(v16i8 (bitconvert (v2i64 MQPR:$src))), (v16i8 (MVE_VREV64_8 MQPR:$src))>;
7220  def : Pat<(v16i8 (bitconvert (v4f32 MQPR:$src))), (v16i8 (MVE_VREV32_8 MQPR:$src))>;
7221  def : Pat<(v16i8 (bitconvert (v4i32 MQPR:$src))), (v16i8 (MVE_VREV32_8 MQPR:$src))>;
7222  def : Pat<(v16i8 (bitconvert (v8f16 MQPR:$src))), (v16i8 (MVE_VREV16_8 MQPR:$src))>;
7223  def : Pat<(v16i8 (bitconvert (v8i16 MQPR:$src))), (v16i8 (MVE_VREV16_8 MQPR:$src))>;
7224}
7225