1//===----------- VVPInstrPatternsVec.td - VVP_* SDNode patterns -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file describes how VVP_* SDNodes are lowered to machine instructions.
10//
11//===----------------------------------------------------------------------===//
12
13//===----------------------------------------------------------------------===//
14//
15// VVP SDNode definitions.
16//
17//===----------------------------------------------------------------------===//
18include "VVPInstrInfo.td"
19
20multiclass VectorStore<ValueType DataVT,
21    ValueType PtrVT, ValueType MaskVT,
22    string STWithMask, string STNoMask> {
23  // Unmasked (imm stride).
24  def : Pat<(vvp_store
25               DataVT:$val, PtrVT:$addr,
26               (i64 simm7:$stride), (MaskVT true_mask), i32:$avl),
27            (!cast<Instruction>(STNoMask#"irvl")
28               (LO7 $stride), $addr, $val, $avl)>;
29  // Unmasked.
30  def : Pat<(vvp_store
31               DataVT:$val, PtrVT:$addr,
32               i64:$stride, (MaskVT true_mask), i32:$avl),
33            (!cast<Instruction>(STNoMask#"rrvl")
34               $stride, $addr, $val, $avl)>;
35  // Masked (imm stride).
36  def : Pat<(vvp_store
37               DataVT:$val, PtrVT:$addr,
38               (i64 simm7:$stride), MaskVT:$mask, i32:$avl),
39            (!cast<Instruction>(STWithMask#"irvml")
40               (LO7 $stride), $addr, $val, $mask, $avl)>;
41  // Masked.
42  def : Pat<(vvp_store
43               DataVT:$val, PtrVT:$addr,
44               i64:$stride, MaskVT:$mask, i32:$avl),
45            (!cast<Instruction>(STWithMask#"rrvml")
46               $stride, $addr, $val, $mask, $avl)>;
47}
48
49defm : VectorStore<v256f64, i64, v256i1, "VST",  "VST">;
50defm : VectorStore<v256i64, i64, v256i1, "VST",  "VST">;
51defm : VectorStore<v256f32, i64, v256i1, "VSTU", "VSTU">;
52defm : VectorStore<v256i32, i64, v256i1, "VSTL", "VSTL">;
53
54multiclass VectorLoad<ValueType DataVT,
55    ValueType PtrVT, ValueType MaskVT,
56    string GTWithMask, string LDNoMask> {
57  // Unmasked (imm stride).
58  def : Pat<(DataVT (vvp_load
59               PtrVT:$addr, (i64 simm7:$stride),
60               (MaskVT true_mask), i32:$avl)),
61            (!cast<Instruction>(LDNoMask#"irl")
62               (LO7 $stride), $addr, $avl)>;
63  // Unmasked.
64  def : Pat<(DataVT (vvp_load
65               PtrVT:$addr, i64:$stride,
66               (MaskVT true_mask), i32:$avl)),
67            (!cast<Instruction>(LDNoMask#"rrl")
68               $stride, PtrVT:$addr, $avl)>;
69  // Masked (imm stride).
70  def : Pat<(DataVT (vvp_load
71               PtrVT:$addr, (i64 simm7:$stride),
72               MaskVT:$mask, i32:$avl)),
73            (!cast<Instruction>(GTWithMask#"vizml")
74               (VADDULrvml $addr,
75                  (VMULULivml (LO7 $stride), (VSEQl $avl), $mask, $avl),
76                  $mask, $avl),
77               0, 0,
78               $mask,
79               $avl)>;
80  // Masked.
81  def : Pat<(DataVT (vvp_load
82               PtrVT:$addr, i64:$stride, MaskVT:$mask, i32:$avl)),
83            (!cast<Instruction>(GTWithMask#"vizml")
84               (VADDULrvml $addr,
85                  (VMULULrvml $stride, (VSEQl $avl), $mask, $avl),
86                  $mask, $avl),
87               0, 0,
88               $mask,
89               $avl)>;
90}
91
92defm : VectorLoad<v256f64, i64, v256i1, "VGT",    "VLD">;
93defm : VectorLoad<v256i64, i64, v256i1, "VGT",    "VLD">;
94defm : VectorLoad<v256f32, i64, v256i1, "VGTU",   "VLDU">;
95defm : VectorLoad<v256i32, i64, v256i1, "VGTLZX", "VLDLZX">;
96
97// Vector Gather and scatter
98multiclass VectorGather<ValueType DataVT,
99    ValueType PtrVT, ValueType MaskVT,
100    string GTPrefix> {
101  // Unmasked.
102  def : Pat<(DataVT (vvp_gather
103               PtrVT:$addr, (MaskVT true_mask), i32:$avl)),
104            (!cast<Instruction>(GTPrefix#"vizl") $addr, 0, 0, $avl)>;
105  // Masked.
106  def : Pat<(DataVT (vvp_gather PtrVT:$addr, MaskVT:$mask, i32:$avl)),
107            (!cast<Instruction>(GTPrefix#"vizml") $addr, 0, 0, $mask, $avl)>;
108}
109
110defm : VectorGather<v256f64, v256i64, v256i1, "VGT">;
111defm : VectorGather<v256i64, v256i64, v256i1, "VGT">;
112defm : VectorGather<v256f32, v256i64, v256i1, "VGTU">;
113defm : VectorGather<v256i32, v256i64, v256i1, "VGTLZX">;
114
115multiclass VectorScatter<ValueType DataVT,
116    ValueType PtrVT, ValueType MaskVT,
117    string SCPrefix> {
118  // Unmasked.
119  def : Pat<(vvp_scatter
120               DataVT:$data, PtrVT:$addr, (MaskVT true_mask), i32:$avl),
121            (!cast<Instruction>(SCPrefix#"vizvl") $addr, 0, 0, $data, $avl)>;
122  // Masked.
123  def : Pat<(vvp_scatter
124               DataVT:$data, PtrVT:$addr, MaskVT:$mask, i32:$avl),
125            (!cast<Instruction>(SCPrefix#"vizvml") $addr, 0, 0, $data, $mask, $avl)>;
126}
127
128defm : VectorScatter<v256f64, v256i64, v256i1, "VSC">;
129defm : VectorScatter<v256i64, v256i64, v256i1, "VSC">;
130defm : VectorScatter<v256f32, v256i64, v256i1, "VSCU">;
131defm : VectorScatter<v256i32, v256i64, v256i1, "VSCL">;
132
133
134/// FNEG {
135// Directly modify the sign bit to flip the sign.
136
137// Set sign bits in a pack of <2 x f32>.
138def packed_fneg_imm     : OutPatFrag<(ins ),
139                          (i64 (SLLri (i64 (ORim 1, (i32 32))), 31))>;
140
141
142multiclass FNeg<ValueType DataVT> {
143  // Masked with select.
144  def : Pat<(vvp_select (vvp_fneg DataVT:$vx, (v256i1 srcvalue), (i32 srcvalue)),
145                        DataVT:$vfalse,
146                        v256i1:$mask,
147                        i32:$avl),
148            (VXORmvml_v (i32 1), $vx, $mask, $avl, $vfalse)>;
149
150  // Unmasked.
151  def : Pat<(vvp_fneg DataVT:$vx, (v256i1 true_mask), i32:$avl),
152            (VXORmvl (i32 1), $vx, $avl)>;
153
154  // Masked.
155  def : Pat<(vvp_fneg DataVT:$vx, v256i1:$mask, i32:$avl),
156            (VXORmvml (i32 1), $vx, $mask, $avl)>;
157}
158
159defm: FNeg<v256f32>;
160defm: FNeg<v256f64>;
161
162///// Packed FNeg /////
163
164// Masked with select.
165def : Pat<(vvp_select (vvp_fneg v512f32:$vx, (v512i1 srcvalue), (i32 srcvalue)),
166                      v512f32:$vfalse,
167                      v512i1:$mask,
168                      i32:$avl),
169          (v512f32 (PVXORrvml_v (packed_fneg_imm ), $vx, $mask, $avl, $vfalse))>;
170
171// Unmasked.
172def : Pat<(vvp_fneg v512f32:$vx, (v512i1 true_mask), i32:$avl),
173          (v512f32 (PVXORrvl (packed_fneg_imm ), $vx, $avl))>;
174
175// Masked.
176def : Pat<(vvp_fneg v512f32:$vx, v512i1:$mask, i32:$avl),
177          (v512f32 (PVXORrvml (packed_fneg_imm ), $vx, $mask, $avl))>;
178
179/// } FNEG
180
181multiclass Binary_rv<SDPatternOperator OpNode,
182    ValueType ScalarVT, ValueType DataVT,
183    ValueType MaskVT, string OpBaseName> {
184  // Masked with passthru, broadcast.
185  def : Pat<(vvp_select
186                (OpNode
187                    (any_broadcast ScalarVT:$sx),
188                    DataVT:$vy,
189                    (MaskVT srcvalue),
190                    (i32 srcvalue)),
191                DataVT:$vfalse,
192                MaskVT:$mask,
193                i32:$pivot),
194            (!cast<Instruction>(OpBaseName#"rvml_v")
195                ScalarVT:$sx,
196                $vy,
197                $mask,
198                $pivot,
199                $vfalse)>;
200
201  // Unmasked, broadcast.
202  def : Pat<(OpNode
203                (any_broadcast ScalarVT:$sx), DataVT:$vy,
204                (MaskVT true_mask),
205                i32:$avl),
206            (!cast<Instruction>(OpBaseName#"rvl")
207                ScalarVT:$sx, $vy, $avl)>;
208  // Masked, broadcast.
209  def : Pat<(OpNode
210                (any_broadcast ScalarVT:$sx), DataVT:$vy,
211                MaskVT:$mask,
212                i32:$avl),
213            (!cast<Instruction>(OpBaseName#"rvml")
214                ScalarVT:$sx, $vy, $mask, $avl)>;
215}
216
217multiclass Binary_vr<SDPatternOperator OpNode,
218    ValueType ScalarVT, ValueType DataVT,
219    ValueType MaskVT, string OpBaseName> {
220  // Masked with passthru, broadcast.
221  def : Pat<(vvp_select
222                (OpNode
223                    DataVT:$vx,
224                    (any_broadcast ScalarVT:$sy),
225                    (MaskVT srcvalue),
226                    (i32 srcvalue)),
227                DataVT:$vfalse,
228                MaskVT:$mask,
229                i32:$pivot),
230            (!cast<Instruction>(OpBaseName#"vrml_v")
231                $vx,
232                ScalarVT:$sy,
233                $mask,
234                $pivot,
235                $vfalse)>;
236
237  // Unmasked, broadcast.
238  def : Pat<(OpNode
239                DataVT:$vx, (any_broadcast ScalarVT:$sy),
240                (MaskVT true_mask),
241                i32:$avl),
242            (!cast<Instruction>(OpBaseName#"vrl")
243                $vx, ScalarVT:$sy, $avl)>;
244  // Masked, broadcast.
245  def : Pat<(OpNode
246                DataVT:$vx, (any_broadcast ScalarVT:$sy),
247                MaskVT:$mask,
248                i32:$avl),
249            (!cast<Instruction>(OpBaseName#"vrml")
250                $vx, ScalarVT:$sy, $mask, $avl)>;
251}
252
253multiclass Binary_vv<SDPatternOperator OpNode,
254    ValueType DataVT,
255    ValueType MaskVT, string OpBaseName> {
256  // Masked with passthru, broadcast.
257  def : Pat<(vvp_select
258                (OpNode
259                    DataVT:$vx,
260                    DataVT:$vy,
261                    (MaskVT srcvalue),
262                    (i32 srcvalue)),
263                DataVT:$vfalse,
264                MaskVT:$mask,
265                i32:$pivot),
266            (!cast<Instruction>(OpBaseName#"vvml_v")
267                $vx,
268                $vy,
269                $mask,
270                $pivot,
271                $vfalse)>;
272
273  // Masked with select.
274  // TODO
275
276  // Unmasked.
277  def : Pat<(OpNode
278                DataVT:$vx, DataVT:$vy,
279                (MaskVT true_mask),
280                i32:$avl),
281            (!cast<Instruction>(OpBaseName#"vvl")
282                $vx, $vy, $avl)>;
283
284  // Masked.
285  def : Pat<(OpNode
286                DataVT:$vx, DataVT:$vy,
287                MaskVT:$mask,
288                i32:$avl),
289            (!cast<Instruction>(OpBaseName#"vvml")
290                $vx, $vy, $mask, $avl)>;
291}
292
293multiclass Binary_rv_vv<
294    SDPatternOperator OpNode,
295    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
296    string OpBaseName> {
297  defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
298  defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>;
299}
300
301multiclass Binary_vr_vv<
302    SDPatternOperator OpNode,
303    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
304    string OpBaseName> {
305  defm : Binary_vr<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
306  defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>;
307}
308
309multiclass Binary_rv_vr_vv<
310    SDPatternOperator OpNode,
311    ValueType ScalarVT, ValueType DataVT, ValueType MaskVT,
312    string OpBaseName> {
313  defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
314  defm : Binary_vr_vv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
315}
316
317// Expand both 64bit and 32 bit variant (256 elements)
318multiclass Binary_rv_vv_ShortLong<
319    SDPatternOperator OpNode,
320    ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
321    ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
322  defm : Binary_rv_vv<OpNode,
323                      LongScalarVT, LongDataVT, v256i1,
324                      LongOpBaseName>;
325  defm : Binary_rv_vv<OpNode,
326                      ShortScalarVT, ShortDataVT, v256i1,
327                      ShortOpBaseName>;
328}
329
330multiclass Binary_vr_vv_ShortLong<
331    SDPatternOperator OpNode,
332    ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
333    ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
334  defm : Binary_vr_vv<OpNode,
335                      LongScalarVT, LongDataVT, v256i1,
336                      LongOpBaseName>;
337  defm : Binary_vr_vv<OpNode,
338                      ShortScalarVT, ShortDataVT, v256i1,
339                      ShortOpBaseName>;
340}
341
342multiclass Binary_rv_vr_vv_ShortLong<
343    SDPatternOperator OpNode,
344    ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
345    ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
346  defm : Binary_rv_vr_vv<OpNode,
347                      LongScalarVT, LongDataVT, v256i1,
348                      LongOpBaseName>;
349  defm : Binary_rv_vr_vv<OpNode,
350                      ShortScalarVT, ShortDataVT, v256i1,
351                      ShortOpBaseName>;
352}
353
354defm : Binary_rv_vv_ShortLong<c_vvp_add,
355                              i64, v256i64, "VADDSL",
356                              i32, v256i32, "VADDSWSX">;
357defm : Binary_rv_vv_ShortLong<vvp_sub,
358                              i64, v256i64, "VSUBSL",
359                              i32, v256i32, "VSUBSWSX">;
360defm : Binary_rv_vv_ShortLong<c_vvp_mul,
361                              i64, v256i64, "VMULSL",
362                              i32, v256i32, "VMULSWSX">;
363defm : Binary_rv_vr_vv_ShortLong<vvp_sdiv,
364                              i64, v256i64, "VDIVSL",
365                              i32, v256i32, "VDIVSWSX">;
366defm : Binary_rv_vr_vv_ShortLong<vvp_udiv,
367                              i64, v256i64, "VDIVUL",
368                              i32, v256i32, "VDIVUW">;
369defm : Binary_rv_vv_ShortLong<c_vvp_and,
370                              i64, v256i64, "VAND",
371                              i32, v256i32, "PVANDLO">;
372defm : Binary_rv_vv_ShortLong<c_vvp_or,
373                              i64, v256i64, "VOR",
374                              i32, v256i32, "PVORLO">;
375defm : Binary_rv_vv_ShortLong<c_vvp_xor,
376                              i64, v256i64, "VXOR",
377                              i32, v256i32, "PVXORLO">;
378defm : Binary_vr_vv_ShortLong<vvp_shl,
379                              i64, v256i64, "VSLL",
380                              i32, v256i32, "PVSLLLO">;
381defm : Binary_vr_vv_ShortLong<vvp_sra,
382                              i64, v256i64, "VSRAL",
383                              i32, v256i32, "PVSRALO">;
384defm : Binary_vr_vv_ShortLong<vvp_srl,
385                              i64, v256i64, "VSRL",
386                              i32, v256i32, "PVSRLLO">;
387
388defm : Binary_rv_vv_ShortLong<c_vvp_fadd,
389                              f64, v256f64, "VFADDD",
390                              f32, v256f32, "PVFADDUP">;
391defm : Binary_rv_vv_ShortLong<c_vvp_fmul,
392                              f64, v256f64, "VFMULD",
393                              f32, v256f32, "PVFMULUP">;
394defm : Binary_rv_vv_ShortLong<vvp_fsub,
395                              f64, v256f64, "VFSUBD",
396                              f32, v256f32, "PVFSUBUP">;
397defm : Binary_rv_vr_vv_ShortLong<vvp_fdiv,
398                              f64, v256f64, "VFDIVD",
399                              f32, v256f32, "VFDIVS">;
400
401defm : Binary_rv_vv<c_vvp_and,
402                    i64, v512i32, v512i1, "PVAND">;
403defm : Binary_rv_vv<c_vvp_or,
404                    i64, v512i32, v512i1, "PVOR">;
405defm : Binary_rv_vv<c_vvp_xor,
406                    i64, v512i32, v512i1, "PVXOR">;
407
408defm : Binary_rv_vv<c_vvp_add,
409                    i64, v512i32, v512i1, "PVADDU">;
410defm : Binary_rv_vv<vvp_sub,
411                    i64, v512i32, v512i1, "PVSUBU">;
412defm : Binary_vr_vv<vvp_srl,
413                    i64, v512i32, v512i1, "PVSRL">;
414defm : Binary_vr_vv<vvp_sra,
415                    i64, v512i32, v512i1, "PVSRA">;
416defm : Binary_vr_vv<vvp_shl,
417                    i64, v512i32, v512i1, "PVSLL">;
418
419defm : Binary_rv_vv<c_vvp_fadd,
420                    i64, v512f32, v512i1, "PVFADD">;
421defm : Binary_rv_vv<c_vvp_fmul,
422                    i64, v512f32, v512i1, "PVFMUL">;
423defm : Binary_rv_vv<vvp_fsub,
424                    i64, v512f32, v512i1, "PVFSUB">;
425
426multiclass Ternary_vvv<
427    SDPatternOperator OpNode, ValueType DataVT,
428    ValueType MaskVT, string OpBaseName> {
429  // Masked with passthru.
430  def : Pat<(vvp_select
431              (OpNode DataVT:$vx, DataVT:$vy, DataVT:$vz,
432                (MaskVT srcvalue), (i32 srcvalue)),
433              DataVT:$vfalse,
434              MaskVT:$mask,
435              i32:$avl),
436            (!cast<Instruction>(OpBaseName#"vvvml_v")
437              $vx, $vy, $vz, $mask, $avl, $vfalse)>;
438
439  // Unmasked.
440  def : Pat<(OpNode DataVT:$vx, DataVT:$vy, DataVT:$vz,
441              (MaskVT true_mask), i32:$avl),
442            (!cast<Instruction>(OpBaseName#"vvvl")
443              $vx, $vy, $vz, $avl)>;
444
445  // Masked.
446  def : Pat<(OpNode DataVT:$vx, DataVT:$vy, DataVT:$vz,
447              MaskVT:$mask, i32:$avl),
448            (!cast<Instruction>(OpBaseName#"vvvml")
449              $vx, $vy, $vz, $mask, $avl)>;
450}
451
452multiclass Ternary_rvv<
453    SDPatternOperator OpNode,
454    ValueType ScalarVT, ValueType DataVT,
455    ValueType MaskVT, string OpBaseName> {
456  // Masked with passthru, broadcast first.
457  def : Pat<(vvp_select
458              (OpNode
459                (any_broadcast ScalarVT:$sx), DataVT:$vy, DataVT:$vz,
460                (MaskVT srcvalue), (i32 srcvalue)),
461              DataVT:$vfalse,
462              MaskVT:$mask,
463              i32:$avl),
464            (!cast<Instruction>(OpBaseName#"rvvml_v")
465              $sx, $vy, $vz, $mask, $avl, $vfalse)>;
466
467  // Unmasked, broadcast first.
468  def : Pat<(OpNode
469              (any_broadcast ScalarVT:$sx), DataVT:$vy, DataVT:$vz,
470              (MaskVT true_mask), i32:$avl),
471            (!cast<Instruction>(OpBaseName#"rvvl")
472              $sx, $vy, $vz, $avl)>;
473
474  // Masked, broadcast first.
475  def : Pat<(OpNode
476              (any_broadcast ScalarVT:$sx), DataVT:$vy, DataVT:$vz,
477              MaskVT:$mask, i32:$avl),
478            (!cast<Instruction>(OpBaseName#"rvvml")
479              $sx, $vy, $vz, $mask, $avl)>;
480}
481
482multiclass Ternary_vrv<
483    SDPatternOperator OpNode,
484    ValueType ScalarVT, ValueType DataVT,
485    ValueType MaskVT, string OpBaseName> {
486  // Masked with passthru, broadcast second.
487  def : Pat<(vvp_select
488              (OpNode
489                DataVT:$vx, (any_broadcast ScalarVT:$sy), DataVT:$vz,
490                (MaskVT srcvalue), (i32 srcvalue)),
491              DataVT:$vfalse,
492              MaskVT:$mask,
493              i32:$avl),
494            (!cast<Instruction>(OpBaseName#"vrvml_v")
495              $vx, $sy, $vz,
496              $mask, $avl, $vfalse)>;
497
498  // Unmasked, broadcast second.
499  def : Pat<(OpNode
500              DataVT:$vx, (any_broadcast ScalarVT:$sy), DataVT:$vz,
501              (MaskVT true_mask), i32:$avl),
502            (!cast<Instruction>(OpBaseName#"vrvl")
503              $vx, $sy, $vz, $avl)>;
504
505  // Masked, broadcast second.
506  def : Pat<(OpNode
507              DataVT:$vx, (any_broadcast ScalarVT:$sy), DataVT:$vz,
508              MaskVT:$mask, i32:$avl),
509            (!cast<Instruction>(OpBaseName#"vrvml")
510              $vx, $sy, $vz, $mask, $avl)>;
511}
512
513multiclass Ternary_rvv_vrv_vvv<
514    SDPatternOperator OpNode,
515    ValueType ScalarVT, ValueType DataVT,
516    ValueType MaskVT, string OpBaseName> {
517  defm : Ternary_rvv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
518  defm : Ternary_vrv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>;
519  defm : Ternary_vvv<OpNode, DataVT, MaskVT, OpBaseName>;
520}
521
522// Expand both 64bit and 32 bit variant (256 elements)
523multiclass Ternary_ShortLong<
524    SDPatternOperator OpNode,
525    ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName,
526    ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> {
527  defm : Ternary_rvv_vrv_vvv<OpNode, LongScalarVT, LongDataVT,
528                             v256i1, LongOpBaseName>;
529  defm : Ternary_rvv_vrv_vvv<OpNode, ShortScalarVT, ShortDataVT,
530                             v256i1, ShortOpBaseName>;
531}
532
533defm : Ternary_ShortLong<c_vvp_ffma,
534                         f64, v256f64, "VFMADD", f32, v256f32, "VFMADS">;
535defm : Ternary_rvv_vrv_vvv<c_vvp_ffma,
536                           i64, v512f32, v512i1, "PVFMAD">;
537
538multiclass Merge_mvv<
539    SDPatternOperator OpNode,
540    ValueType DataVT, ValueType MaskVT,
541    string OpBaseName> {
542  // Masked.
543  def : Pat<(OpNode
544                DataVT:$vtrue, DataVT:$vfalse,
545                MaskVT:$vm,
546                i32:$avl),
547            (!cast<Instruction>(OpBaseName#"vvml_v")
548                $vfalse, $vtrue, $vm, $avl, $vfalse)>;
549}
550
551multiclass Merge_mvv_ShortLong<
552    SDPatternOperator OpNode,
553    ValueType LongDataVT, ValueType ShortDataVT,
554    string OpBaseName> {
555  defm : Merge_mvv<OpNode,
556                   LongDataVT, v256i1,
557                   OpBaseName>;
558  defm : Merge_mvv<OpNode,
559                   ShortDataVT, v256i1,
560                   OpBaseName>;
561}
562
563defm : Merge_mvv_ShortLong<vvp_select,
564                           v256f64,
565                           v256f32, "VMRG">;
566defm : Merge_mvv_ShortLong<vvp_select,
567                           v256i64,
568                           v256i32, "VMRG">;
569
570multiclass Set_CC<ValueType DataVT, string FmkBaseName, string CmpBaseName, SDPatternOperator CCMatcher, SDNodeXForm CCConv> {
571  // Unmasked.
572  def : Pat<(v256i1 (vvp_setcc
573              DataVT:$LHS, DataVT:$RHS, CCMatcher:$cond, (v256i1 true_mask), i32:$vl)),
574              (!cast<Instruction>(FmkBaseName#"vl")
575                (CCConv $cond),
576                (!cast<Instruction>(CmpBaseName#"vvl")
577                  $LHS, $RHS, $vl),
578                $vl)>;
579  // Masked.
580  def : Pat<(v256i1 (vvp_setcc
581              DataVT:$LHS, DataVT:$RHS, CCMatcher:$cond, v256i1:$vm, i32:$vl)),
582              (!cast<Instruction>(FmkBaseName#"vml")
583                (CCConv $cond),
584                (!cast<Instruction>(CmpBaseName#"vvl")
585                  $LHS, $RHS, $vl),
586                $vm, $vl)>;
587}
588
589defm : Set_CC<v256i64,"VFMKL","VCMPUL",CCUIOp,icond2cc>;
590defm : Set_CC<v256i64,"VFMKL","VCMPSL",CCSIOp,icond2cc>;
591defm : Set_CC<v256f64,"VFMKL","VFCMPD",cond,fcond2cc>;
592
593defm : Set_CC<v256i32,"VFMKW","VCMPUW",CCUIOp,icond2cc>;
594defm : Set_CC<v256i32,"VFMKW","VCMPSWZX",CCSIOp,icond2cc>;
595defm : Set_CC<v256f32,"VFMKS","VFCMPS",cond,fcond2cc>;
596
597multiclass Reduce_GenericInt<ValueType VectorVT,
598    RegisterClass ResRC, ValueType ResVT,
599    string VVPRedOp, string RedInstName> {
600  // Unmasked.
601  def : Pat <(ResVT (!cast<SDPatternOperator>("vvp_reduce_"#VVPRedOp)
602                VectorVT:$vx, (v256i1 true_mask), i32:$vl)),
603             (COPY_TO_REGCLASS
604               (!cast<Instruction>("LVSvi")
605                 (!cast<Instruction>(RedInstName#"vl") $vx, $vl), 0),
606                 ResRC)>;
607
608  // Masked.
609  def : Pat <(ResVT (!cast<SDPatternOperator>("vvp_reduce_"#VVPRedOp)
610                VectorVT:$vx, v256i1:$vm, i32:$vl)),
611             (COPY_TO_REGCLASS
612                (!cast<Instruction>("LVSvi")
613                   (!cast<Instruction>(RedInstName#"vml") $vx, $vm, $vl), 0),
614                   ResRC)>;
615}
616
617multiclass IntReduce_ShortLong<ValueType VectorVT,
618    RegisterClass ResRC, ValueType ResVT,
619    string SumSuffix, string MinMaxSuffix> {
620  defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "or",  "VROR">;
621  defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "and", "VRAND">;
622  defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "xor", "VRXOR">;
623  defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "add", "VSUM"#SumSuffix>;
624  defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "smax", "VRMAX"#MinMaxSuffix>;
625}
626
627defm: IntReduce_ShortLong<v256i64, I64, i64, "L","SLFST">;
628defm: IntReduce_ShortLong<v256i32, I32, i32, "WSX","SWFSTSX">;
629