1 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SystemZTargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "SystemZISelLowering.h"
14 #include "SystemZCallingConv.h"
15 #include "SystemZConstantPoolValue.h"
16 #include "SystemZMachineFunctionInfo.h"
17 #include "SystemZTargetMachine.h"
18 #include "llvm/CodeGen/CallingConvLower.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
22 #include "llvm/IR/IntrinsicInst.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/IntrinsicsS390.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/KnownBits.h"
27 #include <cctype>
28 #include <optional>
29
30 using namespace llvm;
31
32 #define DEBUG_TYPE "systemz-lower"
33
34 namespace {
35 // Represents information about a comparison.
36 struct Comparison {
Comparison__anon968932af0111::Comparison37 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
38 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
39 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
40
41 // The operands to the comparison.
42 SDValue Op0, Op1;
43
44 // Chain if this is a strict floating-point comparison.
45 SDValue Chain;
46
47 // The opcode that should be used to compare Op0 and Op1.
48 unsigned Opcode;
49
50 // A SystemZICMP value. Only used for integer comparisons.
51 unsigned ICmpType;
52
53 // The mask of CC values that Opcode can produce.
54 unsigned CCValid;
55
56 // The mask of CC values for which the original condition is true.
57 unsigned CCMask;
58 };
59 } // end anonymous namespace
60
61 // Classify VT as either 32 or 64 bit.
is32Bit(EVT VT)62 static bool is32Bit(EVT VT) {
63 switch (VT.getSimpleVT().SimpleTy) {
64 case MVT::i32:
65 return true;
66 case MVT::i64:
67 return false;
68 default:
69 llvm_unreachable("Unsupported type");
70 }
71 }
72
73 // Return a version of MachineOperand that can be safely used before the
74 // final use.
earlyUseOperand(MachineOperand Op)75 static MachineOperand earlyUseOperand(MachineOperand Op) {
76 if (Op.isReg())
77 Op.setIsKill(false);
78 return Op;
79 }
80
SystemZTargetLowering(const TargetMachine & TM,const SystemZSubtarget & STI)81 SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
82 const SystemZSubtarget &STI)
83 : TargetLowering(TM), Subtarget(STI) {
84 MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
85
86 auto *Regs = STI.getSpecialRegisters();
87
88 // Set up the register classes.
89 if (Subtarget.hasHighWord())
90 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
91 else
92 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
93 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
94 if (!useSoftFloat()) {
95 if (Subtarget.hasVector()) {
96 addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
97 addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
98 } else {
99 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
100 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
101 }
102 if (Subtarget.hasVectorEnhancements1())
103 addRegisterClass(MVT::f128, &SystemZ::VR128BitRegClass);
104 else
105 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
106
107 if (Subtarget.hasVector()) {
108 addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
109 addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
110 addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
111 addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
112 addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
113 addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
114 }
115
116 if (Subtarget.hasVector())
117 addRegisterClass(MVT::i128, &SystemZ::VR128BitRegClass);
118 }
119
120 // Compute derived properties from the register classes
121 computeRegisterProperties(Subtarget.getRegisterInfo());
122
123 // Set up special registers.
124 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
125
126 // TODO: It may be better to default to latency-oriented scheduling, however
127 // LLVM's current latency-oriented scheduler can't handle physreg definitions
128 // such as SystemZ has with CC, so set this to the register-pressure
129 // scheduler, because it can.
130 setSchedulingPreference(Sched::RegPressure);
131
132 setBooleanContents(ZeroOrOneBooleanContent);
133 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
134
135 setMaxAtomicSizeInBitsSupported(128);
136
137 // Instructions are strings of 2-byte aligned 2-byte values.
138 setMinFunctionAlignment(Align(2));
139 // For performance reasons we prefer 16-byte alignment.
140 setPrefFunctionAlignment(Align(16));
141
142 // Handle operations that are handled in a similar way for all types.
143 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
144 I <= MVT::LAST_FP_VALUETYPE;
145 ++I) {
146 MVT VT = MVT::SimpleValueType(I);
147 if (isTypeLegal(VT)) {
148 // Lower SET_CC into an IPM-based sequence.
149 setOperationAction(ISD::SETCC, VT, Custom);
150 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
151 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
152
153 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
154 setOperationAction(ISD::SELECT, VT, Expand);
155
156 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
157 setOperationAction(ISD::SELECT_CC, VT, Custom);
158 setOperationAction(ISD::BR_CC, VT, Custom);
159 }
160 }
161
162 // Expand jump table branches as address arithmetic followed by an
163 // indirect jump.
164 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
165
166 // Expand BRCOND into a BR_CC (see above).
167 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
168
169 // Handle integer types except i128.
170 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
171 I <= MVT::LAST_INTEGER_VALUETYPE;
172 ++I) {
173 MVT VT = MVT::SimpleValueType(I);
174 if (isTypeLegal(VT) && VT != MVT::i128) {
175 setOperationAction(ISD::ABS, VT, Legal);
176
177 // Expand individual DIV and REMs into DIVREMs.
178 setOperationAction(ISD::SDIV, VT, Expand);
179 setOperationAction(ISD::UDIV, VT, Expand);
180 setOperationAction(ISD::SREM, VT, Expand);
181 setOperationAction(ISD::UREM, VT, Expand);
182 setOperationAction(ISD::SDIVREM, VT, Custom);
183 setOperationAction(ISD::UDIVREM, VT, Custom);
184
185 // Support addition/subtraction with overflow.
186 setOperationAction(ISD::SADDO, VT, Custom);
187 setOperationAction(ISD::SSUBO, VT, Custom);
188
189 // Support addition/subtraction with carry.
190 setOperationAction(ISD::UADDO, VT, Custom);
191 setOperationAction(ISD::USUBO, VT, Custom);
192
193 // Support carry in as value rather than glue.
194 setOperationAction(ISD::UADDO_CARRY, VT, Custom);
195 setOperationAction(ISD::USUBO_CARRY, VT, Custom);
196
197 // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and
198 // stores, putting a serialization instruction after the stores.
199 setOperationAction(ISD::ATOMIC_LOAD, VT, Custom);
200 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
201
202 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
203 // available, or if the operand is constant.
204 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
205
206 // Use POPCNT on z196 and above.
207 if (Subtarget.hasPopulationCount())
208 setOperationAction(ISD::CTPOP, VT, Custom);
209 else
210 setOperationAction(ISD::CTPOP, VT, Expand);
211
212 // No special instructions for these.
213 setOperationAction(ISD::CTTZ, VT, Expand);
214 setOperationAction(ISD::ROTR, VT, Expand);
215
216 // Use *MUL_LOHI where possible instead of MULH*.
217 setOperationAction(ISD::MULHS, VT, Expand);
218 setOperationAction(ISD::MULHU, VT, Expand);
219 setOperationAction(ISD::SMUL_LOHI, VT, Custom);
220 setOperationAction(ISD::UMUL_LOHI, VT, Custom);
221
222 // Only z196 and above have native support for conversions to unsigned.
223 // On z10, promoting to i64 doesn't generate an inexact condition for
224 // values that are outside the i32 range but in the i64 range, so use
225 // the default expansion.
226 if (!Subtarget.hasFPExtension())
227 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
228
229 // Mirror those settings for STRICT_FP_TO_[SU]INT. Note that these all
230 // default to Expand, so need to be modified to Legal where appropriate.
231 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Legal);
232 if (Subtarget.hasFPExtension())
233 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Legal);
234
235 // And similarly for STRICT_[SU]INT_TO_FP.
236 setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Legal);
237 if (Subtarget.hasFPExtension())
238 setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Legal);
239 }
240 }
241
242 // Handle i128 if legal.
243 if (isTypeLegal(MVT::i128)) {
244 // No special instructions for these.
245 setOperationAction(ISD::SDIVREM, MVT::i128, Expand);
246 setOperationAction(ISD::UDIVREM, MVT::i128, Expand);
247 setOperationAction(ISD::SMUL_LOHI, MVT::i128, Expand);
248 setOperationAction(ISD::UMUL_LOHI, MVT::i128, Expand);
249 setOperationAction(ISD::ROTR, MVT::i128, Expand);
250 setOperationAction(ISD::ROTL, MVT::i128, Expand);
251 setOperationAction(ISD::MUL, MVT::i128, Expand);
252 setOperationAction(ISD::MULHS, MVT::i128, Expand);
253 setOperationAction(ISD::MULHU, MVT::i128, Expand);
254 setOperationAction(ISD::SDIV, MVT::i128, Expand);
255 setOperationAction(ISD::UDIV, MVT::i128, Expand);
256 setOperationAction(ISD::SREM, MVT::i128, Expand);
257 setOperationAction(ISD::UREM, MVT::i128, Expand);
258 setOperationAction(ISD::CTLZ, MVT::i128, Expand);
259 setOperationAction(ISD::CTTZ, MVT::i128, Expand);
260
261 // Support addition/subtraction with carry.
262 setOperationAction(ISD::UADDO, MVT::i128, Custom);
263 setOperationAction(ISD::USUBO, MVT::i128, Custom);
264 setOperationAction(ISD::UADDO_CARRY, MVT::i128, Custom);
265 setOperationAction(ISD::USUBO_CARRY, MVT::i128, Custom);
266
267 // Use VPOPCT and add up partial results.
268 setOperationAction(ISD::CTPOP, MVT::i128, Custom);
269
270 // We have to use libcalls for these.
271 setOperationAction(ISD::FP_TO_UINT, MVT::i128, LibCall);
272 setOperationAction(ISD::FP_TO_SINT, MVT::i128, LibCall);
273 setOperationAction(ISD::UINT_TO_FP, MVT::i128, LibCall);
274 setOperationAction(ISD::SINT_TO_FP, MVT::i128, LibCall);
275 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, LibCall);
276 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, LibCall);
277 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, LibCall);
278 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, LibCall);
279 }
280
281 // Type legalization will convert 8- and 16-bit atomic operations into
282 // forms that operate on i32s (but still keeping the original memory VT).
283 // Lower them into full i32 operations.
284 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom);
285 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom);
286 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
287 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
288 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom);
289 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom);
290 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom);
291 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom);
292 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom);
293 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom);
294 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom);
295
296 // Whether or not i128 is not a legal type, we need to custom lower
297 // the atomic operations in order to exploit SystemZ instructions.
298 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
299 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
300
301 // We can use the CC result of compare-and-swap to implement
302 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
303 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Custom);
304 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Custom);
305 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
306
307 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
308
309 // Traps are legal, as we will convert them to "j .+2".
310 setOperationAction(ISD::TRAP, MVT::Other, Legal);
311
312 // z10 has instructions for signed but not unsigned FP conversion.
313 // Handle unsigned 32-bit types as signed 64-bit types.
314 if (!Subtarget.hasFPExtension()) {
315 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote);
316 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
317 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Promote);
318 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand);
319 }
320
321 // We have native support for a 64-bit CTLZ, via FLOGR.
322 setOperationAction(ISD::CTLZ, MVT::i32, Promote);
323 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);
324 setOperationAction(ISD::CTLZ, MVT::i64, Legal);
325
326 // On z15 we have native support for a 64-bit CTPOP.
327 if (Subtarget.hasMiscellaneousExtensions3()) {
328 setOperationAction(ISD::CTPOP, MVT::i32, Promote);
329 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
330 }
331
332 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
333 setOperationAction(ISD::OR, MVT::i64, Custom);
334
335 // Expand 128 bit shifts without using a libcall.
336 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
337 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
338 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
339 setLibcallName(RTLIB::SRL_I128, nullptr);
340 setLibcallName(RTLIB::SHL_I128, nullptr);
341 setLibcallName(RTLIB::SRA_I128, nullptr);
342
343 // Also expand 256 bit shifts if i128 is a legal type.
344 if (isTypeLegal(MVT::i128)) {
345 setOperationAction(ISD::SRL_PARTS, MVT::i128, Expand);
346 setOperationAction(ISD::SHL_PARTS, MVT::i128, Expand);
347 setOperationAction(ISD::SRA_PARTS, MVT::i128, Expand);
348 }
349
350 // Handle bitcast from fp128 to i128.
351 if (!isTypeLegal(MVT::i128))
352 setOperationAction(ISD::BITCAST, MVT::i128, Custom);
353
354 // We have native instructions for i8, i16 and i32 extensions, but not i1.
355 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
356 for (MVT VT : MVT::integer_valuetypes()) {
357 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
358 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
359 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
360 }
361
362 // Handle the various types of symbolic address.
363 setOperationAction(ISD::ConstantPool, PtrVT, Custom);
364 setOperationAction(ISD::GlobalAddress, PtrVT, Custom);
365 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
366 setOperationAction(ISD::BlockAddress, PtrVT, Custom);
367 setOperationAction(ISD::JumpTable, PtrVT, Custom);
368
369 // We need to handle dynamic allocations specially because of the
370 // 160-byte area at the bottom of the stack.
371 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
372 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, PtrVT, Custom);
373
374 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom);
375 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom);
376
377 // Handle prefetches with PFD or PFDRL.
378 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
379
380 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
381 // Assume by default that all vector operations need to be expanded.
382 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
383 if (getOperationAction(Opcode, VT) == Legal)
384 setOperationAction(Opcode, VT, Expand);
385
386 // Likewise all truncating stores and extending loads.
387 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
388 setTruncStoreAction(VT, InnerVT, Expand);
389 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
390 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
391 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
392 }
393
394 if (isTypeLegal(VT)) {
395 // These operations are legal for anything that can be stored in a
396 // vector register, even if there is no native support for the format
397 // as such. In particular, we can do these for v4f32 even though there
398 // are no specific instructions for that format.
399 setOperationAction(ISD::LOAD, VT, Legal);
400 setOperationAction(ISD::STORE, VT, Legal);
401 setOperationAction(ISD::VSELECT, VT, Legal);
402 setOperationAction(ISD::BITCAST, VT, Legal);
403 setOperationAction(ISD::UNDEF, VT, Legal);
404
405 // Likewise, except that we need to replace the nodes with something
406 // more specific.
407 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
408 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
409 }
410 }
411
412 // Handle integer vector types.
413 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
414 if (isTypeLegal(VT)) {
415 // These operations have direct equivalents.
416 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal);
417 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Legal);
418 setOperationAction(ISD::ADD, VT, Legal);
419 setOperationAction(ISD::SUB, VT, Legal);
420 if (VT != MVT::v2i64)
421 setOperationAction(ISD::MUL, VT, Legal);
422 setOperationAction(ISD::ABS, VT, Legal);
423 setOperationAction(ISD::AND, VT, Legal);
424 setOperationAction(ISD::OR, VT, Legal);
425 setOperationAction(ISD::XOR, VT, Legal);
426 if (Subtarget.hasVectorEnhancements1())
427 setOperationAction(ISD::CTPOP, VT, Legal);
428 else
429 setOperationAction(ISD::CTPOP, VT, Custom);
430 setOperationAction(ISD::CTTZ, VT, Legal);
431 setOperationAction(ISD::CTLZ, VT, Legal);
432
433 // Convert a GPR scalar to a vector by inserting it into element 0.
434 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
435
436 // Use a series of unpacks for extensions.
437 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
438 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
439
440 // Detect shifts/rotates by a scalar amount and convert them into
441 // V*_BY_SCALAR.
442 setOperationAction(ISD::SHL, VT, Custom);
443 setOperationAction(ISD::SRA, VT, Custom);
444 setOperationAction(ISD::SRL, VT, Custom);
445 setOperationAction(ISD::ROTL, VT, Custom);
446
447 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
448 // and inverting the result as necessary.
449 setOperationAction(ISD::SETCC, VT, Custom);
450 }
451 }
452
453 if (Subtarget.hasVector()) {
454 // There should be no need to check for float types other than v2f64
455 // since <2 x f32> isn't a legal type.
456 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
457 setOperationAction(ISD::FP_TO_SINT, MVT::v2f64, Legal);
458 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
459 setOperationAction(ISD::FP_TO_UINT, MVT::v2f64, Legal);
460 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
461 setOperationAction(ISD::SINT_TO_FP, MVT::v2f64, Legal);
462 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
463 setOperationAction(ISD::UINT_TO_FP, MVT::v2f64, Legal);
464
465 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i64, Legal);
466 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f64, Legal);
467 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i64, Legal);
468 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f64, Legal);
469 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i64, Legal);
470 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f64, Legal);
471 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i64, Legal);
472 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f64, Legal);
473 }
474
475 if (Subtarget.hasVectorEnhancements2()) {
476 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
477 setOperationAction(ISD::FP_TO_SINT, MVT::v4f32, Legal);
478 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
479 setOperationAction(ISD::FP_TO_UINT, MVT::v4f32, Legal);
480 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
481 setOperationAction(ISD::SINT_TO_FP, MVT::v4f32, Legal);
482 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
483 setOperationAction(ISD::UINT_TO_FP, MVT::v4f32, Legal);
484
485 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal);
486 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f32, Legal);
487 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal);
488 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f32, Legal);
489 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal);
490 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f32, Legal);
491 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal);
492 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f32, Legal);
493 }
494
495 // Handle floating-point types.
496 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
497 I <= MVT::LAST_FP_VALUETYPE;
498 ++I) {
499 MVT VT = MVT::SimpleValueType(I);
500 if (isTypeLegal(VT)) {
501 // We can use FI for FRINT.
502 setOperationAction(ISD::FRINT, VT, Legal);
503
504 // We can use the extended form of FI for other rounding operations.
505 if (Subtarget.hasFPExtension()) {
506 setOperationAction(ISD::FNEARBYINT, VT, Legal);
507 setOperationAction(ISD::FFLOOR, VT, Legal);
508 setOperationAction(ISD::FCEIL, VT, Legal);
509 setOperationAction(ISD::FTRUNC, VT, Legal);
510 setOperationAction(ISD::FROUND, VT, Legal);
511 }
512
513 // No special instructions for these.
514 setOperationAction(ISD::FSIN, VT, Expand);
515 setOperationAction(ISD::FCOS, VT, Expand);
516 setOperationAction(ISD::FSINCOS, VT, Expand);
517 setOperationAction(ISD::FREM, VT, Expand);
518 setOperationAction(ISD::FPOW, VT, Expand);
519
520 // Special treatment.
521 setOperationAction(ISD::IS_FPCLASS, VT, Custom);
522
523 // Handle constrained floating-point operations.
524 setOperationAction(ISD::STRICT_FADD, VT, Legal);
525 setOperationAction(ISD::STRICT_FSUB, VT, Legal);
526 setOperationAction(ISD::STRICT_FMUL, VT, Legal);
527 setOperationAction(ISD::STRICT_FDIV, VT, Legal);
528 setOperationAction(ISD::STRICT_FMA, VT, Legal);
529 setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
530 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
531 setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
532 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
533 if (Subtarget.hasFPExtension()) {
534 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
535 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
536 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
537 setOperationAction(ISD::STRICT_FROUND, VT, Legal);
538 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
539 }
540 }
541 }
542
543 // Handle floating-point vector types.
544 if (Subtarget.hasVector()) {
545 // Scalar-to-vector conversion is just a subreg.
546 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
547 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
548
549 // Some insertions and extractions can be done directly but others
550 // need to go via integers.
551 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
552 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
553 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
554 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
555
556 // These operations have direct equivalents.
557 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
558 setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
559 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
560 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
561 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
562 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
563 setOperationAction(ISD::FABS, MVT::v2f64, Legal);
564 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
565 setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
566 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
567 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
568 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
569 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
570 setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
571
572 // Handle constrained floating-point operations.
573 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
574 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
575 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
576 setOperationAction(ISD::STRICT_FMA, MVT::v2f64, Legal);
577 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
578 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
579 setOperationAction(ISD::STRICT_FRINT, MVT::v2f64, Legal);
580 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v2f64, Legal);
581 setOperationAction(ISD::STRICT_FFLOOR, MVT::v2f64, Legal);
582 setOperationAction(ISD::STRICT_FCEIL, MVT::v2f64, Legal);
583 setOperationAction(ISD::STRICT_FTRUNC, MVT::v2f64, Legal);
584 setOperationAction(ISD::STRICT_FROUND, MVT::v2f64, Legal);
585
586 setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
587 setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
588 setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
589 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
590 if (Subtarget.hasVectorEnhancements1()) {
591 setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
592 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
593 }
594 }
595
596 // The vector enhancements facility 1 has instructions for these.
597 if (Subtarget.hasVectorEnhancements1()) {
598 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
599 setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
600 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
601 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
602 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
603 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
604 setOperationAction(ISD::FABS, MVT::v4f32, Legal);
605 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
606 setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
607 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
608 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
609 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
610 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
611 setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
612
613 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
614 setOperationAction(ISD::FMAXIMUM, MVT::f64, Legal);
615 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
616 setOperationAction(ISD::FMINIMUM, MVT::f64, Legal);
617
618 setOperationAction(ISD::FMAXNUM, MVT::v2f64, Legal);
619 setOperationAction(ISD::FMAXIMUM, MVT::v2f64, Legal);
620 setOperationAction(ISD::FMINNUM, MVT::v2f64, Legal);
621 setOperationAction(ISD::FMINIMUM, MVT::v2f64, Legal);
622
623 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
624 setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
625 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
626 setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
627
628 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
629 setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
630 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
631 setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
632
633 setOperationAction(ISD::FMAXNUM, MVT::f128, Legal);
634 setOperationAction(ISD::FMAXIMUM, MVT::f128, Legal);
635 setOperationAction(ISD::FMINNUM, MVT::f128, Legal);
636 setOperationAction(ISD::FMINIMUM, MVT::f128, Legal);
637
638 // Handle constrained floating-point operations.
639 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
640 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
641 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
642 setOperationAction(ISD::STRICT_FMA, MVT::v4f32, Legal);
643 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
644 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
645 setOperationAction(ISD::STRICT_FRINT, MVT::v4f32, Legal);
646 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v4f32, Legal);
647 setOperationAction(ISD::STRICT_FFLOOR, MVT::v4f32, Legal);
648 setOperationAction(ISD::STRICT_FCEIL, MVT::v4f32, Legal);
649 setOperationAction(ISD::STRICT_FROUND, MVT::v4f32, Legal);
650 setOperationAction(ISD::STRICT_FTRUNC, MVT::v4f32, Legal);
651 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
652 MVT::v4f32, MVT::v2f64 }) {
653 setOperationAction(ISD::STRICT_FMAXNUM, VT, Legal);
654 setOperationAction(ISD::STRICT_FMINNUM, VT, Legal);
655 setOperationAction(ISD::STRICT_FMAXIMUM, VT, Legal);
656 setOperationAction(ISD::STRICT_FMINIMUM, VT, Legal);
657 }
658 }
659
660 // We only have fused f128 multiply-addition on vector registers.
661 if (!Subtarget.hasVectorEnhancements1()) {
662 setOperationAction(ISD::FMA, MVT::f128, Expand);
663 setOperationAction(ISD::STRICT_FMA, MVT::f128, Expand);
664 }
665
666 // We don't have a copysign instruction on vector registers.
667 if (Subtarget.hasVectorEnhancements1())
668 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
669
670 // Needed so that we don't try to implement f128 constant loads using
671 // a load-and-extend of a f80 constant (in cases where the constant
672 // would fit in an f80).
673 for (MVT VT : MVT::fp_valuetypes())
674 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
675
676 // We don't have extending load instruction on vector registers.
677 if (Subtarget.hasVectorEnhancements1()) {
678 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
679 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
680 }
681
682 // Floating-point truncation and stores need to be done separately.
683 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
684 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
685 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
686
687 // We have 64-bit FPR<->GPR moves, but need special handling for
688 // 32-bit forms.
689 if (!Subtarget.hasVector()) {
690 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
691 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
692 }
693
694 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
695 // structure, but VAEND is a no-op.
696 setOperationAction(ISD::VASTART, MVT::Other, Custom);
697 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
698 setOperationAction(ISD::VAEND, MVT::Other, Expand);
699
700 setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
701
702 // Codes for which we want to perform some z-specific combinations.
703 setTargetDAGCombine({ISD::ZERO_EXTEND,
704 ISD::SIGN_EXTEND,
705 ISD::SIGN_EXTEND_INREG,
706 ISD::LOAD,
707 ISD::STORE,
708 ISD::VECTOR_SHUFFLE,
709 ISD::EXTRACT_VECTOR_ELT,
710 ISD::FP_ROUND,
711 ISD::STRICT_FP_ROUND,
712 ISD::FP_EXTEND,
713 ISD::SINT_TO_FP,
714 ISD::UINT_TO_FP,
715 ISD::STRICT_FP_EXTEND,
716 ISD::BSWAP,
717 ISD::SDIV,
718 ISD::UDIV,
719 ISD::SREM,
720 ISD::UREM,
721 ISD::INTRINSIC_VOID,
722 ISD::INTRINSIC_W_CHAIN});
723
724 // Handle intrinsics.
725 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
726 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
727
728 // We want to use MVC in preference to even a single load/store pair.
729 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
730 MaxStoresPerMemcpyOptSize = 0;
731
732 // The main memset sequence is a byte store followed by an MVC.
733 // Two STC or MV..I stores win over that, but the kind of fused stores
734 // generated by target-independent code don't when the byte value is
735 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
736 // than "STC;MVC". Handle the choice in target-specific code instead.
737 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
738 MaxStoresPerMemsetOptSize = 0;
739
740 // Default to having -disable-strictnode-mutation on
741 IsStrictFPEnabled = true;
742
743 if (Subtarget.isTargetzOS()) {
744 struct RTLibCallMapping {
745 RTLIB::Libcall Code;
746 const char *Name;
747 };
748 static RTLibCallMapping RTLibCallCommon[] = {
749 #define HANDLE_LIBCALL(code, name) {RTLIB::code, name},
750 #include "ZOSLibcallNames.def"
751 };
752 for (auto &E : RTLibCallCommon)
753 setLibcallName(E.Code, E.Name);
754 }
755 }
756
useSoftFloat() const757 bool SystemZTargetLowering::useSoftFloat() const {
758 return Subtarget.hasSoftFloat();
759 }
760
getSetCCResultType(const DataLayout & DL,LLVMContext &,EVT VT) const761 EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
762 LLVMContext &, EVT VT) const {
763 if (!VT.isVector())
764 return MVT::i32;
765 return VT.changeVectorElementTypeToInteger();
766 }
767
isFMAFasterThanFMulAndFAdd(const MachineFunction & MF,EVT VT) const768 bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
769 const MachineFunction &MF, EVT VT) const {
770 VT = VT.getScalarType();
771
772 if (!VT.isSimple())
773 return false;
774
775 switch (VT.getSimpleVT().SimpleTy) {
776 case MVT::f32:
777 case MVT::f64:
778 return true;
779 case MVT::f128:
780 return Subtarget.hasVectorEnhancements1();
781 default:
782 break;
783 }
784
785 return false;
786 }
787
788 // Return true if the constant can be generated with a vector instruction,
789 // such as VGM, VGMB or VREPI.
isVectorConstantLegal(const SystemZSubtarget & Subtarget)790 bool SystemZVectorConstantInfo::isVectorConstantLegal(
791 const SystemZSubtarget &Subtarget) {
792 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
793 if (!Subtarget.hasVector() ||
794 (isFP128 && !Subtarget.hasVectorEnhancements1()))
795 return false;
796
797 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
798 // preferred way of creating all-zero and all-one vectors so give it
799 // priority over other methods below.
800 unsigned Mask = 0;
801 unsigned I = 0;
802 for (; I < SystemZ::VectorBytes; ++I) {
803 uint64_t Byte = IntBits.lshr(I * 8).trunc(8).getZExtValue();
804 if (Byte == 0xff)
805 Mask |= 1ULL << I;
806 else if (Byte != 0)
807 break;
808 }
809 if (I == SystemZ::VectorBytes) {
810 Opcode = SystemZISD::BYTE_MASK;
811 OpVals.push_back(Mask);
812 VecVT = MVT::getVectorVT(MVT::getIntegerVT(8), 16);
813 return true;
814 }
815
816 if (SplatBitSize > 64)
817 return false;
818
819 auto tryValue = [&](uint64_t Value) -> bool {
820 // Try VECTOR REPLICATE IMMEDIATE
821 int64_t SignedValue = SignExtend64(Value, SplatBitSize);
822 if (isInt<16>(SignedValue)) {
823 OpVals.push_back(((unsigned) SignedValue));
824 Opcode = SystemZISD::REPLICATE;
825 VecVT = MVT::getVectorVT(MVT::getIntegerVT(SplatBitSize),
826 SystemZ::VectorBits / SplatBitSize);
827 return true;
828 }
829 // Try VECTOR GENERATE MASK
830 unsigned Start, End;
831 if (TII->isRxSBGMask(Value, SplatBitSize, Start, End)) {
832 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
833 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
834 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
835 OpVals.push_back(Start - (64 - SplatBitSize));
836 OpVals.push_back(End - (64 - SplatBitSize));
837 Opcode = SystemZISD::ROTATE_MASK;
838 VecVT = MVT::getVectorVT(MVT::getIntegerVT(SplatBitSize),
839 SystemZ::VectorBits / SplatBitSize);
840 return true;
841 }
842 return false;
843 };
844
845 // First try assuming that any undefined bits above the highest set bit
846 // and below the lowest set bit are 1s. This increases the likelihood of
847 // being able to use a sign-extended element value in VECTOR REPLICATE
848 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
849 uint64_t SplatBitsZ = SplatBits.getZExtValue();
850 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
851 unsigned LowerBits = llvm::countr_zero(SplatBitsZ);
852 unsigned UpperBits = llvm::countl_zero(SplatBitsZ);
853 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(LowerBits);
854 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(UpperBits);
855 if (tryValue(SplatBitsZ | Upper | Lower))
856 return true;
857
858 // Now try assuming that any undefined bits between the first and
859 // last defined set bits are set. This increases the chances of
860 // using a non-wraparound mask.
861 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
862 return tryValue(SplatBitsZ | Middle);
863 }
864
SystemZVectorConstantInfo(APInt IntImm)865 SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
866 if (IntImm.isSingleWord()) {
867 IntBits = APInt(128, IntImm.getZExtValue());
868 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
869 } else
870 IntBits = IntImm;
871 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
872
873 // Find the smallest splat.
874 SplatBits = IntImm;
875 unsigned Width = SplatBits.getBitWidth();
876 while (Width > 8) {
877 unsigned HalfSize = Width / 2;
878 APInt HighValue = SplatBits.lshr(HalfSize).trunc(HalfSize);
879 APInt LowValue = SplatBits.trunc(HalfSize);
880
881 // If the two halves do not match, stop here.
882 if (HighValue != LowValue || 8 > HalfSize)
883 break;
884
885 SplatBits = HighValue;
886 Width = HalfSize;
887 }
888 SplatUndef = 0;
889 SplatBitSize = Width;
890 }
891
SystemZVectorConstantInfo(BuildVectorSDNode * BVN)892 SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
893 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
894 bool HasAnyUndefs;
895
896 // Get IntBits by finding the 128 bit splat.
897 BVN->isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
898 true);
899
900 // Get SplatBits by finding the 8 bit or greater splat.
901 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
902 true);
903 }
904
isFPImmLegal(const APFloat & Imm,EVT VT,bool ForCodeSize) const905 bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
906 bool ForCodeSize) const {
907 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
908 if (Imm.isZero() || Imm.isNegZero())
909 return true;
910
911 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
912 }
913
914 /// Returns true if stack probing through inline assembly is requested.
hasInlineStackProbe(const MachineFunction & MF) const915 bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
916 // If the function specifically requests inline stack probes, emit them.
917 if (MF.getFunction().hasFnAttribute("probe-stack"))
918 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
919 "inline-asm";
920 return false;
921 }
922
923 TargetLowering::AtomicExpansionKind
shouldExpandAtomicRMWInIR(AtomicRMWInst * RMW) const924 SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
925 // Don't expand subword operations as they require special treatment.
926 if (RMW->getType()->isIntegerTy(8) || RMW->getType()->isIntegerTy(16))
927 return AtomicExpansionKind::None;
928
929 // Don't expand if there is a target instruction available.
930 if (Subtarget.hasInterlockedAccess1() &&
931 (RMW->getType()->isIntegerTy(32) || RMW->getType()->isIntegerTy(64)) &&
932 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
933 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
934 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
935 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
936 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
937 return AtomicExpansionKind::None;
938
939 return AtomicExpansionKind::CmpXChg;
940 }
941
isLegalICmpImmediate(int64_t Imm) const942 bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
943 // We can use CGFI or CLGFI.
944 return isInt<32>(Imm) || isUInt<32>(Imm);
945 }
946
isLegalAddImmediate(int64_t Imm) const947 bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
948 // We can use ALGFI or SLGFI.
949 return isUInt<32>(Imm) || isUInt<32>(-Imm);
950 }
951
allowsMisalignedMemoryAccesses(EVT VT,unsigned,Align,MachineMemOperand::Flags,unsigned * Fast) const952 bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
953 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
954 // Unaligned accesses should never be slower than the expanded version.
955 // We check specifically for aligned accesses in the few cases where
956 // they are required.
957 if (Fast)
958 *Fast = 1;
959 return true;
960 }
961
962 // Information about the addressing mode for a memory access.
963 struct AddressingMode {
964 // True if a long displacement is supported.
965 bool LongDisplacement;
966
967 // True if use of index register is supported.
968 bool IndexReg;
969
AddressingModeAddressingMode970 AddressingMode(bool LongDispl, bool IdxReg) :
971 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
972 };
973
974 // Return the desired addressing mode for a Load which has only one use (in
975 // the same block) which is a Store.
getLoadStoreAddrMode(bool HasVector,Type * Ty)976 static AddressingMode getLoadStoreAddrMode(bool HasVector,
977 Type *Ty) {
978 // With vector support a Load->Store combination may be combined to either
979 // an MVC or vector operations and it seems to work best to allow the
980 // vector addressing mode.
981 if (HasVector)
982 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
983
984 // Otherwise only the MVC case is special.
985 bool MVC = Ty->isIntegerTy(8);
986 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
987 }
988
989 // Return the addressing mode which seems most desirable given an LLVM
990 // Instruction pointer.
991 static AddressingMode
supportedAddressingMode(Instruction * I,bool HasVector)992 supportedAddressingMode(Instruction *I, bool HasVector) {
993 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
994 switch (II->getIntrinsicID()) {
995 default: break;
996 case Intrinsic::memset:
997 case Intrinsic::memmove:
998 case Intrinsic::memcpy:
999 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1000 }
1001 }
1002
1003 if (isa<LoadInst>(I) && I->hasOneUse()) {
1004 auto *SingleUser = cast<Instruction>(*I->user_begin());
1005 if (SingleUser->getParent() == I->getParent()) {
1006 if (isa<ICmpInst>(SingleUser)) {
1007 if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
1008 if (C->getBitWidth() <= 64 &&
1009 (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue())))
1010 // Comparison of memory with 16 bit signed / unsigned immediate
1011 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1012 } else if (isa<StoreInst>(SingleUser))
1013 // Load->Store
1014 return getLoadStoreAddrMode(HasVector, I->getType());
1015 }
1016 } else if (auto *StoreI = dyn_cast<StoreInst>(I)) {
1017 if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
1018 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1019 // Load->Store
1020 return getLoadStoreAddrMode(HasVector, LoadI->getType());
1021 }
1022
1023 if (HasVector && (isa<LoadInst>(I) || isa<StoreInst>(I))) {
1024
1025 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1026 // dependencies (LDE only supports small offsets).
1027 // * Utilize the vector registers to hold floating point
1028 // values (vector load / store instructions only support small
1029 // offsets).
1030
1031 Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() :
1032 I->getOperand(0)->getType());
1033 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1034 bool IsVectorAccess = MemAccessTy->isVectorTy();
1035
1036 // A store of an extracted vector element will be combined into a VSTE type
1037 // instruction.
1038 if (!IsVectorAccess && isa<StoreInst>(I)) {
1039 Value *DataOp = I->getOperand(0);
1040 if (isa<ExtractElementInst>(DataOp))
1041 IsVectorAccess = true;
1042 }
1043
1044 // A load which gets inserted into a vector element will be combined into a
1045 // VLE type instruction.
1046 if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) {
1047 User *LoadUser = *I->user_begin();
1048 if (isa<InsertElementInst>(LoadUser))
1049 IsVectorAccess = true;
1050 }
1051
1052 if (IsFPAccess || IsVectorAccess)
1053 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1054 }
1055
1056 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1057 }
1058
isLegalAddressingMode(const DataLayout & DL,const AddrMode & AM,Type * Ty,unsigned AS,Instruction * I) const1059 bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1060 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1061 // Punt on globals for now, although they can be used in limited
1062 // RELATIVE LONG cases.
1063 if (AM.BaseGV)
1064 return false;
1065
1066 // Require a 20-bit signed offset.
1067 if (!isInt<20>(AM.BaseOffs))
1068 return false;
1069
1070 bool RequireD12 =
1071 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(128));
1072 AddressingMode SupportedAM(!RequireD12, true);
1073 if (I != nullptr)
1074 SupportedAM = supportedAddressingMode(I, Subtarget.hasVector());
1075
1076 if (!SupportedAM.LongDisplacement && !isUInt<12>(AM.BaseOffs))
1077 return false;
1078
1079 if (!SupportedAM.IndexReg)
1080 // No indexing allowed.
1081 return AM.Scale == 0;
1082 else
1083 // Indexing is OK but no scale factor can be applied.
1084 return AM.Scale == 0 || AM.Scale == 1;
1085 }
1086
findOptimalMemOpLowering(std::vector<EVT> & MemOps,unsigned Limit,const MemOp & Op,unsigned DstAS,unsigned SrcAS,const AttributeList & FuncAttributes) const1087 bool SystemZTargetLowering::findOptimalMemOpLowering(
1088 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
1089 unsigned SrcAS, const AttributeList &FuncAttributes) const {
1090 const int MVCFastLen = 16;
1091
1092 if (Limit != ~unsigned(0)) {
1093 // Don't expand Op into scalar loads/stores in these cases:
1094 if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen)
1095 return false; // Small memcpy: Use MVC
1096 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1097 return false; // Small memset (first byte with STC/MVI): Use MVC
1098 if (Op.isZeroMemset())
1099 return false; // Memset zero: Use XC
1100 }
1101
1102 return TargetLowering::findOptimalMemOpLowering(MemOps, Limit, Op, DstAS,
1103 SrcAS, FuncAttributes);
1104 }
1105
getOptimalMemOpType(const MemOp & Op,const AttributeList & FuncAttributes) const1106 EVT SystemZTargetLowering::getOptimalMemOpType(const MemOp &Op,
1107 const AttributeList &FuncAttributes) const {
1108 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1109 }
1110
isTruncateFree(Type * FromType,Type * ToType) const1111 bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1112 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1113 return false;
1114 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1115 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1116 return FromBits > ToBits;
1117 }
1118
isTruncateFree(EVT FromVT,EVT ToVT) const1119 bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1120 if (!FromVT.isInteger() || !ToVT.isInteger())
1121 return false;
1122 unsigned FromBits = FromVT.getFixedSizeInBits();
1123 unsigned ToBits = ToVT.getFixedSizeInBits();
1124 return FromBits > ToBits;
1125 }
1126
1127 //===----------------------------------------------------------------------===//
1128 // Inline asm support
1129 //===----------------------------------------------------------------------===//
1130
1131 TargetLowering::ConstraintType
getConstraintType(StringRef Constraint) const1132 SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1133 if (Constraint.size() == 1) {
1134 switch (Constraint[0]) {
1135 case 'a': // Address register
1136 case 'd': // Data register (equivalent to 'r')
1137 case 'f': // Floating-point register
1138 case 'h': // High-part register
1139 case 'r': // General-purpose register
1140 case 'v': // Vector register
1141 return C_RegisterClass;
1142
1143 case 'Q': // Memory with base and unsigned 12-bit displacement
1144 case 'R': // Likewise, plus an index
1145 case 'S': // Memory with base and signed 20-bit displacement
1146 case 'T': // Likewise, plus an index
1147 case 'm': // Equivalent to 'T'.
1148 return C_Memory;
1149
1150 case 'I': // Unsigned 8-bit constant
1151 case 'J': // Unsigned 12-bit constant
1152 case 'K': // Signed 16-bit constant
1153 case 'L': // Signed 20-bit displacement (on all targets we support)
1154 case 'M': // 0x7fffffff
1155 return C_Immediate;
1156
1157 default:
1158 break;
1159 }
1160 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1161 switch (Constraint[1]) {
1162 case 'Q': // Address with base and unsigned 12-bit displacement
1163 case 'R': // Likewise, plus an index
1164 case 'S': // Address with base and signed 20-bit displacement
1165 case 'T': // Likewise, plus an index
1166 return C_Address;
1167
1168 default:
1169 break;
1170 }
1171 }
1172 return TargetLowering::getConstraintType(Constraint);
1173 }
1174
1175 TargetLowering::ConstraintWeight SystemZTargetLowering::
getSingleConstraintMatchWeight(AsmOperandInfo & info,const char * constraint) const1176 getSingleConstraintMatchWeight(AsmOperandInfo &info,
1177 const char *constraint) const {
1178 ConstraintWeight weight = CW_Invalid;
1179 Value *CallOperandVal = info.CallOperandVal;
1180 // If we don't have a value, we can't do a match,
1181 // but allow it at the lowest weight.
1182 if (!CallOperandVal)
1183 return CW_Default;
1184 Type *type = CallOperandVal->getType();
1185 // Look at the constraint type.
1186 switch (*constraint) {
1187 default:
1188 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
1189 break;
1190
1191 case 'a': // Address register
1192 case 'd': // Data register (equivalent to 'r')
1193 case 'h': // High-part register
1194 case 'r': // General-purpose register
1195 weight = CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1196 break;
1197
1198 case 'f': // Floating-point register
1199 if (!useSoftFloat())
1200 weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1201 break;
1202
1203 case 'v': // Vector register
1204 if (Subtarget.hasVector())
1205 weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1206 : CW_Default;
1207 break;
1208
1209 case 'I': // Unsigned 8-bit constant
1210 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1211 if (isUInt<8>(C->getZExtValue()))
1212 weight = CW_Constant;
1213 break;
1214
1215 case 'J': // Unsigned 12-bit constant
1216 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1217 if (isUInt<12>(C->getZExtValue()))
1218 weight = CW_Constant;
1219 break;
1220
1221 case 'K': // Signed 16-bit constant
1222 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1223 if (isInt<16>(C->getSExtValue()))
1224 weight = CW_Constant;
1225 break;
1226
1227 case 'L': // Signed 20-bit displacement (on all targets we support)
1228 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1229 if (isInt<20>(C->getSExtValue()))
1230 weight = CW_Constant;
1231 break;
1232
1233 case 'M': // 0x7fffffff
1234 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
1235 if (C->getZExtValue() == 0x7fffffff)
1236 weight = CW_Constant;
1237 break;
1238 }
1239 return weight;
1240 }
1241
1242 // Parse a "{tNNN}" register constraint for which the register type "t"
1243 // has already been verified. MC is the class associated with "t" and
1244 // Map maps 0-based register numbers to LLVM register numbers.
1245 static std::pair<unsigned, const TargetRegisterClass *>
parseRegisterNumber(StringRef Constraint,const TargetRegisterClass * RC,const unsigned * Map,unsigned Size)1246 parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1247 const unsigned *Map, unsigned Size) {
1248 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1249 if (isdigit(Constraint[2])) {
1250 unsigned Index;
1251 bool Failed =
1252 Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);
1253 if (!Failed && Index < Size && Map[Index])
1254 return std::make_pair(Map[Index], RC);
1255 }
1256 return std::make_pair(0U, nullptr);
1257 }
1258
1259 std::pair<unsigned, const TargetRegisterClass *>
getRegForInlineAsmConstraint(const TargetRegisterInfo * TRI,StringRef Constraint,MVT VT) const1260 SystemZTargetLowering::getRegForInlineAsmConstraint(
1261 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1262 if (Constraint.size() == 1) {
1263 // GCC Constraint Letters
1264 switch (Constraint[0]) {
1265 default: break;
1266 case 'd': // Data register (equivalent to 'r')
1267 case 'r': // General-purpose register
1268 if (VT.getSizeInBits() == 64)
1269 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1270 else if (VT.getSizeInBits() == 128)
1271 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1272 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1273
1274 case 'a': // Address register
1275 if (VT == MVT::i64)
1276 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1277 else if (VT == MVT::i128)
1278 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1279 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1280
1281 case 'h': // High-part register (an LLVM extension)
1282 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1283
1284 case 'f': // Floating-point register
1285 if (!useSoftFloat()) {
1286 if (VT.getSizeInBits() == 64)
1287 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1288 else if (VT.getSizeInBits() == 128)
1289 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1290 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1291 }
1292 break;
1293
1294 case 'v': // Vector register
1295 if (Subtarget.hasVector()) {
1296 if (VT.getSizeInBits() == 32)
1297 return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1298 if (VT.getSizeInBits() == 64)
1299 return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1300 return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1301 }
1302 break;
1303 }
1304 }
1305 if (Constraint.size() > 0 && Constraint[0] == '{') {
1306
1307 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1308 // to check the size on.
1309 auto getVTSizeInBits = [&VT]() {
1310 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1311 };
1312
1313 // We need to override the default register parsing for GPRs and FPRs
1314 // because the interpretation depends on VT. The internal names of
1315 // the registers are also different from the external names
1316 // (F0D and F0S instead of F0, etc.).
1317 if (Constraint[1] == 'r') {
1318 if (getVTSizeInBits() == 32)
1319 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass,
1320 SystemZMC::GR32Regs, 16);
1321 if (getVTSizeInBits() == 128)
1322 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass,
1323 SystemZMC::GR128Regs, 16);
1324 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass,
1325 SystemZMC::GR64Regs, 16);
1326 }
1327 if (Constraint[1] == 'f') {
1328 if (useSoftFloat())
1329 return std::make_pair(
1330 0u, static_cast<const TargetRegisterClass *>(nullptr));
1331 if (getVTSizeInBits() == 32)
1332 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
1333 SystemZMC::FP32Regs, 16);
1334 if (getVTSizeInBits() == 128)
1335 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass,
1336 SystemZMC::FP128Regs, 16);
1337 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass,
1338 SystemZMC::FP64Regs, 16);
1339 }
1340 if (Constraint[1] == 'v') {
1341 if (!Subtarget.hasVector())
1342 return std::make_pair(
1343 0u, static_cast<const TargetRegisterClass *>(nullptr));
1344 if (getVTSizeInBits() == 32)
1345 return parseRegisterNumber(Constraint, &SystemZ::VR32BitRegClass,
1346 SystemZMC::VR32Regs, 32);
1347 if (getVTSizeInBits() == 64)
1348 return parseRegisterNumber(Constraint, &SystemZ::VR64BitRegClass,
1349 SystemZMC::VR64Regs, 32);
1350 return parseRegisterNumber(Constraint, &SystemZ::VR128BitRegClass,
1351 SystemZMC::VR128Regs, 32);
1352 }
1353 }
1354 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1355 }
1356
1357 // FIXME? Maybe this could be a TableGen attribute on some registers and
1358 // this table could be generated automatically from RegInfo.
1359 Register
getRegisterByName(const char * RegName,LLT VT,const MachineFunction & MF) const1360 SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1361 const MachineFunction &MF) const {
1362 Register Reg =
1363 StringSwitch<Register>(RegName)
1364 .Case("r4", Subtarget.isTargetXPLINK64() ? SystemZ::R4D : 0)
1365 .Case("r15", Subtarget.isTargetELF() ? SystemZ::R15D : 0)
1366 .Default(0);
1367
1368 if (Reg)
1369 return Reg;
1370 report_fatal_error("Invalid register name global variable");
1371 }
1372
getExceptionPointerRegister(const Constant * PersonalityFn) const1373 Register SystemZTargetLowering::getExceptionPointerRegister(
1374 const Constant *PersonalityFn) const {
1375 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1376 }
1377
getExceptionSelectorRegister(const Constant * PersonalityFn) const1378 Register SystemZTargetLowering::getExceptionSelectorRegister(
1379 const Constant *PersonalityFn) const {
1380 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1381 }
1382
LowerAsmOperandForConstraint(SDValue Op,StringRef Constraint,std::vector<SDValue> & Ops,SelectionDAG & DAG) const1383 void SystemZTargetLowering::LowerAsmOperandForConstraint(
1384 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1385 SelectionDAG &DAG) const {
1386 // Only support length 1 constraints for now.
1387 if (Constraint.size() == 1) {
1388 switch (Constraint[0]) {
1389 case 'I': // Unsigned 8-bit constant
1390 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1391 if (isUInt<8>(C->getZExtValue()))
1392 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1393 Op.getValueType()));
1394 return;
1395
1396 case 'J': // Unsigned 12-bit constant
1397 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1398 if (isUInt<12>(C->getZExtValue()))
1399 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1400 Op.getValueType()));
1401 return;
1402
1403 case 'K': // Signed 16-bit constant
1404 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1405 if (isInt<16>(C->getSExtValue()))
1406 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
1407 Op.getValueType()));
1408 return;
1409
1410 case 'L': // Signed 20-bit displacement (on all targets we support)
1411 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1412 if (isInt<20>(C->getSExtValue()))
1413 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
1414 Op.getValueType()));
1415 return;
1416
1417 case 'M': // 0x7fffffff
1418 if (auto *C = dyn_cast<ConstantSDNode>(Op))
1419 if (C->getZExtValue() == 0x7fffffff)
1420 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
1421 Op.getValueType()));
1422 return;
1423 }
1424 }
1425 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1426 }
1427
1428 //===----------------------------------------------------------------------===//
1429 // Calling conventions
1430 //===----------------------------------------------------------------------===//
1431
1432 #include "SystemZGenCallingConv.inc"
1433
getScratchRegisters(CallingConv::ID) const1434 const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1435 CallingConv::ID) const {
1436 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1437 SystemZ::R14D, 0 };
1438 return ScratchRegs;
1439 }
1440
allowTruncateForTailCall(Type * FromType,Type * ToType) const1441 bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1442 Type *ToType) const {
1443 return isTruncateFree(FromType, ToType);
1444 }
1445
mayBeEmittedAsTailCall(const CallInst * CI) const1446 bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1447 return CI->isTailCall();
1448 }
1449
1450 // Value is a value that has been passed to us in the location described by VA
1451 // (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1452 // any loads onto Chain.
convertLocVTToValVT(SelectionDAG & DAG,const SDLoc & DL,CCValAssign & VA,SDValue Chain,SDValue Value)1453 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1454 CCValAssign &VA, SDValue Chain,
1455 SDValue Value) {
1456 // If the argument has been promoted from a smaller type, insert an
1457 // assertion to capture this.
1458 if (VA.getLocInfo() == CCValAssign::SExt)
1459 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value,
1460 DAG.getValueType(VA.getValVT()));
1461 else if (VA.getLocInfo() == CCValAssign::ZExt)
1462 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value,
1463 DAG.getValueType(VA.getValVT()));
1464
1465 if (VA.isExtInLoc())
1466 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
1467 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1468 // If this is a short vector argument loaded from the stack,
1469 // extend from i64 to full vector size and then bitcast.
1470 assert(VA.getLocVT() == MVT::i64);
1471 assert(VA.getValVT().isVector());
1472 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
1473 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
1474 } else
1475 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1476 return Value;
1477 }
1478
1479 // Value is a value of type VA.getValVT() that we need to copy into
1480 // the location described by VA. Return a copy of Value converted to
1481 // VA.getValVT(). The caller is responsible for handling indirect values.
convertValVTToLocVT(SelectionDAG & DAG,const SDLoc & DL,CCValAssign & VA,SDValue Value)1482 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1483 CCValAssign &VA, SDValue Value) {
1484 switch (VA.getLocInfo()) {
1485 case CCValAssign::SExt:
1486 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
1487 case CCValAssign::ZExt:
1488 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
1489 case CCValAssign::AExt:
1490 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
1491 case CCValAssign::BCvt: {
1492 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
1493 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
1494 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
1495 // For an f32 vararg we need to first promote it to an f64 and then
1496 // bitcast it to an i64.
1497 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
1498 Value = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, Value);
1499 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
1500 ? MVT::v2i64
1501 : VA.getLocVT();
1502 Value = DAG.getNode(ISD::BITCAST, DL, BitCastToType, Value);
1503 // For ELF, this is a short vector argument to be stored to the stack,
1504 // bitcast to v2i64 and then extract first element.
1505 if (BitCastToType == MVT::v2i64)
1506 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
1507 DAG.getConstant(0, DL, MVT::i32));
1508 return Value;
1509 }
1510 case CCValAssign::Full:
1511 return Value;
1512 default:
1513 llvm_unreachable("Unhandled getLocInfo()");
1514 }
1515 }
1516
lowerI128ToGR128(SelectionDAG & DAG,SDValue In)1517 static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1518 SDLoc DL(In);
1519 SDValue Lo, Hi;
1520 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128)) {
1521 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, In);
1522 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
1523 DAG.getNode(ISD::SRL, DL, MVT::i128, In,
1524 DAG.getConstant(64, DL, MVT::i32)));
1525 } else {
1526 std::tie(Lo, Hi) = DAG.SplitScalar(In, DL, MVT::i64, MVT::i64);
1527 }
1528
1529 SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
1530 MVT::Untyped, Hi, Lo);
1531 return SDValue(Pair, 0);
1532 }
1533
lowerGR128ToI128(SelectionDAG & DAG,SDValue In)1534 static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1535 SDLoc DL(In);
1536 SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
1537 DL, MVT::i64, In);
1538 SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
1539 DL, MVT::i64, In);
1540
1541 if (DAG.getTargetLoweringInfo().isTypeLegal(MVT::i128)) {
1542 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Lo);
1543 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, Hi);
1544 Hi = DAG.getNode(ISD::SHL, DL, MVT::i128, Hi,
1545 DAG.getConstant(64, DL, MVT::i32));
1546 return DAG.getNode(ISD::OR, DL, MVT::i128, Lo, Hi);
1547 } else {
1548 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
1549 }
1550 }
1551
splitValueIntoRegisterParts(SelectionDAG & DAG,const SDLoc & DL,SDValue Val,SDValue * Parts,unsigned NumParts,MVT PartVT,std::optional<CallingConv::ID> CC) const1552 bool SystemZTargetLowering::splitValueIntoRegisterParts(
1553 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1554 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
1555 EVT ValueVT = Val.getValueType();
1556 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1557 // Inline assembly operand.
1558 Parts[0] = lowerI128ToGR128(DAG, DAG.getBitcast(MVT::i128, Val));
1559 return true;
1560 }
1561
1562 return false;
1563 }
1564
joinRegisterPartsIntoValue(SelectionDAG & DAG,const SDLoc & DL,const SDValue * Parts,unsigned NumParts,MVT PartVT,EVT ValueVT,std::optional<CallingConv::ID> CC) const1565 SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
1566 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
1567 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
1568 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1569 // Inline assembly operand.
1570 SDValue Res = lowerGR128ToI128(DAG, Parts[0]);
1571 return DAG.getBitcast(ValueVT, Res);
1572 }
1573
1574 return SDValue();
1575 }
1576
LowerFormalArguments(SDValue Chain,CallingConv::ID CallConv,bool IsVarArg,const SmallVectorImpl<ISD::InputArg> & Ins,const SDLoc & DL,SelectionDAG & DAG,SmallVectorImpl<SDValue> & InVals) const1577 SDValue SystemZTargetLowering::LowerFormalArguments(
1578 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1579 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1580 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1581 MachineFunction &MF = DAG.getMachineFunction();
1582 MachineFrameInfo &MFI = MF.getFrameInfo();
1583 MachineRegisterInfo &MRI = MF.getRegInfo();
1584 SystemZMachineFunctionInfo *FuncInfo =
1585 MF.getInfo<SystemZMachineFunctionInfo>();
1586 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
1587 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1588
1589 // Assign locations to all of the incoming arguments.
1590 SmallVector<CCValAssign, 16> ArgLocs;
1591 SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1592 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
1593 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
1594
1595 unsigned NumFixedGPRs = 0;
1596 unsigned NumFixedFPRs = 0;
1597 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1598 SDValue ArgValue;
1599 CCValAssign &VA = ArgLocs[I];
1600 EVT LocVT = VA.getLocVT();
1601 if (VA.isRegLoc()) {
1602 // Arguments passed in registers
1603 const TargetRegisterClass *RC;
1604 switch (LocVT.getSimpleVT().SimpleTy) {
1605 default:
1606 // Integers smaller than i64 should be promoted to i64.
1607 llvm_unreachable("Unexpected argument type");
1608 case MVT::i32:
1609 NumFixedGPRs += 1;
1610 RC = &SystemZ::GR32BitRegClass;
1611 break;
1612 case MVT::i64:
1613 NumFixedGPRs += 1;
1614 RC = &SystemZ::GR64BitRegClass;
1615 break;
1616 case MVT::f32:
1617 NumFixedFPRs += 1;
1618 RC = &SystemZ::FP32BitRegClass;
1619 break;
1620 case MVT::f64:
1621 NumFixedFPRs += 1;
1622 RC = &SystemZ::FP64BitRegClass;
1623 break;
1624 case MVT::f128:
1625 NumFixedFPRs += 2;
1626 RC = &SystemZ::FP128BitRegClass;
1627 break;
1628 case MVT::v16i8:
1629 case MVT::v8i16:
1630 case MVT::v4i32:
1631 case MVT::v2i64:
1632 case MVT::v4f32:
1633 case MVT::v2f64:
1634 RC = &SystemZ::VR128BitRegClass;
1635 break;
1636 }
1637
1638 Register VReg = MRI.createVirtualRegister(RC);
1639 MRI.addLiveIn(VA.getLocReg(), VReg);
1640 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1641 } else {
1642 assert(VA.isMemLoc() && "Argument not register or memory");
1643
1644 // Create the frame index object for this incoming parameter.
1645 // FIXME: Pre-include call frame size in the offset, should not
1646 // need to manually add it here.
1647 int64_t ArgSPOffset = VA.getLocMemOffset();
1648 if (Subtarget.isTargetXPLINK64()) {
1649 auto &XPRegs =
1650 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1651 ArgSPOffset += XPRegs.getCallFrameSize();
1652 }
1653 int FI =
1654 MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, ArgSPOffset, true);
1655
1656 // Create the SelectionDAG nodes corresponding to a load
1657 // from this parameter. Unpromoted ints and floats are
1658 // passed as right-justified 8-byte values.
1659 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1660 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
1661 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
1662 DAG.getIntPtrConstant(4, DL));
1663 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
1664 MachinePointerInfo::getFixedStack(MF, FI));
1665 }
1666
1667 // Convert the value of the argument register into the value that's
1668 // being passed.
1669 if (VA.getLocInfo() == CCValAssign::Indirect) {
1670 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1671 MachinePointerInfo()));
1672 // If the original argument was split (e.g. i128), we need
1673 // to load all parts of it here (using the same address).
1674 unsigned ArgIndex = Ins[I].OrigArgIndex;
1675 assert (Ins[I].PartOffset == 0);
1676 while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) {
1677 CCValAssign &PartVA = ArgLocs[I + 1];
1678 unsigned PartOffset = Ins[I + 1].PartOffset;
1679 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1680 DAG.getIntPtrConstant(PartOffset, DL));
1681 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1682 MachinePointerInfo()));
1683 ++I;
1684 }
1685 } else
1686 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
1687 }
1688
1689 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
1690 // Save the number of non-varargs registers for later use by va_start, etc.
1691 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
1692 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
1693
1694 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
1695 Subtarget.getSpecialRegisters());
1696
1697 // Likewise the address (in the form of a frame index) of where the
1698 // first stack vararg would be. The 1-byte size here is arbitrary.
1699 // FIXME: Pre-include call frame size in the offset, should not
1700 // need to manually add it here.
1701 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
1702 int FI = MFI.CreateFixedObject(1, VarArgOffset, true);
1703 FuncInfo->setVarArgsFrameIndex(FI);
1704 }
1705
1706 if (IsVarArg && Subtarget.isTargetELF()) {
1707 // Save the number of non-varargs registers for later use by va_start, etc.
1708 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
1709 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
1710
1711 // Likewise the address (in the form of a frame index) of where the
1712 // first stack vararg would be. The 1-byte size here is arbitrary.
1713 int64_t VarArgsOffset = CCInfo.getStackSize();
1714 FuncInfo->setVarArgsFrameIndex(
1715 MFI.CreateFixedObject(1, VarArgsOffset, true));
1716
1717 // ...and a similar frame index for the caller-allocated save area
1718 // that will be used to store the incoming registers.
1719 int64_t RegSaveOffset =
1720 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, SystemZ::R2D) - 16;
1721 unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true);
1722 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
1723
1724 // Store the FPR varargs in the reserved frame slots. (We store the
1725 // GPRs as part of the prologue.)
1726 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
1727 SDValue MemOps[SystemZ::ELFNumArgFPRs];
1728 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
1729 unsigned Offset = TFL->getRegSpillOffset(MF, SystemZ::ELFArgFPRs[I]);
1730 int FI =
1731 MFI.CreateFixedObject(8, -SystemZMC::ELFCallFrameSize + Offset, true);
1732 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1733 Register VReg = MF.addLiveIn(SystemZ::ELFArgFPRs[I],
1734 &SystemZ::FP64BitRegClass);
1735 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
1736 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
1737 MachinePointerInfo::getFixedStack(MF, FI));
1738 }
1739 // Join the stores, which are independent of one another.
1740 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1741 ArrayRef(&MemOps[NumFixedFPRs],
1742 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
1743 }
1744 }
1745
1746 if (Subtarget.isTargetXPLINK64()) {
1747 // Create virual register for handling incoming "ADA" special register (R5)
1748 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
1749 Register ADAvReg = MRI.createVirtualRegister(RC);
1750 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
1751 Subtarget.getSpecialRegisters());
1752 MRI.addLiveIn(Regs->getADARegister(), ADAvReg);
1753 FuncInfo->setADAVirtualRegister(ADAvReg);
1754 }
1755 return Chain;
1756 }
1757
canUseSiblingCall(const CCState & ArgCCInfo,SmallVectorImpl<CCValAssign> & ArgLocs,SmallVectorImpl<ISD::OutputArg> & Outs)1758 static bool canUseSiblingCall(const CCState &ArgCCInfo,
1759 SmallVectorImpl<CCValAssign> &ArgLocs,
1760 SmallVectorImpl<ISD::OutputArg> &Outs) {
1761 // Punt if there are any indirect or stack arguments, or if the call
1762 // needs the callee-saved argument register R6, or if the call uses
1763 // the callee-saved register arguments SwiftSelf and SwiftError.
1764 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1765 CCValAssign &VA = ArgLocs[I];
1766 if (VA.getLocInfo() == CCValAssign::Indirect)
1767 return false;
1768 if (!VA.isRegLoc())
1769 return false;
1770 Register Reg = VA.getLocReg();
1771 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
1772 return false;
1773 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
1774 return false;
1775 }
1776 return true;
1777 }
1778
getADAEntry(SelectionDAG & DAG,SDValue Val,SDLoc DL,unsigned Offset,bool LoadAdr=false)1779 static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
1780 unsigned Offset, bool LoadAdr = false) {
1781 MachineFunction &MF = DAG.getMachineFunction();
1782 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
1783 unsigned ADAvReg = MFI->getADAVirtualRegister();
1784 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1785
1786 SDValue Reg = DAG.getRegister(ADAvReg, PtrVT);
1787 SDValue Ofs = DAG.getTargetConstant(Offset, DL, PtrVT);
1788
1789 SDValue Result = DAG.getNode(SystemZISD::ADA_ENTRY, DL, PtrVT, Val, Reg, Ofs);
1790 if (!LoadAdr)
1791 Result = DAG.getLoad(
1792 PtrVT, DL, DAG.getEntryNode(), Result, MachinePointerInfo(), Align(8),
1793 MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
1794
1795 return Result;
1796 }
1797
1798 // ADA access using Global value
1799 // Note: for functions, address of descriptor is returned
getADAEntry(SelectionDAG & DAG,const GlobalValue * GV,SDLoc DL,EVT PtrVT)1800 static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
1801 EVT PtrVT) {
1802 unsigned ADAtype;
1803 bool LoadAddr = false;
1804 const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV);
1805 bool IsFunction =
1806 (isa<Function>(GV)) || (GA && isa<Function>(GA->getAliaseeObject()));
1807 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
1808
1809 if (IsFunction) {
1810 if (IsInternal) {
1811 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
1812 LoadAddr = true;
1813 } else
1814 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
1815 } else {
1816 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
1817 }
1818 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, ADAtype);
1819
1820 return getADAEntry(DAG, Val, DL, 0, LoadAddr);
1821 }
1822
getzOSCalleeAndADA(SelectionDAG & DAG,SDValue & Callee,SDValue & ADA,SDLoc & DL,SDValue & Chain)1823 static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
1824 SDLoc &DL, SDValue &Chain) {
1825 unsigned ADADelta = 0; // ADA offset in desc.
1826 unsigned EPADelta = 8; // EPA offset in desc.
1827 MachineFunction &MF = DAG.getMachineFunction();
1828 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1829
1830 // XPLink calling convention.
1831 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1832 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
1833 G->getGlobal()->hasPrivateLinkage());
1834 if (IsInternal) {
1835 SystemZMachineFunctionInfo *MFI =
1836 MF.getInfo<SystemZMachineFunctionInfo>();
1837 unsigned ADAvReg = MFI->getADAVirtualRegister();
1838 ADA = DAG.getCopyFromReg(Chain, DL, ADAvReg, PtrVT);
1839 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
1840 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1841 return true;
1842 } else {
1843 SDValue GA = DAG.getTargetGlobalAddress(
1844 G->getGlobal(), DL, PtrVT, 0, SystemZII::MO_ADA_DIRECT_FUNC_DESC);
1845 ADA = getADAEntry(DAG, GA, DL, ADADelta);
1846 Callee = getADAEntry(DAG, GA, DL, EPADelta);
1847 }
1848 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1849 SDValue ES = DAG.getTargetExternalSymbol(
1850 E->getSymbol(), PtrVT, SystemZII::MO_ADA_DIRECT_FUNC_DESC);
1851 ADA = getADAEntry(DAG, ES, DL, ADADelta);
1852 Callee = getADAEntry(DAG, ES, DL, EPADelta);
1853 } else {
1854 // Function pointer case
1855 ADA = DAG.getNode(ISD::ADD, DL, PtrVT, Callee,
1856 DAG.getConstant(ADADelta, DL, PtrVT));
1857 ADA = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), ADA,
1858 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
1859 Callee = DAG.getNode(ISD::ADD, DL, PtrVT, Callee,
1860 DAG.getConstant(EPADelta, DL, PtrVT));
1861 Callee = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Callee,
1862 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
1863 }
1864 return false;
1865 }
1866
1867 SDValue
LowerCall(CallLoweringInfo & CLI,SmallVectorImpl<SDValue> & InVals) const1868 SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
1869 SmallVectorImpl<SDValue> &InVals) const {
1870 SelectionDAG &DAG = CLI.DAG;
1871 SDLoc &DL = CLI.DL;
1872 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1873 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1874 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1875 SDValue Chain = CLI.Chain;
1876 SDValue Callee = CLI.Callee;
1877 bool &IsTailCall = CLI.IsTailCall;
1878 CallingConv::ID CallConv = CLI.CallConv;
1879 bool IsVarArg = CLI.IsVarArg;
1880 MachineFunction &MF = DAG.getMachineFunction();
1881 EVT PtrVT = getPointerTy(MF.getDataLayout());
1882 LLVMContext &Ctx = *DAG.getContext();
1883 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
1884
1885 // FIXME: z/OS support to be added in later.
1886 if (Subtarget.isTargetXPLINK64())
1887 IsTailCall = false;
1888
1889 // Analyze the operands of the call, assigning locations to each operand.
1890 SmallVector<CCValAssign, 16> ArgLocs;
1891 SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
1892 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
1893
1894 // We don't support GuaranteedTailCallOpt, only automatically-detected
1895 // sibling calls.
1896 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
1897 IsTailCall = false;
1898
1899 // Get a count of how many bytes are to be pushed on the stack.
1900 unsigned NumBytes = ArgCCInfo.getStackSize();
1901
1902 // Mark the start of the call.
1903 if (!IsTailCall)
1904 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
1905
1906 // Copy argument values to their designated locations.
1907 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
1908 SmallVector<SDValue, 8> MemOpChains;
1909 SDValue StackPtr;
1910 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
1911 CCValAssign &VA = ArgLocs[I];
1912 SDValue ArgValue = OutVals[I];
1913
1914 if (VA.getLocInfo() == CCValAssign::Indirect) {
1915 // Store the argument in a stack slot and pass its address.
1916 unsigned ArgIndex = Outs[I].OrigArgIndex;
1917 EVT SlotVT;
1918 if (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
1919 // Allocate the full stack space for a promoted (and split) argument.
1920 Type *OrigArgType = CLI.Args[Outs[I].OrigArgIndex].Ty;
1921 EVT OrigArgVT = getValueType(MF.getDataLayout(), OrigArgType);
1922 MVT PartVT = getRegisterTypeForCallingConv(Ctx, CLI.CallConv, OrigArgVT);
1923 unsigned N = getNumRegistersForCallingConv(Ctx, CLI.CallConv, OrigArgVT);
1924 SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * N);
1925 } else {
1926 SlotVT = Outs[I].VT;
1927 }
1928 SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT);
1929 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1930 MemOpChains.push_back(
1931 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
1932 MachinePointerInfo::getFixedStack(MF, FI)));
1933 // If the original argument was split (e.g. i128), we need
1934 // to store all parts of it here (and pass just one address).
1935 assert (Outs[I].PartOffset == 0);
1936 while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
1937 SDValue PartValue = OutVals[I + 1];
1938 unsigned PartOffset = Outs[I + 1].PartOffset;
1939 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
1940 DAG.getIntPtrConstant(PartOffset, DL));
1941 MemOpChains.push_back(
1942 DAG.getStore(Chain, DL, PartValue, Address,
1943 MachinePointerInfo::getFixedStack(MF, FI)));
1944 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
1945 SlotVT.getStoreSize()) && "Not enough space for argument part!");
1946 ++I;
1947 }
1948 ArgValue = SpillSlot;
1949 } else
1950 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
1951
1952 if (VA.isRegLoc()) {
1953 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
1954 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
1955 // and low values.
1956 if (VA.getLocVT() == MVT::i128)
1957 ArgValue = lowerI128ToGR128(DAG, ArgValue);
1958 // Queue up the argument copies and emit them at the end.
1959 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
1960 } else {
1961 assert(VA.isMemLoc() && "Argument not register or memory");
1962
1963 // Work out the address of the stack slot. Unpromoted ints and
1964 // floats are passed as right-justified 8-byte values.
1965 if (!StackPtr.getNode())
1966 StackPtr = DAG.getCopyFromReg(Chain, DL,
1967 Regs->getStackPointerRegister(), PtrVT);
1968 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
1969 VA.getLocMemOffset();
1970 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
1971 Offset += 4;
1972 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
1973 DAG.getIntPtrConstant(Offset, DL));
1974
1975 // Emit the store.
1976 MemOpChains.push_back(
1977 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
1978
1979 // Although long doubles or vectors are passed through the stack when
1980 // they are vararg (non-fixed arguments), if a long double or vector
1981 // occupies the third and fourth slot of the argument list GPR3 should
1982 // still shadow the third slot of the argument list.
1983 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
1984 SDValue ShadowArgValue =
1985 DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, ArgValue,
1986 DAG.getIntPtrConstant(1, DL));
1987 RegsToPass.push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
1988 }
1989 }
1990 }
1991
1992 // Join the stores, which are independent of one another.
1993 if (!MemOpChains.empty())
1994 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
1995
1996 // Accept direct calls by converting symbolic call addresses to the
1997 // associated Target* opcodes. Force %r1 to be used for indirect
1998 // tail calls.
1999 SDValue Glue;
2000
2001 if (Subtarget.isTargetXPLINK64()) {
2002 SDValue ADA;
2003 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2004 if (!IsBRASL) {
2005 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2006 ->getAddressOfCalleeRegister();
2007 Chain = DAG.getCopyToReg(Chain, DL, CalleeReg, Callee, Glue);
2008 Glue = Chain.getValue(1);
2009 Callee = DAG.getRegister(CalleeReg, Callee.getValueType());
2010 }
2011 RegsToPass.push_back(std::make_pair(
2012 static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), ADA));
2013 } else {
2014 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2015 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
2016 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2017 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2018 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);
2019 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
2020 } else if (IsTailCall) {
2021 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
2022 Glue = Chain.getValue(1);
2023 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
2024 }
2025 }
2026
2027 // Build a sequence of copy-to-reg nodes, chained and glued together.
2028 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
2029 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
2030 RegsToPass[I].second, Glue);
2031 Glue = Chain.getValue(1);
2032 }
2033
2034 // The first call operand is the chain and the second is the target address.
2035 SmallVector<SDValue, 8> Ops;
2036 Ops.push_back(Chain);
2037 Ops.push_back(Callee);
2038
2039 // Add argument registers to the end of the list so that they are
2040 // known live into the call.
2041 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
2042 Ops.push_back(DAG.getRegister(RegsToPass[I].first,
2043 RegsToPass[I].second.getValueType()));
2044
2045 // Add a register mask operand representing the call-preserved registers.
2046 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2047 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2048 assert(Mask && "Missing call preserved mask for calling convention");
2049 Ops.push_back(DAG.getRegisterMask(Mask));
2050
2051 // Glue the call to the argument copies, if any.
2052 if (Glue.getNode())
2053 Ops.push_back(Glue);
2054
2055 // Emit the call.
2056 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2057 if (IsTailCall) {
2058 SDValue Ret = DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
2059 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
2060 return Ret;
2061 }
2062 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
2063 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
2064 Glue = Chain.getValue(1);
2065
2066 // Mark the end of the call, which is glued to the call itself.
2067 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL);
2068 Glue = Chain.getValue(1);
2069
2070 // Assign locations to each value returned by this call.
2071 SmallVector<CCValAssign, 16> RetLocs;
2072 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2073 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ);
2074
2075 // Copy all of the result registers out of their specified physreg.
2076 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2077 CCValAssign &VA = RetLocs[I];
2078
2079 // Copy the value out, gluing the copy to the end of the call sequence.
2080 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
2081 VA.getLocVT(), Glue);
2082 Chain = RetValue.getValue(1);
2083 Glue = RetValue.getValue(2);
2084
2085 // Convert the value of the return register into the value that's
2086 // being returned.
2087 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
2088 }
2089
2090 return Chain;
2091 }
2092
2093 // Generate a call taking the given operands as arguments and returning a
2094 // result of type RetVT.
makeExternalCall(SDValue Chain,SelectionDAG & DAG,const char * CalleeName,EVT RetVT,ArrayRef<SDValue> Ops,CallingConv::ID CallConv,bool IsSigned,SDLoc DL,bool DoesNotReturn,bool IsReturnValueUsed) const2095 std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2096 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2097 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2098 bool DoesNotReturn, bool IsReturnValueUsed) const {
2099 TargetLowering::ArgListTy Args;
2100 Args.reserve(Ops.size());
2101
2102 TargetLowering::ArgListEntry Entry;
2103 for (SDValue Op : Ops) {
2104 Entry.Node = Op;
2105 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2106 Entry.IsSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), IsSigned);
2107 Entry.IsZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), IsSigned);
2108 Args.push_back(Entry);
2109 }
2110
2111 SDValue Callee =
2112 DAG.getExternalSymbol(CalleeName, getPointerTy(DAG.getDataLayout()));
2113
2114 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2115 TargetLowering::CallLoweringInfo CLI(DAG);
2116 bool SignExtend = shouldSignExtendTypeInLibCall(RetVT, IsSigned);
2117 CLI.setDebugLoc(DL)
2118 .setChain(Chain)
2119 .setCallee(CallConv, RetTy, Callee, std::move(Args))
2120 .setNoReturn(DoesNotReturn)
2121 .setDiscardResult(!IsReturnValueUsed)
2122 .setSExtResult(SignExtend)
2123 .setZExtResult(!SignExtend);
2124 return LowerCallTo(CLI);
2125 }
2126
2127 bool SystemZTargetLowering::
CanLowerReturn(CallingConv::ID CallConv,MachineFunction & MF,bool isVarArg,const SmallVectorImpl<ISD::OutputArg> & Outs,LLVMContext & Context) const2128 CanLowerReturn(CallingConv::ID CallConv,
2129 MachineFunction &MF, bool isVarArg,
2130 const SmallVectorImpl<ISD::OutputArg> &Outs,
2131 LLVMContext &Context) const {
2132 // Special case that we cannot easily detect in RetCC_SystemZ since
2133 // i128 may not be a legal type.
2134 for (auto &Out : Outs)
2135 if (Out.ArgVT == MVT::i128)
2136 return false;
2137
2138 SmallVector<CCValAssign, 16> RetLocs;
2139 CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context);
2140 return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ);
2141 }
2142
2143 SDValue
LowerReturn(SDValue Chain,CallingConv::ID CallConv,bool IsVarArg,const SmallVectorImpl<ISD::OutputArg> & Outs,const SmallVectorImpl<SDValue> & OutVals,const SDLoc & DL,SelectionDAG & DAG) const2144 SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2145 bool IsVarArg,
2146 const SmallVectorImpl<ISD::OutputArg> &Outs,
2147 const SmallVectorImpl<SDValue> &OutVals,
2148 const SDLoc &DL, SelectionDAG &DAG) const {
2149 MachineFunction &MF = DAG.getMachineFunction();
2150
2151 // Assign locations to each returned value.
2152 SmallVector<CCValAssign, 16> RetLocs;
2153 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2154 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ);
2155
2156 // Quick exit for void returns
2157 if (RetLocs.empty())
2158 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, Chain);
2159
2160 if (CallConv == CallingConv::GHC)
2161 report_fatal_error("GHC functions return void only");
2162
2163 // Copy the result values into the output registers.
2164 SDValue Glue;
2165 SmallVector<SDValue, 4> RetOps;
2166 RetOps.push_back(Chain);
2167 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2168 CCValAssign &VA = RetLocs[I];
2169 SDValue RetValue = OutVals[I];
2170
2171 // Make the return register live on exit.
2172 assert(VA.isRegLoc() && "Can only return in registers!");
2173
2174 // Promote the value as required.
2175 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
2176
2177 // Chain and glue the copies together.
2178 Register Reg = VA.getLocReg();
2179 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
2180 Glue = Chain.getValue(1);
2181 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT()));
2182 }
2183
2184 // Update chain and glue.
2185 RetOps[0] = Chain;
2186 if (Glue.getNode())
2187 RetOps.push_back(Glue);
2188
2189 return DAG.getNode(SystemZISD::RET_GLUE, DL, MVT::Other, RetOps);
2190 }
2191
2192 // Return true if Op is an intrinsic node with chain that returns the CC value
2193 // as its only (other) argument. Provide the associated SystemZISD opcode and
2194 // the mask of valid CC values if so.
isIntrinsicWithCCAndChain(SDValue Op,unsigned & Opcode,unsigned & CCValid)2195 static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2196 unsigned &CCValid) {
2197 unsigned Id = Op.getConstantOperandVal(1);
2198 switch (Id) {
2199 case Intrinsic::s390_tbegin:
2200 Opcode = SystemZISD::TBEGIN;
2201 CCValid = SystemZ::CCMASK_TBEGIN;
2202 return true;
2203
2204 case Intrinsic::s390_tbegin_nofloat:
2205 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2206 CCValid = SystemZ::CCMASK_TBEGIN;
2207 return true;
2208
2209 case Intrinsic::s390_tend:
2210 Opcode = SystemZISD::TEND;
2211 CCValid = SystemZ::CCMASK_TEND;
2212 return true;
2213
2214 default:
2215 return false;
2216 }
2217 }
2218
2219 // Return true if Op is an intrinsic node without chain that returns the
2220 // CC value as its final argument. Provide the associated SystemZISD
2221 // opcode and the mask of valid CC values if so.
isIntrinsicWithCC(SDValue Op,unsigned & Opcode,unsigned & CCValid)2222 static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2223 unsigned Id = Op.getConstantOperandVal(0);
2224 switch (Id) {
2225 case Intrinsic::s390_vpkshs:
2226 case Intrinsic::s390_vpksfs:
2227 case Intrinsic::s390_vpksgs:
2228 Opcode = SystemZISD::PACKS_CC;
2229 CCValid = SystemZ::CCMASK_VCMP;
2230 return true;
2231
2232 case Intrinsic::s390_vpklshs:
2233 case Intrinsic::s390_vpklsfs:
2234 case Intrinsic::s390_vpklsgs:
2235 Opcode = SystemZISD::PACKLS_CC;
2236 CCValid = SystemZ::CCMASK_VCMP;
2237 return true;
2238
2239 case Intrinsic::s390_vceqbs:
2240 case Intrinsic::s390_vceqhs:
2241 case Intrinsic::s390_vceqfs:
2242 case Intrinsic::s390_vceqgs:
2243 Opcode = SystemZISD::VICMPES;
2244 CCValid = SystemZ::CCMASK_VCMP;
2245 return true;
2246
2247 case Intrinsic::s390_vchbs:
2248 case Intrinsic::s390_vchhs:
2249 case Intrinsic::s390_vchfs:
2250 case Intrinsic::s390_vchgs:
2251 Opcode = SystemZISD::VICMPHS;
2252 CCValid = SystemZ::CCMASK_VCMP;
2253 return true;
2254
2255 case Intrinsic::s390_vchlbs:
2256 case Intrinsic::s390_vchlhs:
2257 case Intrinsic::s390_vchlfs:
2258 case Intrinsic::s390_vchlgs:
2259 Opcode = SystemZISD::VICMPHLS;
2260 CCValid = SystemZ::CCMASK_VCMP;
2261 return true;
2262
2263 case Intrinsic::s390_vtm:
2264 Opcode = SystemZISD::VTM;
2265 CCValid = SystemZ::CCMASK_VCMP;
2266 return true;
2267
2268 case Intrinsic::s390_vfaebs:
2269 case Intrinsic::s390_vfaehs:
2270 case Intrinsic::s390_vfaefs:
2271 Opcode = SystemZISD::VFAE_CC;
2272 CCValid = SystemZ::CCMASK_ANY;
2273 return true;
2274
2275 case Intrinsic::s390_vfaezbs:
2276 case Intrinsic::s390_vfaezhs:
2277 case Intrinsic::s390_vfaezfs:
2278 Opcode = SystemZISD::VFAEZ_CC;
2279 CCValid = SystemZ::CCMASK_ANY;
2280 return true;
2281
2282 case Intrinsic::s390_vfeebs:
2283 case Intrinsic::s390_vfeehs:
2284 case Intrinsic::s390_vfeefs:
2285 Opcode = SystemZISD::VFEE_CC;
2286 CCValid = SystemZ::CCMASK_ANY;
2287 return true;
2288
2289 case Intrinsic::s390_vfeezbs:
2290 case Intrinsic::s390_vfeezhs:
2291 case Intrinsic::s390_vfeezfs:
2292 Opcode = SystemZISD::VFEEZ_CC;
2293 CCValid = SystemZ::CCMASK_ANY;
2294 return true;
2295
2296 case Intrinsic::s390_vfenebs:
2297 case Intrinsic::s390_vfenehs:
2298 case Intrinsic::s390_vfenefs:
2299 Opcode = SystemZISD::VFENE_CC;
2300 CCValid = SystemZ::CCMASK_ANY;
2301 return true;
2302
2303 case Intrinsic::s390_vfenezbs:
2304 case Intrinsic::s390_vfenezhs:
2305 case Intrinsic::s390_vfenezfs:
2306 Opcode = SystemZISD::VFENEZ_CC;
2307 CCValid = SystemZ::CCMASK_ANY;
2308 return true;
2309
2310 case Intrinsic::s390_vistrbs:
2311 case Intrinsic::s390_vistrhs:
2312 case Intrinsic::s390_vistrfs:
2313 Opcode = SystemZISD::VISTR_CC;
2314 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2315 return true;
2316
2317 case Intrinsic::s390_vstrcbs:
2318 case Intrinsic::s390_vstrchs:
2319 case Intrinsic::s390_vstrcfs:
2320 Opcode = SystemZISD::VSTRC_CC;
2321 CCValid = SystemZ::CCMASK_ANY;
2322 return true;
2323
2324 case Intrinsic::s390_vstrczbs:
2325 case Intrinsic::s390_vstrczhs:
2326 case Intrinsic::s390_vstrczfs:
2327 Opcode = SystemZISD::VSTRCZ_CC;
2328 CCValid = SystemZ::CCMASK_ANY;
2329 return true;
2330
2331 case Intrinsic::s390_vstrsb:
2332 case Intrinsic::s390_vstrsh:
2333 case Intrinsic::s390_vstrsf:
2334 Opcode = SystemZISD::VSTRS_CC;
2335 CCValid = SystemZ::CCMASK_ANY;
2336 return true;
2337
2338 case Intrinsic::s390_vstrszb:
2339 case Intrinsic::s390_vstrszh:
2340 case Intrinsic::s390_vstrszf:
2341 Opcode = SystemZISD::VSTRSZ_CC;
2342 CCValid = SystemZ::CCMASK_ANY;
2343 return true;
2344
2345 case Intrinsic::s390_vfcedbs:
2346 case Intrinsic::s390_vfcesbs:
2347 Opcode = SystemZISD::VFCMPES;
2348 CCValid = SystemZ::CCMASK_VCMP;
2349 return true;
2350
2351 case Intrinsic::s390_vfchdbs:
2352 case Intrinsic::s390_vfchsbs:
2353 Opcode = SystemZISD::VFCMPHS;
2354 CCValid = SystemZ::CCMASK_VCMP;
2355 return true;
2356
2357 case Intrinsic::s390_vfchedbs:
2358 case Intrinsic::s390_vfchesbs:
2359 Opcode = SystemZISD::VFCMPHES;
2360 CCValid = SystemZ::CCMASK_VCMP;
2361 return true;
2362
2363 case Intrinsic::s390_vftcidb:
2364 case Intrinsic::s390_vftcisb:
2365 Opcode = SystemZISD::VFTCI;
2366 CCValid = SystemZ::CCMASK_VCMP;
2367 return true;
2368
2369 case Intrinsic::s390_tdc:
2370 Opcode = SystemZISD::TDC;
2371 CCValid = SystemZ::CCMASK_TDC;
2372 return true;
2373
2374 default:
2375 return false;
2376 }
2377 }
2378
2379 // Emit an intrinsic with chain and an explicit CC register result.
emitIntrinsicWithCCAndChain(SelectionDAG & DAG,SDValue Op,unsigned Opcode)2380 static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2381 unsigned Opcode) {
2382 // Copy all operands except the intrinsic ID.
2383 unsigned NumOps = Op.getNumOperands();
2384 SmallVector<SDValue, 6> Ops;
2385 Ops.reserve(NumOps - 1);
2386 Ops.push_back(Op.getOperand(0));
2387 for (unsigned I = 2; I < NumOps; ++I)
2388 Ops.push_back(Op.getOperand(I));
2389
2390 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2391 SDVTList RawVTs = DAG.getVTList(MVT::i32, MVT::Other);
2392 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
2393 SDValue OldChain = SDValue(Op.getNode(), 1);
2394 SDValue NewChain = SDValue(Intr.getNode(), 1);
2395 DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain);
2396 return Intr.getNode();
2397 }
2398
2399 // Emit an intrinsic with an explicit CC register result.
emitIntrinsicWithCC(SelectionDAG & DAG,SDValue Op,unsigned Opcode)2400 static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2401 unsigned Opcode) {
2402 // Copy all operands except the intrinsic ID.
2403 unsigned NumOps = Op.getNumOperands();
2404 SmallVector<SDValue, 6> Ops;
2405 Ops.reserve(NumOps - 1);
2406 for (unsigned I = 1; I < NumOps; ++I)
2407 Ops.push_back(Op.getOperand(I));
2408
2409 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);
2410 return Intr.getNode();
2411 }
2412
2413 // CC is a comparison that will be implemented using an integer or
2414 // floating-point comparison. Return the condition code mask for
2415 // a branch on true. In the integer case, CCMASK_CMP_UO is set for
2416 // unsigned comparisons and clear for signed ones. In the floating-point
2417 // case, CCMASK_CMP_UO has its normal mask meaning (unordered).
CCMaskForCondCode(ISD::CondCode CC)2418 static unsigned CCMaskForCondCode(ISD::CondCode CC) {
2419 #define CONV(X) \
2420 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2421 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2422 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2423
2424 switch (CC) {
2425 default:
2426 llvm_unreachable("Invalid integer condition!");
2427
2428 CONV(EQ);
2429 CONV(NE);
2430 CONV(GT);
2431 CONV(GE);
2432 CONV(LT);
2433 CONV(LE);
2434
2435 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
2436 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
2437 }
2438 #undef CONV
2439 }
2440
2441 // If C can be converted to a comparison against zero, adjust the operands
2442 // as necessary.
adjustZeroCmp(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2443 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2444 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2445 return;
2446
2447 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode());
2448 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
2449 return;
2450
2451 int64_t Value = ConstOp1->getSExtValue();
2452 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
2453 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
2454 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
2455 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
2456 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2457 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
2458 }
2459 }
2460
2461 // If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
2462 // adjust the operands as necessary.
adjustSubwordCmp(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2463 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2464 Comparison &C) {
2465 // For us to make any changes, it must a comparison between a single-use
2466 // load and a constant.
2467 if (!C.Op0.hasOneUse() ||
2468 C.Op0.getOpcode() != ISD::LOAD ||
2469 C.Op1.getOpcode() != ISD::Constant)
2470 return;
2471
2472 // We must have an 8- or 16-bit load.
2473 auto *Load = cast<LoadSDNode>(C.Op0);
2474 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2475 if ((NumBits != 8 && NumBits != 16) ||
2476 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2477 return;
2478
2479 // The load must be an extending one and the constant must be within the
2480 // range of the unextended value.
2481 auto *ConstOp1 = cast<ConstantSDNode>(C.Op1);
2482 if (!ConstOp1 || ConstOp1->getValueSizeInBits(0) > 64)
2483 return;
2484 uint64_t Value = ConstOp1->getZExtValue();
2485 uint64_t Mask = (1 << NumBits) - 1;
2486 if (Load->getExtensionType() == ISD::SEXTLOAD) {
2487 // Make sure that ConstOp1 is in range of C.Op0.
2488 int64_t SignedValue = ConstOp1->getSExtValue();
2489 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
2490 return;
2491 if (C.ICmpType != SystemZICMP::SignedOnly) {
2492 // Unsigned comparison between two sign-extended values is equivalent
2493 // to unsigned comparison between two zero-extended values.
2494 Value &= Mask;
2495 } else if (NumBits == 8) {
2496 // Try to treat the comparison as unsigned, so that we can use CLI.
2497 // Adjust CCMask and Value as necessary.
2498 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
2499 // Test whether the high bit of the byte is set.
2500 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
2501 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
2502 // Test whether the high bit of the byte is clear.
2503 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
2504 else
2505 // No instruction exists for this combination.
2506 return;
2507 C.ICmpType = SystemZICMP::UnsignedOnly;
2508 }
2509 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
2510 if (Value > Mask)
2511 return;
2512 // If the constant is in range, we can use any comparison.
2513 C.ICmpType = SystemZICMP::Any;
2514 } else
2515 return;
2516
2517 // Make sure that the first operand is an i32 of the right extension type.
2518 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
2519 ISD::SEXTLOAD :
2520 ISD::ZEXTLOAD);
2521 if (C.Op0.getValueType() != MVT::i32 ||
2522 Load->getExtensionType() != ExtType) {
2523 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),
2524 Load->getBasePtr(), Load->getPointerInfo(),
2525 Load->getMemoryVT(), Load->getAlign(),
2526 Load->getMemOperand()->getFlags());
2527 // Update the chain uses.
2528 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1));
2529 }
2530
2531 // Make sure that the second operand is an i32 with the right value.
2532 if (C.Op1.getValueType() != MVT::i32 ||
2533 Value != ConstOp1->getZExtValue())
2534 C.Op1 = DAG.getConstant(Value, DL, MVT::i32);
2535 }
2536
2537 // Return true if Op is either an unextended load, or a load suitable
2538 // for integer register-memory comparisons of type ICmpType.
isNaturalMemoryOperand(SDValue Op,unsigned ICmpType)2539 static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
2540 auto *Load = dyn_cast<LoadSDNode>(Op.getNode());
2541 if (Load) {
2542 // There are no instructions to compare a register with a memory byte.
2543 if (Load->getMemoryVT() == MVT::i8)
2544 return false;
2545 // Otherwise decide on extension type.
2546 switch (Load->getExtensionType()) {
2547 case ISD::NON_EXTLOAD:
2548 return true;
2549 case ISD::SEXTLOAD:
2550 return ICmpType != SystemZICMP::UnsignedOnly;
2551 case ISD::ZEXTLOAD:
2552 return ICmpType != SystemZICMP::SignedOnly;
2553 default:
2554 break;
2555 }
2556 }
2557 return false;
2558 }
2559
2560 // Return true if it is better to swap the operands of C.
shouldSwapCmpOperands(const Comparison & C)2561 static bool shouldSwapCmpOperands(const Comparison &C) {
2562 // Leave i128 and f128 comparisons alone, since they have no memory forms.
2563 if (C.Op0.getValueType() == MVT::i128)
2564 return false;
2565 if (C.Op0.getValueType() == MVT::f128)
2566 return false;
2567
2568 // Always keep a floating-point constant second, since comparisons with
2569 // zero can use LOAD TEST and comparisons with other constants make a
2570 // natural memory operand.
2571 if (isa<ConstantFPSDNode>(C.Op1))
2572 return false;
2573
2574 // Never swap comparisons with zero since there are many ways to optimize
2575 // those later.
2576 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
2577 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
2578 return false;
2579
2580 // Also keep natural memory operands second if the loaded value is
2581 // only used here. Several comparisons have memory forms.
2582 if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse())
2583 return false;
2584
2585 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
2586 // In that case we generally prefer the memory to be second.
2587 if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) {
2588 // The only exceptions are when the second operand is a constant and
2589 // we can use things like CHHSI.
2590 if (!ConstOp1)
2591 return true;
2592 // The unsigned memory-immediate instructions can handle 16-bit
2593 // unsigned integers.
2594 if (C.ICmpType != SystemZICMP::SignedOnly &&
2595 isUInt<16>(ConstOp1->getZExtValue()))
2596 return false;
2597 // The signed memory-immediate instructions can handle 16-bit
2598 // signed integers.
2599 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
2600 isInt<16>(ConstOp1->getSExtValue()))
2601 return false;
2602 return true;
2603 }
2604
2605 // Try to promote the use of CGFR and CLGFR.
2606 unsigned Opcode0 = C.Op0.getOpcode();
2607 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
2608 return true;
2609 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
2610 return true;
2611 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
2612 C.Op0.getOperand(1).getOpcode() == ISD::Constant &&
2613 C.Op0.getConstantOperandVal(1) == 0xffffffff)
2614 return true;
2615
2616 return false;
2617 }
2618
2619 // Check whether C tests for equality between X and Y and whether X - Y
2620 // or Y - X is also computed. In that case it's better to compare the
2621 // result of the subtraction against zero.
adjustForSubtraction(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2622 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
2623 Comparison &C) {
2624 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2625 C.CCMask == SystemZ::CCMASK_CMP_NE) {
2626 for (SDNode *N : C.Op0->uses()) {
2627 if (N->getOpcode() == ISD::SUB &&
2628 ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||
2629 (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
2630 // Disable the nsw and nuw flags: the backend needs to handle
2631 // overflow as well during comparison elimination.
2632 SDNodeFlags Flags = N->getFlags();
2633 Flags.setNoSignedWrap(false);
2634 Flags.setNoUnsignedWrap(false);
2635 N->setFlags(Flags);
2636 C.Op0 = SDValue(N, 0);
2637 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
2638 return;
2639 }
2640 }
2641 }
2642 }
2643
2644 // Check whether C compares a floating-point value with zero and if that
2645 // floating-point value is also negated. In this case we can use the
2646 // negation to set CC, so avoiding separate LOAD AND TEST and
2647 // LOAD (NEGATIVE/COMPLEMENT) instructions.
adjustForFNeg(Comparison & C)2648 static void adjustForFNeg(Comparison &C) {
2649 // This optimization is invalid for strict comparisons, since FNEG
2650 // does not raise any exceptions.
2651 if (C.Chain)
2652 return;
2653 auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1);
2654 if (C1 && C1->isZero()) {
2655 for (SDNode *N : C.Op0->uses()) {
2656 if (N->getOpcode() == ISD::FNEG) {
2657 C.Op0 = SDValue(N, 0);
2658 C.CCMask = SystemZ::reverseCCMask(C.CCMask);
2659 return;
2660 }
2661 }
2662 }
2663 }
2664
2665 // Check whether C compares (shl X, 32) with 0 and whether X is
2666 // also sign-extended. In that case it is better to test the result
2667 // of the sign extension using LTGFR.
2668 //
2669 // This case is important because InstCombine transforms a comparison
2670 // with (sext (trunc X)) into a comparison with (shl X, 32).
adjustForLTGFR(Comparison & C)2671 static void adjustForLTGFR(Comparison &C) {
2672 // Check for a comparison between (shl X, 32) and 0.
2673 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
2674 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
2675 auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
2676 if (C1 && C1->getZExtValue() == 32) {
2677 SDValue ShlOp0 = C.Op0.getOperand(0);
2678 // See whether X has any SIGN_EXTEND_INREG uses.
2679 for (SDNode *N : ShlOp0->uses()) {
2680 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
2681 cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) {
2682 C.Op0 = SDValue(N, 0);
2683 return;
2684 }
2685 }
2686 }
2687 }
2688 }
2689
2690 // If C compares the truncation of an extending load, try to compare
2691 // the untruncated value instead. This exposes more opportunities to
2692 // reuse CC.
adjustICmpTruncate(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2693 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
2694 Comparison &C) {
2695 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
2696 C.Op0.getOperand(0).getOpcode() == ISD::LOAD &&
2697 C.Op1.getOpcode() == ISD::Constant &&
2698 cast<ConstantSDNode>(C.Op1)->getValueSizeInBits(0) <= 64 &&
2699 C.Op1->getAsZExtVal() == 0) {
2700 auto *L = cast<LoadSDNode>(C.Op0.getOperand(0));
2701 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
2702 C.Op0.getValueSizeInBits().getFixedValue()) {
2703 unsigned Type = L->getExtensionType();
2704 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
2705 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
2706 C.Op0 = C.Op0.getOperand(0);
2707 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
2708 }
2709 }
2710 }
2711 }
2712
2713 // Return true if shift operation N has an in-range constant shift value.
2714 // Store it in ShiftVal if so.
isSimpleShift(SDValue N,unsigned & ShiftVal)2715 static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
2716 auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1));
2717 if (!Shift)
2718 return false;
2719
2720 uint64_t Amount = Shift->getZExtValue();
2721 if (Amount >= N.getValueSizeInBits())
2722 return false;
2723
2724 ShiftVal = Amount;
2725 return true;
2726 }
2727
2728 // Check whether an AND with Mask is suitable for a TEST UNDER MASK
2729 // instruction and whether the CC value is descriptive enough to handle
2730 // a comparison of type Opcode between the AND result and CmpVal.
2731 // CCMask says which comparison result is being tested and BitSize is
2732 // the number of bits in the operands. If TEST UNDER MASK can be used,
2733 // return the corresponding CC mask, otherwise return 0.
getTestUnderMaskCond(unsigned BitSize,unsigned CCMask,uint64_t Mask,uint64_t CmpVal,unsigned ICmpType)2734 static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
2735 uint64_t Mask, uint64_t CmpVal,
2736 unsigned ICmpType) {
2737 assert(Mask != 0 && "ANDs with zero should have been removed by now");
2738
2739 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
2740 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) &&
2741 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask))
2742 return 0;
2743
2744 // Work out the masks for the lowest and highest bits.
2745 uint64_t High = llvm::bit_floor(Mask);
2746 uint64_t Low = uint64_t(1) << llvm::countr_zero(Mask);
2747
2748 // Signed ordered comparisons are effectively unsigned if the sign
2749 // bit is dropped.
2750 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
2751
2752 // Check for equality comparisons with 0, or the equivalent.
2753 if (CmpVal == 0) {
2754 if (CCMask == SystemZ::CCMASK_CMP_EQ)
2755 return SystemZ::CCMASK_TM_ALL_0;
2756 if (CCMask == SystemZ::CCMASK_CMP_NE)
2757 return SystemZ::CCMASK_TM_SOME_1;
2758 }
2759 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
2760 if (CCMask == SystemZ::CCMASK_CMP_LT)
2761 return SystemZ::CCMASK_TM_ALL_0;
2762 if (CCMask == SystemZ::CCMASK_CMP_GE)
2763 return SystemZ::CCMASK_TM_SOME_1;
2764 }
2765 if (EffectivelyUnsigned && CmpVal < Low) {
2766 if (CCMask == SystemZ::CCMASK_CMP_LE)
2767 return SystemZ::CCMASK_TM_ALL_0;
2768 if (CCMask == SystemZ::CCMASK_CMP_GT)
2769 return SystemZ::CCMASK_TM_SOME_1;
2770 }
2771
2772 // Check for equality comparisons with the mask, or the equivalent.
2773 if (CmpVal == Mask) {
2774 if (CCMask == SystemZ::CCMASK_CMP_EQ)
2775 return SystemZ::CCMASK_TM_ALL_1;
2776 if (CCMask == SystemZ::CCMASK_CMP_NE)
2777 return SystemZ::CCMASK_TM_SOME_0;
2778 }
2779 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
2780 if (CCMask == SystemZ::CCMASK_CMP_GT)
2781 return SystemZ::CCMASK_TM_ALL_1;
2782 if (CCMask == SystemZ::CCMASK_CMP_LE)
2783 return SystemZ::CCMASK_TM_SOME_0;
2784 }
2785 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
2786 if (CCMask == SystemZ::CCMASK_CMP_GE)
2787 return SystemZ::CCMASK_TM_ALL_1;
2788 if (CCMask == SystemZ::CCMASK_CMP_LT)
2789 return SystemZ::CCMASK_TM_SOME_0;
2790 }
2791
2792 // Check for ordered comparisons with the top bit.
2793 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
2794 if (CCMask == SystemZ::CCMASK_CMP_LE)
2795 return SystemZ::CCMASK_TM_MSB_0;
2796 if (CCMask == SystemZ::CCMASK_CMP_GT)
2797 return SystemZ::CCMASK_TM_MSB_1;
2798 }
2799 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
2800 if (CCMask == SystemZ::CCMASK_CMP_LT)
2801 return SystemZ::CCMASK_TM_MSB_0;
2802 if (CCMask == SystemZ::CCMASK_CMP_GE)
2803 return SystemZ::CCMASK_TM_MSB_1;
2804 }
2805
2806 // If there are just two bits, we can do equality checks for Low and High
2807 // as well.
2808 if (Mask == Low + High) {
2809 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
2810 return SystemZ::CCMASK_TM_MIXED_MSB_0;
2811 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
2812 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
2813 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
2814 return SystemZ::CCMASK_TM_MIXED_MSB_1;
2815 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
2816 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
2817 }
2818
2819 // Looks like we've exhausted our options.
2820 return 0;
2821 }
2822
2823 // See whether C can be implemented as a TEST UNDER MASK instruction.
2824 // Update the arguments with the TM version if so.
adjustForTestUnderMask(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2825 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
2826 Comparison &C) {
2827 // Use VECTOR TEST UNDER MASK for i128 operations.
2828 if (C.Op0.getValueType() == MVT::i128) {
2829 // We can use VTM for EQ/NE comparisons of x & y against 0.
2830 if (C.Op0.getOpcode() == ISD::AND &&
2831 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2832 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
2833 auto *Mask = dyn_cast<ConstantSDNode>(C.Op1);
2834 if (Mask && Mask->getAPIntValue() == 0) {
2835 C.Opcode = SystemZISD::VTM;
2836 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(1));
2837 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, C.Op0.getOperand(0));
2838 C.CCValid = SystemZ::CCMASK_VCMP;
2839 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
2840 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
2841 else
2842 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
2843 }
2844 }
2845 return;
2846 }
2847
2848 // Check that we have a comparison with a constant.
2849 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
2850 if (!ConstOp1)
2851 return;
2852 uint64_t CmpVal = ConstOp1->getZExtValue();
2853
2854 // Check whether the nonconstant input is an AND with a constant mask.
2855 Comparison NewC(C);
2856 uint64_t MaskVal;
2857 ConstantSDNode *Mask = nullptr;
2858 if (C.Op0.getOpcode() == ISD::AND) {
2859 NewC.Op0 = C.Op0.getOperand(0);
2860 NewC.Op1 = C.Op0.getOperand(1);
2861 Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
2862 if (!Mask)
2863 return;
2864 MaskVal = Mask->getZExtValue();
2865 } else {
2866 // There is no instruction to compare with a 64-bit immediate
2867 // so use TMHH instead if possible. We need an unsigned ordered
2868 // comparison with an i64 immediate.
2869 if (NewC.Op0.getValueType() != MVT::i64 ||
2870 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
2871 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
2872 NewC.ICmpType == SystemZICMP::SignedOnly)
2873 return;
2874 // Convert LE and GT comparisons into LT and GE.
2875 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
2876 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
2877 if (CmpVal == uint64_t(-1))
2878 return;
2879 CmpVal += 1;
2880 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2881 }
2882 // If the low N bits of Op1 are zero than the low N bits of Op0 can
2883 // be masked off without changing the result.
2884 MaskVal = -(CmpVal & -CmpVal);
2885 NewC.ICmpType = SystemZICMP::UnsignedOnly;
2886 }
2887 if (!MaskVal)
2888 return;
2889
2890 // Check whether the combination of mask, comparison value and comparison
2891 // type are suitable.
2892 unsigned BitSize = NewC.Op0.getValueSizeInBits();
2893 unsigned NewCCMask, ShiftVal;
2894 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
2895 NewC.Op0.getOpcode() == ISD::SHL &&
2896 isSimpleShift(NewC.Op0, ShiftVal) &&
2897 (MaskVal >> ShiftVal != 0) &&
2898 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
2899 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
2900 MaskVal >> ShiftVal,
2901 CmpVal >> ShiftVal,
2902 SystemZICMP::Any))) {
2903 NewC.Op0 = NewC.Op0.getOperand(0);
2904 MaskVal >>= ShiftVal;
2905 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
2906 NewC.Op0.getOpcode() == ISD::SRL &&
2907 isSimpleShift(NewC.Op0, ShiftVal) &&
2908 (MaskVal << ShiftVal != 0) &&
2909 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
2910 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
2911 MaskVal << ShiftVal,
2912 CmpVal << ShiftVal,
2913 SystemZICMP::UnsignedOnly))) {
2914 NewC.Op0 = NewC.Op0.getOperand(0);
2915 MaskVal <<= ShiftVal;
2916 } else {
2917 NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal,
2918 NewC.ICmpType);
2919 if (!NewCCMask)
2920 return;
2921 }
2922
2923 // Go ahead and make the change.
2924 C.Opcode = SystemZISD::TM;
2925 C.Op0 = NewC.Op0;
2926 if (Mask && Mask->getZExtValue() == MaskVal)
2927 C.Op1 = SDValue(Mask, 0);
2928 else
2929 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
2930 C.CCValid = SystemZ::CCMASK_TM;
2931 C.CCMask = NewCCMask;
2932 }
2933
2934 // Implement i128 comparison in vector registers.
adjustICmp128(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2935 static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
2936 Comparison &C) {
2937 if (C.Opcode != SystemZISD::ICMP)
2938 return;
2939 if (C.Op0.getValueType() != MVT::i128)
2940 return;
2941
2942 // (In-)Equality comparisons can be implemented via VCEQGS.
2943 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2944 C.CCMask == SystemZ::CCMASK_CMP_NE) {
2945 C.Opcode = SystemZISD::VICMPES;
2946 C.Op0 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op0);
2947 C.Op1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, C.Op1);
2948 C.CCValid = SystemZ::CCMASK_VCMP;
2949 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
2950 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
2951 else
2952 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
2953 return;
2954 }
2955
2956 // Normalize other comparisons to GT.
2957 bool Swap = false, Invert = false;
2958 switch (C.CCMask) {
2959 case SystemZ::CCMASK_CMP_GT: break;
2960 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
2961 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
2962 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
2963 default: llvm_unreachable("Invalid integer condition!");
2964 }
2965 if (Swap)
2966 std::swap(C.Op0, C.Op1);
2967
2968 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2969 C.Opcode = SystemZISD::UCMP128HI;
2970 else
2971 C.Opcode = SystemZISD::SCMP128HI;
2972 C.CCValid = SystemZ::CCMASK_ANY;
2973 C.CCMask = SystemZ::CCMASK_1;
2974
2975 if (Invert)
2976 C.CCMask ^= C.CCValid;
2977 }
2978
2979 // See whether the comparison argument contains a redundant AND
2980 // and remove it if so. This sometimes happens due to the generic
2981 // BRCOND expansion.
adjustForRedundantAnd(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)2982 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
2983 Comparison &C) {
2984 if (C.Op0.getOpcode() != ISD::AND)
2985 return;
2986 auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
2987 if (!Mask || Mask->getValueSizeInBits(0) > 64)
2988 return;
2989 KnownBits Known = DAG.computeKnownBits(C.Op0.getOperand(0));
2990 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
2991 return;
2992
2993 C.Op0 = C.Op0.getOperand(0);
2994 }
2995
2996 // Return a Comparison that tests the condition-code result of intrinsic
2997 // node Call against constant integer CC using comparison code Cond.
2998 // Opcode is the opcode of the SystemZISD operation for the intrinsic
2999 // and CCValid is the set of possible condition-code results.
getIntrinsicCmp(SelectionDAG & DAG,unsigned Opcode,SDValue Call,unsigned CCValid,uint64_t CC,ISD::CondCode Cond)3000 static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3001 SDValue Call, unsigned CCValid, uint64_t CC,
3002 ISD::CondCode Cond) {
3003 Comparison C(Call, SDValue(), SDValue());
3004 C.Opcode = Opcode;
3005 C.CCValid = CCValid;
3006 if (Cond == ISD::SETEQ)
3007 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3008 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3009 else if (Cond == ISD::SETNE)
3010 // ...and the inverse of that.
3011 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3012 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3013 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3014 // always true for CC>3.
3015 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3016 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3017 // ...and the inverse of that.
3018 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3019 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3020 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3021 // always true for CC>3.
3022 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3023 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3024 // ...and the inverse of that.
3025 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3026 else
3027 llvm_unreachable("Unexpected integer comparison type");
3028 C.CCMask &= CCValid;
3029 return C;
3030 }
3031
3032 // Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
getCmp(SelectionDAG & DAG,SDValue CmpOp0,SDValue CmpOp1,ISD::CondCode Cond,const SDLoc & DL,SDValue Chain=SDValue (),bool IsSignaling=false)3033 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3034 ISD::CondCode Cond, const SDLoc &DL,
3035 SDValue Chain = SDValue(),
3036 bool IsSignaling = false) {
3037 if (CmpOp1.getOpcode() == ISD::Constant) {
3038 assert(!Chain);
3039 unsigned Opcode, CCValid;
3040 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3041 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&
3042 isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid))
3043 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid,
3044 CmpOp1->getAsZExtVal(), Cond);
3045 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3046 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3047 isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
3048 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid,
3049 CmpOp1->getAsZExtVal(), Cond);
3050 }
3051 Comparison C(CmpOp0, CmpOp1, Chain);
3052 C.CCMask = CCMaskForCondCode(Cond);
3053 if (C.Op0.getValueType().isFloatingPoint()) {
3054 C.CCValid = SystemZ::CCMASK_FCMP;
3055 if (!C.Chain)
3056 C.Opcode = SystemZISD::FCMP;
3057 else if (!IsSignaling)
3058 C.Opcode = SystemZISD::STRICT_FCMP;
3059 else
3060 C.Opcode = SystemZISD::STRICT_FCMPS;
3061 adjustForFNeg(C);
3062 } else {
3063 assert(!C.Chain);
3064 C.CCValid = SystemZ::CCMASK_ICMP;
3065 C.Opcode = SystemZISD::ICMP;
3066 // Choose the type of comparison. Equality and inequality tests can
3067 // use either signed or unsigned comparisons. The choice also doesn't
3068 // matter if both sign bits are known to be clear. In those cases we
3069 // want to give the main isel code the freedom to choose whichever
3070 // form fits best.
3071 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3072 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3073 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1)))
3074 C.ICmpType = SystemZICMP::Any;
3075 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3076 C.ICmpType = SystemZICMP::UnsignedOnly;
3077 else
3078 C.ICmpType = SystemZICMP::SignedOnly;
3079 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3080 adjustForRedundantAnd(DAG, DL, C);
3081 adjustZeroCmp(DAG, DL, C);
3082 adjustSubwordCmp(DAG, DL, C);
3083 adjustForSubtraction(DAG, DL, C);
3084 adjustForLTGFR(C);
3085 adjustICmpTruncate(DAG, DL, C);
3086 }
3087
3088 if (shouldSwapCmpOperands(C)) {
3089 std::swap(C.Op0, C.Op1);
3090 C.CCMask = SystemZ::reverseCCMask(C.CCMask);
3091 }
3092
3093 adjustForTestUnderMask(DAG, DL, C);
3094 adjustICmp128(DAG, DL, C);
3095 return C;
3096 }
3097
3098 // Emit the comparison instruction described by C.
emitCmp(SelectionDAG & DAG,const SDLoc & DL,Comparison & C)3099 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3100 if (!C.Op1.getNode()) {
3101 SDNode *Node;
3102 switch (C.Op0.getOpcode()) {
3103 case ISD::INTRINSIC_W_CHAIN:
3104 Node = emitIntrinsicWithCCAndChain(DAG, C.Op0, C.Opcode);
3105 return SDValue(Node, 0);
3106 case ISD::INTRINSIC_WO_CHAIN:
3107 Node = emitIntrinsicWithCC(DAG, C.Op0, C.Opcode);
3108 return SDValue(Node, Node->getNumValues() - 1);
3109 default:
3110 llvm_unreachable("Invalid comparison operands");
3111 }
3112 }
3113 if (C.Opcode == SystemZISD::ICMP)
3114 return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
3115 DAG.getTargetConstant(C.ICmpType, DL, MVT::i32));
3116 if (C.Opcode == SystemZISD::TM) {
3117 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3118 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3119 return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
3120 DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
3121 }
3122 if (C.Opcode == SystemZISD::VICMPES) {
3123 SDVTList VTs = DAG.getVTList(C.Op0.getValueType(), MVT::i32);
3124 SDValue Val = DAG.getNode(C.Opcode, DL, VTs, C.Op0, C.Op1);
3125 return SDValue(Val.getNode(), 1);
3126 }
3127 if (C.Chain) {
3128 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
3129 return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1);
3130 }
3131 return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
3132 }
3133
3134 // Implement a 32-bit *MUL_LOHI operation by extending both operands to
3135 // 64 bits. Extend is the extension type to use. Store the high part
3136 // in Hi and the low part in Lo.
lowerMUL_LOHI32(SelectionDAG & DAG,const SDLoc & DL,unsigned Extend,SDValue Op0,SDValue Op1,SDValue & Hi,SDValue & Lo)3137 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3138 SDValue Op0, SDValue Op1, SDValue &Hi,
3139 SDValue &Lo) {
3140 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
3141 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
3142 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
3143 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
3144 DAG.getConstant(32, DL, MVT::i64));
3145 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
3146 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
3147 }
3148
3149 // Lower a binary operation that produces two VT results, one in each
3150 // half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3151 // and Opcode performs the GR128 operation. Store the even register result
3152 // in Even and the odd register result in Odd.
lowerGR128Binary(SelectionDAG & DAG,const SDLoc & DL,EVT VT,unsigned Opcode,SDValue Op0,SDValue Op1,SDValue & Even,SDValue & Odd)3153 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3154 unsigned Opcode, SDValue Op0, SDValue Op1,
3155 SDValue &Even, SDValue &Odd) {
3156 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
3157 bool Is32Bit = is32Bit(VT);
3158 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
3159 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
3160 }
3161
3162 // Return an i32 value that is 1 if the CC value produced by CCReg is
3163 // in the mask CCMask and 0 otherwise. CC is known to have a value
3164 // in CCValid, so other values can be ignored.
emitSETCC(SelectionDAG & DAG,const SDLoc & DL,SDValue CCReg,unsigned CCValid,unsigned CCMask)3165 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3166 unsigned CCValid, unsigned CCMask) {
3167 SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
3168 DAG.getConstant(0, DL, MVT::i32),
3169 DAG.getTargetConstant(CCValid, DL, MVT::i32),
3170 DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg};
3171 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
3172 }
3173
3174 // Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3175 // be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3176 // for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3177 // floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3178 // floating-point comparisons.
3179 enum class CmpMode { Int, FP, StrictFP, SignalingFP };
getVectorComparison(ISD::CondCode CC,CmpMode Mode)3180 static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3181 switch (CC) {
3182 case ISD::SETOEQ:
3183 case ISD::SETEQ:
3184 switch (Mode) {
3185 case CmpMode::Int: return SystemZISD::VICMPE;
3186 case CmpMode::FP: return SystemZISD::VFCMPE;
3187 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3188 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3189 }
3190 llvm_unreachable("Bad mode");
3191
3192 case ISD::SETOGE:
3193 case ISD::SETGE:
3194 switch (Mode) {
3195 case CmpMode::Int: return 0;
3196 case CmpMode::FP: return SystemZISD::VFCMPHE;
3197 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3198 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3199 }
3200 llvm_unreachable("Bad mode");
3201
3202 case ISD::SETOGT:
3203 case ISD::SETGT:
3204 switch (Mode) {
3205 case CmpMode::Int: return SystemZISD::VICMPH;
3206 case CmpMode::FP: return SystemZISD::VFCMPH;
3207 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3208 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3209 }
3210 llvm_unreachable("Bad mode");
3211
3212 case ISD::SETUGT:
3213 switch (Mode) {
3214 case CmpMode::Int: return SystemZISD::VICMPHL;
3215 case CmpMode::FP: return 0;
3216 case CmpMode::StrictFP: return 0;
3217 case CmpMode::SignalingFP: return 0;
3218 }
3219 llvm_unreachable("Bad mode");
3220
3221 default:
3222 return 0;
3223 }
3224 }
3225
3226 // Return the SystemZISD vector comparison operation for CC or its inverse,
3227 // or 0 if neither can be done directly. Indicate in Invert whether the
3228 // result is for the inverse of CC. Mode is as above.
getVectorComparisonOrInvert(ISD::CondCode CC,CmpMode Mode,bool & Invert)3229 static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3230 bool &Invert) {
3231 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3232 Invert = false;
3233 return Opcode;
3234 }
3235
3236 CC = ISD::getSetCCInverse(CC, Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3237 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3238 Invert = true;
3239 return Opcode;
3240 }
3241
3242 return 0;
3243 }
3244
3245 // Return a v2f64 that contains the extended form of elements Start and Start+1
3246 // of v4f32 value Op. If Chain is nonnull, return the strict form.
expandV4F32ToV2F64(SelectionDAG & DAG,int Start,const SDLoc & DL,SDValue Op,SDValue Chain)3247 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3248 SDValue Op, SDValue Chain) {
3249 int Mask[] = { Start, -1, Start + 1, -1 };
3250 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
3251 if (Chain) {
3252 SDVTList VTs = DAG.getVTList(MVT::v2f64, MVT::Other);
3253 return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op);
3254 }
3255 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
3256 }
3257
3258 // Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3259 // producing a result of type VT. If Chain is nonnull, return the strict form.
getVectorCmp(SelectionDAG & DAG,unsigned Opcode,const SDLoc & DL,EVT VT,SDValue CmpOp0,SDValue CmpOp1,SDValue Chain) const3260 SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3261 const SDLoc &DL, EVT VT,
3262 SDValue CmpOp0,
3263 SDValue CmpOp1,
3264 SDValue Chain) const {
3265 // There is no hardware support for v4f32 (unless we have the vector
3266 // enhancements facility 1), so extend the vector into two v2f64s
3267 // and compare those.
3268 if (CmpOp0.getValueType() == MVT::v4f32 &&
3269 !Subtarget.hasVectorEnhancements1()) {
3270 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain);
3271 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain);
3272 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain);
3273 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain);
3274 if (Chain) {
3275 SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::Other);
3276 SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1);
3277 SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1);
3278 SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
3279 SDValue Chains[6] = { H0.getValue(1), L0.getValue(1),
3280 H1.getValue(1), L1.getValue(1),
3281 HRes.getValue(1), LRes.getValue(1) };
3282 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
3283 SDValue Ops[2] = { Res, NewChain };
3284 return DAG.getMergeValues(Ops, DL);
3285 }
3286 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
3287 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
3288 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
3289 }
3290 if (Chain) {
3291 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
3292 return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1);
3293 }
3294 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
3295 }
3296
3297 // Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3298 // an integer mask of type VT. If Chain is nonnull, we have a strict
3299 // floating-point comparison. If in addition IsSignaling is true, we have
3300 // a strict signaling floating-point comparison.
lowerVectorSETCC(SelectionDAG & DAG,const SDLoc & DL,EVT VT,ISD::CondCode CC,SDValue CmpOp0,SDValue CmpOp1,SDValue Chain,bool IsSignaling) const3301 SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3302 const SDLoc &DL, EVT VT,
3303 ISD::CondCode CC,
3304 SDValue CmpOp0,
3305 SDValue CmpOp1,
3306 SDValue Chain,
3307 bool IsSignaling) const {
3308 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3309 assert (!Chain || IsFP);
3310 assert (!IsSignaling || Chain);
3311 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3312 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3313 bool Invert = false;
3314 SDValue Cmp;
3315 switch (CC) {
3316 // Handle tests for order using (or (ogt y x) (oge x y)).
3317 case ISD::SETUO:
3318 Invert = true;
3319 [[fallthrough]];
3320 case ISD::SETO: {
3321 assert(IsFP && "Unexpected integer comparison");
3322 SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3323 DL, VT, CmpOp1, CmpOp0, Chain);
3324 SDValue GE = getVectorCmp(DAG, getVectorComparison(ISD::SETOGE, Mode),
3325 DL, VT, CmpOp0, CmpOp1, Chain);
3326 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
3327 if (Chain)
3328 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3329 LT.getValue(1), GE.getValue(1));
3330 break;
3331 }
3332
3333 // Handle <> tests using (or (ogt y x) (ogt x y)).
3334 case ISD::SETUEQ:
3335 Invert = true;
3336 [[fallthrough]];
3337 case ISD::SETONE: {
3338 assert(IsFP && "Unexpected integer comparison");
3339 SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3340 DL, VT, CmpOp1, CmpOp0, Chain);
3341 SDValue GT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
3342 DL, VT, CmpOp0, CmpOp1, Chain);
3343 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
3344 if (Chain)
3345 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
3346 LT.getValue(1), GT.getValue(1));
3347 break;
3348 }
3349
3350 // Otherwise a single comparison is enough. It doesn't really
3351 // matter whether we try the inversion or the swap first, since
3352 // there are no cases where both work.
3353 default:
3354 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3355 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3356 else {
3357 CC = ISD::getSetCCSwappedOperands(CC);
3358 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3359 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain);
3360 else
3361 llvm_unreachable("Unhandled comparison");
3362 }
3363 if (Chain)
3364 Chain = Cmp.getValue(1);
3365 break;
3366 }
3367 if (Invert) {
3368 SDValue Mask =
3369 DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));
3370 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
3371 }
3372 if (Chain && Chain.getNode() != Cmp.getNode()) {
3373 SDValue Ops[2] = { Cmp, Chain };
3374 Cmp = DAG.getMergeValues(Ops, DL);
3375 }
3376 return Cmp;
3377 }
3378
lowerSETCC(SDValue Op,SelectionDAG & DAG) const3379 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
3380 SelectionDAG &DAG) const {
3381 SDValue CmpOp0 = Op.getOperand(0);
3382 SDValue CmpOp1 = Op.getOperand(1);
3383 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
3384 SDLoc DL(Op);
3385 EVT VT = Op.getValueType();
3386 if (VT.isVector())
3387 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3388
3389 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
3390 SDValue CCReg = emitCmp(DAG, DL, C);
3391 return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
3392 }
3393
lowerSTRICT_FSETCC(SDValue Op,SelectionDAG & DAG,bool IsSignaling) const3394 SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
3395 SelectionDAG &DAG,
3396 bool IsSignaling) const {
3397 SDValue Chain = Op.getOperand(0);
3398 SDValue CmpOp0 = Op.getOperand(1);
3399 SDValue CmpOp1 = Op.getOperand(2);
3400 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get();
3401 SDLoc DL(Op);
3402 EVT VT = Op.getNode()->getValueType(0);
3403 if (VT.isVector()) {
3404 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
3405 Chain, IsSignaling);
3406 return Res.getValue(Op.getResNo());
3407 }
3408
3409 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling));
3410 SDValue CCReg = emitCmp(DAG, DL, C);
3411 CCReg->setFlags(Op->getFlags());
3412 SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
3413 SDValue Ops[2] = { Result, CCReg.getValue(1) };
3414 return DAG.getMergeValues(Ops, DL);
3415 }
3416
lowerBR_CC(SDValue Op,SelectionDAG & DAG) const3417 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3418 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
3419 SDValue CmpOp0 = Op.getOperand(2);
3420 SDValue CmpOp1 = Op.getOperand(3);
3421 SDValue Dest = Op.getOperand(4);
3422 SDLoc DL(Op);
3423
3424 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
3425 SDValue CCReg = emitCmp(DAG, DL, C);
3426 return DAG.getNode(
3427 SystemZISD::BR_CCMASK, DL, Op.getValueType(), Op.getOperand(0),
3428 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
3429 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
3430 }
3431
3432 // Return true if Pos is CmpOp and Neg is the negative of CmpOp,
3433 // allowing Pos and Neg to be wider than CmpOp.
isAbsolute(SDValue CmpOp,SDValue Pos,SDValue Neg)3434 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
3435 return (Neg.getOpcode() == ISD::SUB &&
3436 Neg.getOperand(0).getOpcode() == ISD::Constant &&
3437 Neg.getConstantOperandVal(0) == 0 && Neg.getOperand(1) == Pos &&
3438 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
3439 Pos.getOperand(0) == CmpOp)));
3440 }
3441
3442 // Return the absolute or negative absolute of Op; IsNegative decides which.
getAbsolute(SelectionDAG & DAG,const SDLoc & DL,SDValue Op,bool IsNegative)3443 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
3444 bool IsNegative) {
3445 Op = DAG.getNode(ISD::ABS, DL, Op.getValueType(), Op);
3446 if (IsNegative)
3447 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
3448 DAG.getConstant(0, DL, Op.getValueType()), Op);
3449 return Op;
3450 }
3451
lowerSELECT_CC(SDValue Op,SelectionDAG & DAG) const3452 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
3453 SelectionDAG &DAG) const {
3454 SDValue CmpOp0 = Op.getOperand(0);
3455 SDValue CmpOp1 = Op.getOperand(1);
3456 SDValue TrueOp = Op.getOperand(2);
3457 SDValue FalseOp = Op.getOperand(3);
3458 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
3459 SDLoc DL(Op);
3460
3461 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
3462
3463 // Check for absolute and negative-absolute selections, including those
3464 // where the comparison value is sign-extended (for LPGFR and LNGFR).
3465 // This check supplements the one in DAGCombiner.
3466 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
3467 C.CCMask != SystemZ::CCMASK_CMP_NE &&
3468 C.Op1.getOpcode() == ISD::Constant &&
3469 cast<ConstantSDNode>(C.Op1)->getValueSizeInBits(0) <= 64 &&
3470 C.Op1->getAsZExtVal() == 0) {
3471 if (isAbsolute(C.Op0, TrueOp, FalseOp))
3472 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
3473 if (isAbsolute(C.Op0, FalseOp, TrueOp))
3474 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
3475 }
3476
3477 SDValue CCReg = emitCmp(DAG, DL, C);
3478 SDValue Ops[] = {TrueOp, FalseOp,
3479 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
3480 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg};
3481
3482 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
3483 }
3484
lowerGlobalAddress(GlobalAddressSDNode * Node,SelectionDAG & DAG) const3485 SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
3486 SelectionDAG &DAG) const {
3487 SDLoc DL(Node);
3488 const GlobalValue *GV = Node->getGlobal();
3489 int64_t Offset = Node->getOffset();
3490 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3491 CodeModel::Model CM = DAG.getTarget().getCodeModel();
3492
3493 SDValue Result;
3494 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
3495 if (isInt<32>(Offset)) {
3496 // Assign anchors at 1<<12 byte boundaries.
3497 uint64_t Anchor = Offset & ~uint64_t(0xfff);
3498 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
3499 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3500
3501 // The offset can be folded into the address if it is aligned to a
3502 // halfword.
3503 Offset -= Anchor;
3504 if (Offset != 0 && (Offset & 1) == 0) {
3505 SDValue Full =
3506 DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
3507 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
3508 Offset = 0;
3509 }
3510 } else {
3511 // Conservatively load a constant offset greater than 32 bits into a
3512 // register below.
3513 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT);
3514 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3515 }
3516 } else if (Subtarget.isTargetELF()) {
3517 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
3518 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3519 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
3520 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3521 } else if (Subtarget.isTargetzOS()) {
3522 Result = getADAEntry(DAG, GV, DL, PtrVT);
3523 } else
3524 llvm_unreachable("Unexpected Subtarget");
3525
3526 // If there was a non-zero offset that we didn't fold, create an explicit
3527 // addition for it.
3528 if (Offset != 0)
3529 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
3530 DAG.getConstant(Offset, DL, PtrVT));
3531
3532 return Result;
3533 }
3534
lowerTLSGetOffset(GlobalAddressSDNode * Node,SelectionDAG & DAG,unsigned Opcode,SDValue GOTOffset) const3535 SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
3536 SelectionDAG &DAG,
3537 unsigned Opcode,
3538 SDValue GOTOffset) const {
3539 SDLoc DL(Node);
3540 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3541 SDValue Chain = DAG.getEntryNode();
3542 SDValue Glue;
3543
3544 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3545 CallingConv::GHC)
3546 report_fatal_error("In GHC calling convention TLS is not supported");
3547
3548 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
3549 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
3550 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
3551 Glue = Chain.getValue(1);
3552 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
3553 Glue = Chain.getValue(1);
3554
3555 // The first call operand is the chain and the second is the TLS symbol.
3556 SmallVector<SDValue, 8> Ops;
3557 Ops.push_back(Chain);
3558 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
3559 Node->getValueType(0),
3560 0, 0));
3561
3562 // Add argument registers to the end of the list so that they are
3563 // known live into the call.
3564 Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
3565 Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
3566
3567 // Add a register mask operand representing the call-preserved registers.
3568 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
3569 const uint32_t *Mask =
3570 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
3571 assert(Mask && "Missing call preserved mask for calling convention");
3572 Ops.push_back(DAG.getRegisterMask(Mask));
3573
3574 // Glue the call to the argument copies.
3575 Ops.push_back(Glue);
3576
3577 // Emit the call.
3578 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3579 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
3580 Glue = Chain.getValue(1);
3581
3582 // Copy the return value from %r2.
3583 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
3584 }
3585
lowerThreadPointer(const SDLoc & DL,SelectionDAG & DAG) const3586 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
3587 SelectionDAG &DAG) const {
3588 SDValue Chain = DAG.getEntryNode();
3589 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3590
3591 // The high part of the thread pointer is in access register 0.
3592 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
3593 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
3594
3595 // The low part of the thread pointer is in access register 1.
3596 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
3597 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
3598
3599 // Merge them into a single 64-bit address.
3600 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
3601 DAG.getConstant(32, DL, PtrVT));
3602 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
3603 }
3604
lowerGlobalTLSAddress(GlobalAddressSDNode * Node,SelectionDAG & DAG) const3605 SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
3606 SelectionDAG &DAG) const {
3607 if (DAG.getTarget().useEmulatedTLS())
3608 return LowerToTLSEmulatedModel(Node, DAG);
3609 SDLoc DL(Node);
3610 const GlobalValue *GV = Node->getGlobal();
3611 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3612 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
3613
3614 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3615 CallingConv::GHC)
3616 report_fatal_error("In GHC calling convention TLS is not supported");
3617
3618 SDValue TP = lowerThreadPointer(DL, DAG);
3619
3620 // Get the offset of GA from the thread pointer, based on the TLS model.
3621 SDValue Offset;
3622 switch (model) {
3623 case TLSModel::GeneralDynamic: {
3624 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
3625 SystemZConstantPoolValue *CPV =
3626 SystemZConstantPoolValue::Create(GV, SystemZCP::TLSGD);
3627
3628 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
3629 Offset = DAG.getLoad(
3630 PtrVT, DL, DAG.getEntryNode(), Offset,
3631 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3632
3633 // Call __tls_get_offset to retrieve the offset.
3634 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset);
3635 break;
3636 }
3637
3638 case TLSModel::LocalDynamic: {
3639 // Load the GOT offset of the module ID.
3640 SystemZConstantPoolValue *CPV =
3641 SystemZConstantPoolValue::Create(GV, SystemZCP::TLSLDM);
3642
3643 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
3644 Offset = DAG.getLoad(
3645 PtrVT, DL, DAG.getEntryNode(), Offset,
3646 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3647
3648 // Call __tls_get_offset to retrieve the module base offset.
3649 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset);
3650
3651 // Note: The SystemZLDCleanupPass will remove redundant computations
3652 // of the module base offset. Count total number of local-dynamic
3653 // accesses to trigger execution of that pass.
3654 SystemZMachineFunctionInfo* MFI =
3655 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
3656 MFI->incNumLocalDynamicTLSAccesses();
3657
3658 // Add the per-symbol offset.
3659 CPV = SystemZConstantPoolValue::Create(GV, SystemZCP::DTPOFF);
3660
3661 SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, Align(8));
3662 DTPOffset = DAG.getLoad(
3663 PtrVT, DL, DAG.getEntryNode(), DTPOffset,
3664 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3665
3666 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
3667 break;
3668 }
3669
3670 case TLSModel::InitialExec: {
3671 // Load the offset from the GOT.
3672 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
3673 SystemZII::MO_INDNTPOFF);
3674 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
3675 Offset =
3676 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
3677 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3678 break;
3679 }
3680
3681 case TLSModel::LocalExec: {
3682 // Force the offset into the constant pool and load it from there.
3683 SystemZConstantPoolValue *CPV =
3684 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF);
3685
3686 Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
3687 Offset = DAG.getLoad(
3688 PtrVT, DL, DAG.getEntryNode(), Offset,
3689 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3690 break;
3691 }
3692 }
3693
3694 // Add the base and offset together.
3695 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
3696 }
3697
lowerBlockAddress(BlockAddressSDNode * Node,SelectionDAG & DAG) const3698 SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
3699 SelectionDAG &DAG) const {
3700 SDLoc DL(Node);
3701 const BlockAddress *BA = Node->getBlockAddress();
3702 int64_t Offset = Node->getOffset();
3703 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3704
3705 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
3706 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3707 return Result;
3708 }
3709
lowerJumpTable(JumpTableSDNode * JT,SelectionDAG & DAG) const3710 SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
3711 SelectionDAG &DAG) const {
3712 SDLoc DL(JT);
3713 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3714 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
3715
3716 // Use LARL to load the address of the table.
3717 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3718 }
3719
lowerConstantPool(ConstantPoolSDNode * CP,SelectionDAG & DAG) const3720 SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
3721 SelectionDAG &DAG) const {
3722 SDLoc DL(CP);
3723 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3724
3725 SDValue Result;
3726 if (CP->isMachineConstantPoolEntry())
3727 Result =
3728 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CP->getAlign());
3729 else
3730 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlign(),
3731 CP->getOffset());
3732
3733 // Use LARL to load the address of the constant pool entry.
3734 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3735 }
3736
lowerFRAMEADDR(SDValue Op,SelectionDAG & DAG) const3737 SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
3738 SelectionDAG &DAG) const {
3739 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
3740 MachineFunction &MF = DAG.getMachineFunction();
3741 MachineFrameInfo &MFI = MF.getFrameInfo();
3742 MFI.setFrameAddressIsTaken(true);
3743
3744 SDLoc DL(Op);
3745 unsigned Depth = Op.getConstantOperandVal(0);
3746 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3747
3748 // By definition, the frame address is the address of the back chain. (In
3749 // the case of packed stack without backchain, return the address where the
3750 // backchain would have been stored. This will either be an unused space or
3751 // contain a saved register).
3752 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
3753 SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
3754
3755 if (Depth > 0) {
3756 // FIXME The frontend should detect this case.
3757 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
3758 report_fatal_error("Unsupported stack frame traversal count");
3759
3760 SDValue Offset = DAG.getConstant(TFL->getBackchainOffset(MF), DL, PtrVT);
3761 while (Depth--) {
3762 BackChain = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), BackChain,
3763 MachinePointerInfo());
3764 BackChain = DAG.getNode(ISD::ADD, DL, PtrVT, BackChain, Offset);
3765 }
3766 }
3767
3768 return BackChain;
3769 }
3770
lowerRETURNADDR(SDValue Op,SelectionDAG & DAG) const3771 SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
3772 SelectionDAG &DAG) const {
3773 MachineFunction &MF = DAG.getMachineFunction();
3774 MachineFrameInfo &MFI = MF.getFrameInfo();
3775 MFI.setReturnAddressIsTaken(true);
3776
3777 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3778 return SDValue();
3779
3780 SDLoc DL(Op);
3781 unsigned Depth = Op.getConstantOperandVal(0);
3782 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3783
3784 if (Depth > 0) {
3785 // FIXME The frontend should detect this case.
3786 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
3787 report_fatal_error("Unsupported stack frame traversal count");
3788
3789 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3790 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
3791 int Offset = (TFL->usePackedStack(MF) ? -2 : 14) *
3792 getTargetMachine().getPointerSize(0);
3793 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, FrameAddr,
3794 DAG.getConstant(Offset, DL, PtrVT));
3795 return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr,
3796 MachinePointerInfo());
3797 }
3798
3799 // Return R14D, which has the return address. Mark it an implicit live-in.
3800 Register LinkReg = MF.addLiveIn(SystemZ::R14D, &SystemZ::GR64BitRegClass);
3801 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
3802 }
3803
lowerBITCAST(SDValue Op,SelectionDAG & DAG) const3804 SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
3805 SelectionDAG &DAG) const {
3806 SDLoc DL(Op);
3807 SDValue In = Op.getOperand(0);
3808 EVT InVT = In.getValueType();
3809 EVT ResVT = Op.getValueType();
3810
3811 // Convert loads directly. This is normally done by DAGCombiner,
3812 // but we need this case for bitcasts that are created during lowering
3813 // and which are then lowered themselves.
3814 if (auto *LoadN = dyn_cast<LoadSDNode>(In))
3815 if (ISD::isNormalLoad(LoadN)) {
3816 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
3817 LoadN->getBasePtr(), LoadN->getMemOperand());
3818 // Update the chain uses.
3819 DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1));
3820 return NewLoad;
3821 }
3822
3823 if (InVT == MVT::i32 && ResVT == MVT::f32) {
3824 SDValue In64;
3825 if (Subtarget.hasHighWord()) {
3826 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
3827 MVT::i64);
3828 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3829 MVT::i64, SDValue(U64, 0), In);
3830 } else {
3831 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
3832 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
3833 DAG.getConstant(32, DL, MVT::i64));
3834 }
3835 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
3836 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32,
3837 DL, MVT::f32, Out64);
3838 }
3839 if (InVT == MVT::f32 && ResVT == MVT::i32) {
3840 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
3841 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3842 MVT::f64, SDValue(U64, 0), In);
3843 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
3844 if (Subtarget.hasHighWord())
3845 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
3846 MVT::i32, Out64);
3847 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
3848 DAG.getConstant(32, DL, MVT::i64));
3849 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
3850 }
3851 llvm_unreachable("Unexpected bitcast combination");
3852 }
3853
lowerVASTART(SDValue Op,SelectionDAG & DAG) const3854 SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
3855 SelectionDAG &DAG) const {
3856
3857 if (Subtarget.isTargetXPLINK64())
3858 return lowerVASTART_XPLINK(Op, DAG);
3859 else
3860 return lowerVASTART_ELF(Op, DAG);
3861 }
3862
lowerVASTART_XPLINK(SDValue Op,SelectionDAG & DAG) const3863 SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
3864 SelectionDAG &DAG) const {
3865 MachineFunction &MF = DAG.getMachineFunction();
3866 SystemZMachineFunctionInfo *FuncInfo =
3867 MF.getInfo<SystemZMachineFunctionInfo>();
3868
3869 SDLoc DL(Op);
3870
3871 // vastart just stores the address of the VarArgsFrameIndex slot into the
3872 // memory location argument.
3873 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3874 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
3875 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3876 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
3877 MachinePointerInfo(SV));
3878 }
3879
lowerVASTART_ELF(SDValue Op,SelectionDAG & DAG) const3880 SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
3881 SelectionDAG &DAG) const {
3882 MachineFunction &MF = DAG.getMachineFunction();
3883 SystemZMachineFunctionInfo *FuncInfo =
3884 MF.getInfo<SystemZMachineFunctionInfo>();
3885 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3886
3887 SDValue Chain = Op.getOperand(0);
3888 SDValue Addr = Op.getOperand(1);
3889 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3890 SDLoc DL(Op);
3891
3892 // The initial values of each field.
3893 const unsigned NumFields = 4;
3894 SDValue Fields[NumFields] = {
3895 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
3896 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
3897 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
3898 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
3899 };
3900
3901 // Store each field into its respective slot.
3902 SDValue MemOps[NumFields];
3903 unsigned Offset = 0;
3904 for (unsigned I = 0; I < NumFields; ++I) {
3905 SDValue FieldAddr = Addr;
3906 if (Offset != 0)
3907 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
3908 DAG.getIntPtrConstant(Offset, DL));
3909 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
3910 MachinePointerInfo(SV, Offset));
3911 Offset += 8;
3912 }
3913 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3914 }
3915
lowerVACOPY(SDValue Op,SelectionDAG & DAG) const3916 SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
3917 SelectionDAG &DAG) const {
3918 SDValue Chain = Op.getOperand(0);
3919 SDValue DstPtr = Op.getOperand(1);
3920 SDValue SrcPtr = Op.getOperand(2);
3921 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
3922 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
3923 SDLoc DL(Op);
3924
3925 uint32_t Sz =
3926 Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(0) : 32;
3927 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL),
3928 Align(8), /*isVolatile*/ false, /*AlwaysInline*/ false,
3929 /*isTailCall*/ false, MachinePointerInfo(DstSV),
3930 MachinePointerInfo(SrcSV));
3931 }
3932
3933 SDValue
lowerDYNAMIC_STACKALLOC(SDValue Op,SelectionDAG & DAG) const3934 SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
3935 SelectionDAG &DAG) const {
3936 if (Subtarget.isTargetXPLINK64())
3937 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
3938 else
3939 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
3940 }
3941
3942 SDValue
lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,SelectionDAG & DAG) const3943 SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
3944 SelectionDAG &DAG) const {
3945 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
3946 MachineFunction &MF = DAG.getMachineFunction();
3947 bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
3948 SDValue Chain = Op.getOperand(0);
3949 SDValue Size = Op.getOperand(1);
3950 SDValue Align = Op.getOperand(2);
3951 SDLoc DL(Op);
3952
3953 // If user has set the no alignment function attribute, ignore
3954 // alloca alignments.
3955 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
3956
3957 uint64_t StackAlign = TFI->getStackAlignment();
3958 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
3959 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
3960
3961 SDValue NeededSpace = Size;
3962
3963 // Add extra space for alignment if needed.
3964 EVT PtrVT = getPointerTy(MF.getDataLayout());
3965 if (ExtraAlignSpace)
3966 NeededSpace = DAG.getNode(ISD::ADD, DL, PtrVT, NeededSpace,
3967 DAG.getConstant(ExtraAlignSpace, DL, PtrVT));
3968
3969 bool IsSigned = false;
3970 bool DoesNotReturn = false;
3971 bool IsReturnValueUsed = false;
3972 EVT VT = Op.getValueType();
3973 SDValue AllocaCall =
3974 makeExternalCall(Chain, DAG, "@@ALCAXP", VT, ArrayRef(NeededSpace),
3975 CallingConv::C, IsSigned, DL, DoesNotReturn,
3976 IsReturnValueUsed)
3977 .first;
3978
3979 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
3980 // to end of call in order to ensure it isn't broken up from the call
3981 // sequence.
3982 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
3983 Register SPReg = Regs.getStackPointerRegister();
3984 Chain = AllocaCall.getValue(1);
3985 SDValue Glue = AllocaCall.getValue(2);
3986 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, DL, SPReg, PtrVT, Glue);
3987 Chain = NewSPRegNode.getValue(1);
3988
3989 MVT PtrMVT = getPointerMemTy(MF.getDataLayout());
3990 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, PtrMVT);
3991 SDValue Result = DAG.getNode(ISD::ADD, DL, PtrMVT, NewSPRegNode, ArgAdjust);
3992
3993 // Dynamically realign if needed.
3994 if (ExtraAlignSpace) {
3995 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
3996 DAG.getConstant(ExtraAlignSpace, DL, PtrVT));
3997 Result = DAG.getNode(ISD::AND, DL, PtrVT, Result,
3998 DAG.getConstant(~(RequiredAlign - 1), DL, PtrVT));
3999 }
4000
4001 SDValue Ops[2] = {Result, Chain};
4002 return DAG.getMergeValues(Ops, DL);
4003 }
4004
4005 SDValue
lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,SelectionDAG & DAG) const4006 SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4007 SelectionDAG &DAG) const {
4008 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4009 MachineFunction &MF = DAG.getMachineFunction();
4010 bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
4011 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4012
4013 SDValue Chain = Op.getOperand(0);
4014 SDValue Size = Op.getOperand(1);
4015 SDValue Align = Op.getOperand(2);
4016 SDLoc DL(Op);
4017
4018 // If user has set the no alignment function attribute, ignore
4019 // alloca alignments.
4020 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4021
4022 uint64_t StackAlign = TFI->getStackAlignment();
4023 uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
4024 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4025
4026 Register SPReg = getStackPointerRegisterToSaveRestore();
4027 SDValue NeededSpace = Size;
4028
4029 // Get a reference to the stack pointer.
4030 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
4031
4032 // If we need a backchain, save it now.
4033 SDValue Backchain;
4034 if (StoreBackchain)
4035 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
4036 MachinePointerInfo());
4037
4038 // Add extra space for alignment if needed.
4039 if (ExtraAlignSpace)
4040 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
4041 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
4042
4043 // Get the new stack pointer value.
4044 SDValue NewSP;
4045 if (hasInlineStackProbe(MF)) {
4046 NewSP = DAG.getNode(SystemZISD::PROBED_ALLOCA, DL,
4047 DAG.getVTList(MVT::i64, MVT::Other), Chain, OldSP, NeededSpace);
4048 Chain = NewSP.getValue(1);
4049 }
4050 else {
4051 NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
4052 // Copy the new stack pointer back.
4053 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
4054 }
4055
4056 // The allocated data lives above the 160 bytes allocated for the standard
4057 // frame, plus any outgoing stack arguments. We don't know how much that
4058 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4059 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
4060 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
4061
4062 // Dynamically realign if needed.
4063 if (RequiredAlign > StackAlign) {
4064 Result =
4065 DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
4066 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
4067 Result =
4068 DAG.getNode(ISD::AND, DL, MVT::i64, Result,
4069 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
4070 }
4071
4072 if (StoreBackchain)
4073 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
4074 MachinePointerInfo());
4075
4076 SDValue Ops[2] = { Result, Chain };
4077 return DAG.getMergeValues(Ops, DL);
4078 }
4079
lowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,SelectionDAG & DAG) const4080 SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4081 SDValue Op, SelectionDAG &DAG) const {
4082 SDLoc DL(Op);
4083
4084 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
4085 }
4086
lowerSMUL_LOHI(SDValue Op,SelectionDAG & DAG) const4087 SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4088 SelectionDAG &DAG) const {
4089 EVT VT = Op.getValueType();
4090 SDLoc DL(Op);
4091 SDValue Ops[2];
4092 if (is32Bit(VT))
4093 // Just do a normal 64-bit multiplication and extract the results.
4094 // We define this so that it can be used for constant division.
4095 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0),
4096 Op.getOperand(1), Ops[1], Ops[0]);
4097 else if (Subtarget.hasMiscellaneousExtensions2())
4098 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4099 // the high result in the even register. ISD::SMUL_LOHI is defined to
4100 // return the low half first, so the results are in reverse order.
4101 lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI,
4102 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
4103 else {
4104 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4105 //
4106 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4107 //
4108 // but using the fact that the upper halves are either all zeros
4109 // or all ones:
4110 //
4111 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4112 //
4113 // and grouping the right terms together since they are quicker than the
4114 // multiplication:
4115 //
4116 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4117 SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
4118 SDValue LL = Op.getOperand(0);
4119 SDValue RL = Op.getOperand(1);
4120 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
4121 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
4122 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4123 // the high result in the even register. ISD::SMUL_LOHI is defined to
4124 // return the low half first, so the results are in reverse order.
4125 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
4126 LL, RL, Ops[1], Ops[0]);
4127 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
4128 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
4129 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
4130 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
4131 }
4132 return DAG.getMergeValues(Ops, DL);
4133 }
4134
lowerUMUL_LOHI(SDValue Op,SelectionDAG & DAG) const4135 SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4136 SelectionDAG &DAG) const {
4137 EVT VT = Op.getValueType();
4138 SDLoc DL(Op);
4139 SDValue Ops[2];
4140 if (is32Bit(VT))
4141 // Just do a normal 64-bit multiplication and extract the results.
4142 // We define this so that it can be used for constant division.
4143 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0),
4144 Op.getOperand(1), Ops[1], Ops[0]);
4145 else
4146 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4147 // the high result in the even register. ISD::UMUL_LOHI is defined to
4148 // return the low half first, so the results are in reverse order.
4149 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
4150 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
4151 return DAG.getMergeValues(Ops, DL);
4152 }
4153
lowerSDIVREM(SDValue Op,SelectionDAG & DAG) const4154 SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4155 SelectionDAG &DAG) const {
4156 SDValue Op0 = Op.getOperand(0);
4157 SDValue Op1 = Op.getOperand(1);
4158 EVT VT = Op.getValueType();
4159 SDLoc DL(Op);
4160
4161 // We use DSGF for 32-bit division. This means the first operand must
4162 // always be 64-bit, and the second operand should be 32-bit whenever
4163 // that is possible, to improve performance.
4164 if (is32Bit(VT))
4165 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
4166 else if (DAG.ComputeNumSignBits(Op1) > 32)
4167 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
4168
4169 // DSG(F) returns the remainder in the even register and the
4170 // quotient in the odd register.
4171 SDValue Ops[2];
4172 lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
4173 return DAG.getMergeValues(Ops, DL);
4174 }
4175
lowerUDIVREM(SDValue Op,SelectionDAG & DAG) const4176 SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4177 SelectionDAG &DAG) const {
4178 EVT VT = Op.getValueType();
4179 SDLoc DL(Op);
4180
4181 // DL(G) returns the remainder in the even register and the
4182 // quotient in the odd register.
4183 SDValue Ops[2];
4184 lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM,
4185 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
4186 return DAG.getMergeValues(Ops, DL);
4187 }
4188
lowerOR(SDValue Op,SelectionDAG & DAG) const4189 SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4190 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4191
4192 // Get the known-zero masks for each operand.
4193 SDValue Ops[] = {Op.getOperand(0), Op.getOperand(1)};
4194 KnownBits Known[2] = {DAG.computeKnownBits(Ops[0]),
4195 DAG.computeKnownBits(Ops[1])};
4196
4197 // See if the upper 32 bits of one operand and the lower 32 bits of the
4198 // other are known zero. They are the low and high operands respectively.
4199 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4200 Known[1].Zero.getZExtValue() };
4201 unsigned High, Low;
4202 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4203 High = 1, Low = 0;
4204 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4205 High = 0, Low = 1;
4206 else
4207 return Op;
4208
4209 SDValue LowOp = Ops[Low];
4210 SDValue HighOp = Ops[High];
4211
4212 // If the high part is a constant, we're better off using IILH.
4213 if (HighOp.getOpcode() == ISD::Constant)
4214 return Op;
4215
4216 // If the low part is a constant that is outside the range of LHI,
4217 // then we're better off using IILF.
4218 if (LowOp.getOpcode() == ISD::Constant) {
4219 int64_t Value = int32_t(LowOp->getAsZExtVal());
4220 if (!isInt<16>(Value))
4221 return Op;
4222 }
4223
4224 // Check whether the high part is an AND that doesn't change the
4225 // high 32 bits and just masks out low bits. We can skip it if so.
4226 if (HighOp.getOpcode() == ISD::AND &&
4227 HighOp.getOperand(1).getOpcode() == ISD::Constant) {
4228 SDValue HighOp0 = HighOp.getOperand(0);
4229 uint64_t Mask = HighOp.getConstantOperandVal(1);
4230 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff))))
4231 HighOp = HighOp0;
4232 }
4233
4234 // Take advantage of the fact that all GR32 operations only change the
4235 // low 32 bits by truncating Low to an i32 and inserting it directly
4236 // using a subreg. The interesting cases are those where the truncation
4237 // can be folded.
4238 SDLoc DL(Op);
4239 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
4240 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
4241 MVT::i64, HighOp, Low32);
4242 }
4243
4244 // Lower SADDO/SSUBO/UADDO/USUBO nodes.
lowerXALUO(SDValue Op,SelectionDAG & DAG) const4245 SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4246 SelectionDAG &DAG) const {
4247 SDNode *N = Op.getNode();
4248 SDValue LHS = N->getOperand(0);
4249 SDValue RHS = N->getOperand(1);
4250 SDLoc DL(N);
4251
4252 if (N->getValueType(0) == MVT::i128) {
4253 unsigned BaseOp = 0;
4254 unsigned FlagOp = 0;
4255 bool IsBorrow = false;
4256 switch (Op.getOpcode()) {
4257 default: llvm_unreachable("Unknown instruction!");
4258 case ISD::UADDO:
4259 BaseOp = ISD::ADD;
4260 FlagOp = SystemZISD::VACC;
4261 break;
4262 case ISD::USUBO:
4263 BaseOp = ISD::SUB;
4264 FlagOp = SystemZISD::VSCBI;
4265 IsBorrow = true;
4266 break;
4267 }
4268 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS);
4269 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS);
4270 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag,
4271 DAG.getValueType(MVT::i1));
4272 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));
4273 if (IsBorrow)
4274 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(),
4275 Flag, DAG.getConstant(1, DL, Flag.getValueType()));
4276 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);
4277 }
4278
4279 unsigned BaseOp = 0;
4280 unsigned CCValid = 0;
4281 unsigned CCMask = 0;
4282
4283 switch (Op.getOpcode()) {
4284 default: llvm_unreachable("Unknown instruction!");
4285 case ISD::SADDO:
4286 BaseOp = SystemZISD::SADDO;
4287 CCValid = SystemZ::CCMASK_ARITH;
4288 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4289 break;
4290 case ISD::SSUBO:
4291 BaseOp = SystemZISD::SSUBO;
4292 CCValid = SystemZ::CCMASK_ARITH;
4293 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4294 break;
4295 case ISD::UADDO:
4296 BaseOp = SystemZISD::UADDO;
4297 CCValid = SystemZ::CCMASK_LOGICAL;
4298 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4299 break;
4300 case ISD::USUBO:
4301 BaseOp = SystemZISD::USUBO;
4302 CCValid = SystemZ::CCMASK_LOGICAL;
4303 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4304 break;
4305 }
4306
4307 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
4308 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
4309
4310 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
4311 if (N->getValueType(1) == MVT::i1)
4312 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
4313
4314 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
4315 }
4316
isAddCarryChain(SDValue Carry)4317 static bool isAddCarryChain(SDValue Carry) {
4318 while (Carry.getOpcode() == ISD::UADDO_CARRY)
4319 Carry = Carry.getOperand(2);
4320 return Carry.getOpcode() == ISD::UADDO;
4321 }
4322
isSubBorrowChain(SDValue Carry)4323 static bool isSubBorrowChain(SDValue Carry) {
4324 while (Carry.getOpcode() == ISD::USUBO_CARRY)
4325 Carry = Carry.getOperand(2);
4326 return Carry.getOpcode() == ISD::USUBO;
4327 }
4328
4329 // Lower UADDO_CARRY/USUBO_CARRY nodes.
lowerUADDSUBO_CARRY(SDValue Op,SelectionDAG & DAG) const4330 SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
4331 SelectionDAG &DAG) const {
4332
4333 SDNode *N = Op.getNode();
4334 MVT VT = N->getSimpleValueType(0);
4335
4336 // Let legalize expand this if it isn't a legal type yet.
4337 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
4338 return SDValue();
4339
4340 SDValue LHS = N->getOperand(0);
4341 SDValue RHS = N->getOperand(1);
4342 SDValue Carry = Op.getOperand(2);
4343 SDLoc DL(N);
4344
4345 if (VT == MVT::i128) {
4346 unsigned BaseOp = 0;
4347 unsigned FlagOp = 0;
4348 bool IsBorrow = false;
4349 switch (Op.getOpcode()) {
4350 default: llvm_unreachable("Unknown instruction!");
4351 case ISD::UADDO_CARRY:
4352 BaseOp = SystemZISD::VAC;
4353 FlagOp = SystemZISD::VACCC;
4354 break;
4355 case ISD::USUBO_CARRY:
4356 BaseOp = SystemZISD::VSBI;
4357 FlagOp = SystemZISD::VSBCBI;
4358 IsBorrow = true;
4359 break;
4360 }
4361 if (IsBorrow)
4362 Carry = DAG.getNode(ISD::XOR, DL, Carry.getValueType(),
4363 Carry, DAG.getConstant(1, DL, Carry.getValueType()));
4364 Carry = DAG.getZExtOrTrunc(Carry, DL, MVT::i128);
4365 SDValue Result = DAG.getNode(BaseOp, DL, MVT::i128, LHS, RHS, Carry);
4366 SDValue Flag = DAG.getNode(FlagOp, DL, MVT::i128, LHS, RHS, Carry);
4367 Flag = DAG.getNode(ISD::AssertZext, DL, MVT::i128, Flag,
4368 DAG.getValueType(MVT::i1));
4369 Flag = DAG.getZExtOrTrunc(Flag, DL, N->getValueType(1));
4370 if (IsBorrow)
4371 Flag = DAG.getNode(ISD::XOR, DL, Flag.getValueType(),
4372 Flag, DAG.getConstant(1, DL, Flag.getValueType()));
4373 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Flag);
4374 }
4375
4376 unsigned BaseOp = 0;
4377 unsigned CCValid = 0;
4378 unsigned CCMask = 0;
4379
4380 switch (Op.getOpcode()) {
4381 default: llvm_unreachable("Unknown instruction!");
4382 case ISD::UADDO_CARRY:
4383 if (!isAddCarryChain(Carry))
4384 return SDValue();
4385
4386 BaseOp = SystemZISD::ADDCARRY;
4387 CCValid = SystemZ::CCMASK_LOGICAL;
4388 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4389 break;
4390 case ISD::USUBO_CARRY:
4391 if (!isSubBorrowChain(Carry))
4392 return SDValue();
4393
4394 BaseOp = SystemZISD::SUBCARRY;
4395 CCValid = SystemZ::CCMASK_LOGICAL;
4396 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4397 break;
4398 }
4399
4400 // Set the condition code from the carry flag.
4401 Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry,
4402 DAG.getConstant(CCValid, DL, MVT::i32),
4403 DAG.getConstant(CCMask, DL, MVT::i32));
4404
4405 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
4406 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
4407
4408 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
4409 if (N->getValueType(1) == MVT::i1)
4410 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
4411
4412 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
4413 }
4414
lowerCTPOP(SDValue Op,SelectionDAG & DAG) const4415 SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
4416 SelectionDAG &DAG) const {
4417 EVT VT = Op.getValueType();
4418 SDLoc DL(Op);
4419 Op = Op.getOperand(0);
4420
4421 if (VT.getScalarSizeInBits() == 128) {
4422 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op);
4423 Op = DAG.getNode(ISD::CTPOP, DL, MVT::v2i64, Op);
4424 SDValue Tmp = DAG.getSplatBuildVector(MVT::v2i64, DL,
4425 DAG.getConstant(0, DL, MVT::i64));
4426 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
4427 return Op;
4428 }
4429
4430 // Handle vector types via VPOPCT.
4431 if (VT.isVector()) {
4432 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
4433 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
4434 switch (VT.getScalarSizeInBits()) {
4435 case 8:
4436 break;
4437 case 16: {
4438 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
4439 SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
4440 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
4441 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
4442 Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
4443 break;
4444 }
4445 case 32: {
4446 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
4447 DAG.getConstant(0, DL, MVT::i32));
4448 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
4449 break;
4450 }
4451 case 64: {
4452 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
4453 DAG.getConstant(0, DL, MVT::i32));
4454 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
4455 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
4456 break;
4457 }
4458 default:
4459 llvm_unreachable("Unexpected type");
4460 }
4461 return Op;
4462 }
4463
4464 // Get the known-zero mask for the operand.
4465 KnownBits Known = DAG.computeKnownBits(Op);
4466 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
4467 if (NumSignificantBits == 0)
4468 return DAG.getConstant(0, DL, VT);
4469
4470 // Skip known-zero high parts of the operand.
4471 int64_t OrigBitSize = VT.getSizeInBits();
4472 int64_t BitSize = llvm::bit_ceil(NumSignificantBits);
4473 BitSize = std::min(BitSize, OrigBitSize);
4474
4475 // The POPCNT instruction counts the number of bits in each byte.
4476 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
4477 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
4478 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
4479
4480 // Add up per-byte counts in a binary tree. All bits of Op at
4481 // position larger than BitSize remain zero throughout.
4482 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
4483 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
4484 if (BitSize != OrigBitSize)
4485 Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
4486 DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
4487 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
4488 }
4489
4490 // Extract overall result from high byte.
4491 if (BitSize > 8)
4492 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
4493 DAG.getConstant(BitSize - 8, DL, VT));
4494
4495 return Op;
4496 }
4497
lowerATOMIC_FENCE(SDValue Op,SelectionDAG & DAG) const4498 SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4499 SelectionDAG &DAG) const {
4500 SDLoc DL(Op);
4501 AtomicOrdering FenceOrdering =
4502 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
4503 SyncScope::ID FenceSSID =
4504 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
4505
4506 // The only fence that needs an instruction is a sequentially-consistent
4507 // cross-thread fence.
4508 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
4509 FenceSSID == SyncScope::System) {
4510 return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
4511 Op.getOperand(0)),
4512 0);
4513 }
4514
4515 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
4516 return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
4517 }
4518
4519 // Op is an atomic load. Lower it into a normal volatile load.
lowerATOMIC_LOAD(SDValue Op,SelectionDAG & DAG) const4520 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
4521 SelectionDAG &DAG) const {
4522 auto *Node = cast<AtomicSDNode>(Op.getNode());
4523 if (Node->getMemoryVT() == MVT::i128) {
4524 // Use same code to handle both legal and non-legal i128 types.
4525 SmallVector<SDValue, 2> Results;
4526 LowerOperationWrapper(Node, Results, DAG);
4527 return DAG.getMergeValues(Results, SDLoc(Op));
4528 }
4529 return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
4530 Node->getChain(), Node->getBasePtr(),
4531 Node->getMemoryVT(), Node->getMemOperand());
4532 }
4533
4534 // Op is an atomic store. Lower it into a normal volatile store.
lowerATOMIC_STORE(SDValue Op,SelectionDAG & DAG) const4535 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
4536 SelectionDAG &DAG) const {
4537 auto *Node = cast<AtomicSDNode>(Op.getNode());
4538 if (Node->getMemoryVT() == MVT::i128) {
4539 // Use same code to handle both legal and non-legal i128 types.
4540 SmallVector<SDValue, 1> Results;
4541 LowerOperationWrapper(Node, Results, DAG);
4542 return DAG.getMergeValues(Results, SDLoc(Op));
4543 }
4544 SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
4545 Node->getBasePtr(), Node->getMemoryVT(),
4546 Node->getMemOperand());
4547 // We have to enforce sequential consistency by performing a
4548 // serialization operation after the store.
4549 if (Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
4550 Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
4551 MVT::Other, Chain), 0);
4552 return Chain;
4553 }
4554
4555 // Prepare for a Compare And Swap for a subword operation. This needs to be
4556 // done in memory with 4 bytes at natural alignment.
getCSAddressAndShifts(SDValue Addr,SelectionDAG & DAG,SDLoc DL,SDValue & AlignedAddr,SDValue & BitShift,SDValue & NegBitShift)4557 static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
4558 SDValue &AlignedAddr, SDValue &BitShift,
4559 SDValue &NegBitShift) {
4560 EVT PtrVT = Addr.getValueType();
4561 EVT WideVT = MVT::i32;
4562
4563 // Get the address of the containing word.
4564 AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
4565 DAG.getConstant(-4, DL, PtrVT));
4566
4567 // Get the number of bits that the word must be rotated left in order
4568 // to bring the field to the top bits of a GR32.
4569 BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
4570 DAG.getConstant(3, DL, PtrVT));
4571 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
4572
4573 // Get the complementing shift amount, for rotating a field in the top
4574 // bits back to its proper position.
4575 NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
4576 DAG.getConstant(0, DL, WideVT), BitShift);
4577
4578 }
4579
4580 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
4581 // two into the fullword ATOMIC_LOADW_* operation given by Opcode.
lowerATOMIC_LOAD_OP(SDValue Op,SelectionDAG & DAG,unsigned Opcode) const4582 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
4583 SelectionDAG &DAG,
4584 unsigned Opcode) const {
4585 auto *Node = cast<AtomicSDNode>(Op.getNode());
4586
4587 // 32-bit operations need no special handling.
4588 EVT NarrowVT = Node->getMemoryVT();
4589 EVT WideVT = MVT::i32;
4590 if (NarrowVT == WideVT)
4591 return Op;
4592
4593 int64_t BitSize = NarrowVT.getSizeInBits();
4594 SDValue ChainIn = Node->getChain();
4595 SDValue Addr = Node->getBasePtr();
4596 SDValue Src2 = Node->getVal();
4597 MachineMemOperand *MMO = Node->getMemOperand();
4598 SDLoc DL(Node);
4599
4600 // Convert atomic subtracts of constants into additions.
4601 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
4602 if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
4603 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
4604 Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());
4605 }
4606
4607 SDValue AlignedAddr, BitShift, NegBitShift;
4608 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
4609
4610 // Extend the source operand to 32 bits and prepare it for the inner loop.
4611 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
4612 // operations require the source to be shifted in advance. (This shift
4613 // can be folded if the source is constant.) For AND and NAND, the lower
4614 // bits must be set, while for other opcodes they should be left clear.
4615 if (Opcode != SystemZISD::ATOMIC_SWAPW)
4616 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
4617 DAG.getConstant(32 - BitSize, DL, WideVT));
4618 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
4619 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
4620 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
4621 DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
4622
4623 // Construct the ATOMIC_LOADW_* node.
4624 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
4625 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
4626 DAG.getConstant(BitSize, DL, WideVT) };
4627 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
4628 NarrowVT, MMO);
4629
4630 // Rotate the result of the final CS so that the field is in the lower
4631 // bits of a GR32, then truncate it.
4632 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
4633 DAG.getConstant(BitSize, DL, WideVT));
4634 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
4635
4636 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
4637 return DAG.getMergeValues(RetOps, DL);
4638 }
4639
4640 // Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
4641 // ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
lowerATOMIC_LOAD_SUB(SDValue Op,SelectionDAG & DAG) const4642 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
4643 SelectionDAG &DAG) const {
4644 auto *Node = cast<AtomicSDNode>(Op.getNode());
4645 EVT MemVT = Node->getMemoryVT();
4646 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
4647 // A full-width operation: negate and use LAA(G).
4648 assert(Op.getValueType() == MemVT && "Mismatched VTs");
4649 assert(Subtarget.hasInterlockedAccess1() &&
4650 "Should have been expanded by AtomicExpand pass.");
4651 SDValue Src2 = Node->getVal();
4652 SDLoc DL(Src2);
4653 SDValue NegSrc2 =
4654 DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2);
4655 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
4656 Node->getChain(), Node->getBasePtr(), NegSrc2,
4657 Node->getMemOperand());
4658 }
4659
4660 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
4661 }
4662
4663 // Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
lowerATOMIC_CMP_SWAP(SDValue Op,SelectionDAG & DAG) const4664 SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
4665 SelectionDAG &DAG) const {
4666 auto *Node = cast<AtomicSDNode>(Op.getNode());
4667 SDValue ChainIn = Node->getOperand(0);
4668 SDValue Addr = Node->getOperand(1);
4669 SDValue CmpVal = Node->getOperand(2);
4670 SDValue SwapVal = Node->getOperand(3);
4671 MachineMemOperand *MMO = Node->getMemOperand();
4672 SDLoc DL(Node);
4673
4674 if (Node->getMemoryVT() == MVT::i128) {
4675 // Use same code to handle both legal and non-legal i128 types.
4676 SmallVector<SDValue, 3> Results;
4677 LowerOperationWrapper(Node, Results, DAG);
4678 return DAG.getMergeValues(Results, DL);
4679 }
4680
4681 // We have native support for 32-bit and 64-bit compare and swap, but we
4682 // still need to expand extracting the "success" result from the CC.
4683 EVT NarrowVT = Node->getMemoryVT();
4684 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
4685 if (NarrowVT == WideVT) {
4686 SDVTList Tys = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
4687 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
4688 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
4689 DL, Tys, Ops, NarrowVT, MMO);
4690 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
4691 SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
4692
4693 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
4694 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
4695 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
4696 return SDValue();
4697 }
4698
4699 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
4700 // via a fullword ATOMIC_CMP_SWAPW operation.
4701 int64_t BitSize = NarrowVT.getSizeInBits();
4702
4703 SDValue AlignedAddr, BitShift, NegBitShift;
4704 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
4705
4706 // Construct the ATOMIC_CMP_SWAPW node.
4707 SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
4708 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
4709 NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
4710 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
4711 VTList, Ops, NarrowVT, MMO);
4712 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
4713 SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
4714
4715 // emitAtomicCmpSwapW() will zero extend the result (original value).
4716 SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
4717 DAG.getValueType(NarrowVT));
4718 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
4719 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
4720 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
4721 return SDValue();
4722 }
4723
4724 MachineMemOperand::Flags
getTargetMMOFlags(const Instruction & I) const4725 SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
4726 // Because of how we convert atomic_load and atomic_store to normal loads and
4727 // stores in the DAG, we need to ensure that the MMOs are marked volatile
4728 // since DAGCombine hasn't been updated to account for atomic, but non
4729 // volatile loads. (See D57601)
4730 if (auto *SI = dyn_cast<StoreInst>(&I))
4731 if (SI->isAtomic())
4732 return MachineMemOperand::MOVolatile;
4733 if (auto *LI = dyn_cast<LoadInst>(&I))
4734 if (LI->isAtomic())
4735 return MachineMemOperand::MOVolatile;
4736 if (auto *AI = dyn_cast<AtomicRMWInst>(&I))
4737 if (AI->isAtomic())
4738 return MachineMemOperand::MOVolatile;
4739 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(&I))
4740 if (AI->isAtomic())
4741 return MachineMemOperand::MOVolatile;
4742 return MachineMemOperand::MONone;
4743 }
4744
lowerSTACKSAVE(SDValue Op,SelectionDAG & DAG) const4745 SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
4746 SelectionDAG &DAG) const {
4747 MachineFunction &MF = DAG.getMachineFunction();
4748 auto *Regs = Subtarget.getSpecialRegisters();
4749 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
4750 report_fatal_error("Variable-sized stack allocations are not supported "
4751 "in GHC calling convention");
4752 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op),
4753 Regs->getStackPointerRegister(), Op.getValueType());
4754 }
4755
lowerSTACKRESTORE(SDValue Op,SelectionDAG & DAG) const4756 SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
4757 SelectionDAG &DAG) const {
4758 MachineFunction &MF = DAG.getMachineFunction();
4759 auto *Regs = Subtarget.getSpecialRegisters();
4760 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4761
4762 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
4763 report_fatal_error("Variable-sized stack allocations are not supported "
4764 "in GHC calling convention");
4765
4766 SDValue Chain = Op.getOperand(0);
4767 SDValue NewSP = Op.getOperand(1);
4768 SDValue Backchain;
4769 SDLoc DL(Op);
4770
4771 if (StoreBackchain) {
4772 SDValue OldSP = DAG.getCopyFromReg(
4773 Chain, DL, Regs->getStackPointerRegister(), MVT::i64);
4774 Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
4775 MachinePointerInfo());
4776 }
4777
4778 Chain = DAG.getCopyToReg(Chain, DL, Regs->getStackPointerRegister(), NewSP);
4779
4780 if (StoreBackchain)
4781 Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
4782 MachinePointerInfo());
4783
4784 return Chain;
4785 }
4786
lowerPREFETCH(SDValue Op,SelectionDAG & DAG) const4787 SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
4788 SelectionDAG &DAG) const {
4789 bool IsData = Op.getConstantOperandVal(4);
4790 if (!IsData)
4791 // Just preserve the chain.
4792 return Op.getOperand(0);
4793
4794 SDLoc DL(Op);
4795 bool IsWrite = Op.getConstantOperandVal(2);
4796 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
4797 auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
4798 SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
4799 Op.getOperand(1)};
4800 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
4801 Node->getVTList(), Ops,
4802 Node->getMemoryVT(), Node->getMemOperand());
4803 }
4804
4805 // Convert condition code in CCReg to an i32 value.
getCCResult(SelectionDAG & DAG,SDValue CCReg)4806 static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
4807 SDLoc DL(CCReg);
4808 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
4809 return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
4810 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
4811 }
4812
4813 SDValue
lowerINTRINSIC_W_CHAIN(SDValue Op,SelectionDAG & DAG) const4814 SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
4815 SelectionDAG &DAG) const {
4816 unsigned Opcode, CCValid;
4817 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
4818 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
4819 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
4820 SDValue CC = getCCResult(DAG, SDValue(Node, 0));
4821 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC);
4822 return SDValue();
4823 }
4824
4825 return SDValue();
4826 }
4827
4828 SDValue
lowerINTRINSIC_WO_CHAIN(SDValue Op,SelectionDAG & DAG) const4829 SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
4830 SelectionDAG &DAG) const {
4831 unsigned Opcode, CCValid;
4832 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
4833 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
4834 if (Op->getNumValues() == 1)
4835 return getCCResult(DAG, SDValue(Node, 0));
4836 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
4837 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),
4838 SDValue(Node, 0), getCCResult(DAG, SDValue(Node, 1)));
4839 }
4840
4841 unsigned Id = Op.getConstantOperandVal(0);
4842 switch (Id) {
4843 case Intrinsic::thread_pointer:
4844 return lowerThreadPointer(SDLoc(Op), DAG);
4845
4846 case Intrinsic::s390_vpdi:
4847 return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
4848 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4849
4850 case Intrinsic::s390_vperm:
4851 return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
4852 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4853
4854 case Intrinsic::s390_vuphb:
4855 case Intrinsic::s390_vuphh:
4856 case Intrinsic::s390_vuphf:
4857 return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
4858 Op.getOperand(1));
4859
4860 case Intrinsic::s390_vuplhb:
4861 case Intrinsic::s390_vuplhh:
4862 case Intrinsic::s390_vuplhf:
4863 return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
4864 Op.getOperand(1));
4865
4866 case Intrinsic::s390_vuplb:
4867 case Intrinsic::s390_vuplhw:
4868 case Intrinsic::s390_vuplf:
4869 return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
4870 Op.getOperand(1));
4871
4872 case Intrinsic::s390_vupllb:
4873 case Intrinsic::s390_vupllh:
4874 case Intrinsic::s390_vupllf:
4875 return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
4876 Op.getOperand(1));
4877
4878 case Intrinsic::s390_vsumb:
4879 case Intrinsic::s390_vsumh:
4880 case Intrinsic::s390_vsumgh:
4881 case Intrinsic::s390_vsumgf:
4882 case Intrinsic::s390_vsumqf:
4883 case Intrinsic::s390_vsumqg:
4884 return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
4885 Op.getOperand(1), Op.getOperand(2));
4886
4887 case Intrinsic::s390_vaq:
4888 return DAG.getNode(ISD::ADD, SDLoc(Op), Op.getValueType(),
4889 Op.getOperand(1), Op.getOperand(2));
4890 case Intrinsic::s390_vaccb:
4891 case Intrinsic::s390_vacch:
4892 case Intrinsic::s390_vaccf:
4893 case Intrinsic::s390_vaccg:
4894 case Intrinsic::s390_vaccq:
4895 return DAG.getNode(SystemZISD::VACC, SDLoc(Op), Op.getValueType(),
4896 Op.getOperand(1), Op.getOperand(2));
4897 case Intrinsic::s390_vacq:
4898 return DAG.getNode(SystemZISD::VAC, SDLoc(Op), Op.getValueType(),
4899 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4900 case Intrinsic::s390_vacccq:
4901 return DAG.getNode(SystemZISD::VACCC, SDLoc(Op), Op.getValueType(),
4902 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4903
4904 case Intrinsic::s390_vsq:
4905 return DAG.getNode(ISD::SUB, SDLoc(Op), Op.getValueType(),
4906 Op.getOperand(1), Op.getOperand(2));
4907 case Intrinsic::s390_vscbib:
4908 case Intrinsic::s390_vscbih:
4909 case Intrinsic::s390_vscbif:
4910 case Intrinsic::s390_vscbig:
4911 case Intrinsic::s390_vscbiq:
4912 return DAG.getNode(SystemZISD::VSCBI, SDLoc(Op), Op.getValueType(),
4913 Op.getOperand(1), Op.getOperand(2));
4914 case Intrinsic::s390_vsbiq:
4915 return DAG.getNode(SystemZISD::VSBI, SDLoc(Op), Op.getValueType(),
4916 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4917 case Intrinsic::s390_vsbcbiq:
4918 return DAG.getNode(SystemZISD::VSBCBI, SDLoc(Op), Op.getValueType(),
4919 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4920 }
4921
4922 return SDValue();
4923 }
4924
4925 namespace {
4926 // Says that SystemZISD operation Opcode can be used to perform the equivalent
4927 // of a VPERM with permute vector Bytes. If Opcode takes three operands,
4928 // Operand is the constant third operand, otherwise it is the number of
4929 // bytes in each element of the result.
4930 struct Permute {
4931 unsigned Opcode;
4932 unsigned Operand;
4933 unsigned char Bytes[SystemZ::VectorBytes];
4934 };
4935 }
4936
4937 static const Permute PermuteForms[] = {
4938 // VMRHG
4939 { SystemZISD::MERGE_HIGH, 8,
4940 { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
4941 // VMRHF
4942 { SystemZISD::MERGE_HIGH, 4,
4943 { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
4944 // VMRHH
4945 { SystemZISD::MERGE_HIGH, 2,
4946 { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
4947 // VMRHB
4948 { SystemZISD::MERGE_HIGH, 1,
4949 { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
4950 // VMRLG
4951 { SystemZISD::MERGE_LOW, 8,
4952 { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
4953 // VMRLF
4954 { SystemZISD::MERGE_LOW, 4,
4955 { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
4956 // VMRLH
4957 { SystemZISD::MERGE_LOW, 2,
4958 { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
4959 // VMRLB
4960 { SystemZISD::MERGE_LOW, 1,
4961 { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
4962 // VPKG
4963 { SystemZISD::PACK, 4,
4964 { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
4965 // VPKF
4966 { SystemZISD::PACK, 2,
4967 { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
4968 // VPKH
4969 { SystemZISD::PACK, 1,
4970 { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
4971 // VPDI V1, V2, 4 (low half of V1, high half of V2)
4972 { SystemZISD::PERMUTE_DWORDS, 4,
4973 { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
4974 // VPDI V1, V2, 1 (high half of V1, low half of V2)
4975 { SystemZISD::PERMUTE_DWORDS, 1,
4976 { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
4977 };
4978
4979 // Called after matching a vector shuffle against a particular pattern.
4980 // Both the original shuffle and the pattern have two vector operands.
4981 // OpNos[0] is the operand of the original shuffle that should be used for
4982 // operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
4983 // OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
4984 // set OpNo0 and OpNo1 to the shuffle operands that should actually be used
4985 // for operands 0 and 1 of the pattern.
chooseShuffleOpNos(int * OpNos,unsigned & OpNo0,unsigned & OpNo1)4986 static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
4987 if (OpNos[0] < 0) {
4988 if (OpNos[1] < 0)
4989 return false;
4990 OpNo0 = OpNo1 = OpNos[1];
4991 } else if (OpNos[1] < 0) {
4992 OpNo0 = OpNo1 = OpNos[0];
4993 } else {
4994 OpNo0 = OpNos[0];
4995 OpNo1 = OpNos[1];
4996 }
4997 return true;
4998 }
4999
5000 // Bytes is a VPERM-like permute vector, except that -1 is used for
5001 // undefined bytes. Return true if the VPERM can be implemented using P.
5002 // When returning true set OpNo0 to the VPERM operand that should be
5003 // used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5004 //
5005 // For example, if swapping the VPERM operands allows P to match, OpNo0
5006 // will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5007 // operand, but rewriting it to use two duplicated operands allows it to
5008 // match P, then OpNo0 and OpNo1 will be the same.
matchPermute(const SmallVectorImpl<int> & Bytes,const Permute & P,unsigned & OpNo0,unsigned & OpNo1)5009 static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5010 unsigned &OpNo0, unsigned &OpNo1) {
5011 int OpNos[] = { -1, -1 };
5012 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5013 int Elt = Bytes[I];
5014 if (Elt >= 0) {
5015 // Make sure that the two permute vectors use the same suboperand
5016 // byte number. Only the operand numbers (the high bits) are
5017 // allowed to differ.
5018 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5019 return false;
5020 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5021 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5022 // Make sure that the operand mappings are consistent with previous
5023 // elements.
5024 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5025 return false;
5026 OpNos[ModelOpNo] = RealOpNo;
5027 }
5028 }
5029 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5030 }
5031
5032 // As above, but search for a matching permute.
matchPermute(const SmallVectorImpl<int> & Bytes,unsigned & OpNo0,unsigned & OpNo1)5033 static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5034 unsigned &OpNo0, unsigned &OpNo1) {
5035 for (auto &P : PermuteForms)
5036 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5037 return &P;
5038 return nullptr;
5039 }
5040
5041 // Bytes is a VPERM-like permute vector, except that -1 is used for
5042 // undefined bytes. This permute is an operand of an outer permute.
5043 // See whether redistributing the -1 bytes gives a shuffle that can be
5044 // implemented using P. If so, set Transform to a VPERM-like permute vector
5045 // that, when applied to the result of P, gives the original permute in Bytes.
matchDoublePermute(const SmallVectorImpl<int> & Bytes,const Permute & P,SmallVectorImpl<int> & Transform)5046 static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5047 const Permute &P,
5048 SmallVectorImpl<int> &Transform) {
5049 unsigned To = 0;
5050 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5051 int Elt = Bytes[From];
5052 if (Elt < 0)
5053 // Byte number From of the result is undefined.
5054 Transform[From] = -1;
5055 else {
5056 while (P.Bytes[To] != Elt) {
5057 To += 1;
5058 if (To == SystemZ::VectorBytes)
5059 return false;
5060 }
5061 Transform[From] = To;
5062 }
5063 }
5064 return true;
5065 }
5066
5067 // As above, but search for a matching permute.
matchDoublePermute(const SmallVectorImpl<int> & Bytes,SmallVectorImpl<int> & Transform)5068 static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5069 SmallVectorImpl<int> &Transform) {
5070 for (auto &P : PermuteForms)
5071 if (matchDoublePermute(Bytes, P, Transform))
5072 return &P;
5073 return nullptr;
5074 }
5075
5076 // Convert the mask of the given shuffle op into a byte-level mask,
5077 // as if it had type vNi8.
getVPermMask(SDValue ShuffleOp,SmallVectorImpl<int> & Bytes)5078 static bool getVPermMask(SDValue ShuffleOp,
5079 SmallVectorImpl<int> &Bytes) {
5080 EVT VT = ShuffleOp.getValueType();
5081 unsigned NumElements = VT.getVectorNumElements();
5082 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5083
5084 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(ShuffleOp)) {
5085 Bytes.resize(NumElements * BytesPerElement, -1);
5086 for (unsigned I = 0; I < NumElements; ++I) {
5087 int Index = VSN->getMaskElt(I);
5088 if (Index >= 0)
5089 for (unsigned J = 0; J < BytesPerElement; ++J)
5090 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5091 }
5092 return true;
5093 }
5094 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5095 isa<ConstantSDNode>(ShuffleOp.getOperand(1))) {
5096 unsigned Index = ShuffleOp.getConstantOperandVal(1);
5097 Bytes.resize(NumElements * BytesPerElement, -1);
5098 for (unsigned I = 0; I < NumElements; ++I)
5099 for (unsigned J = 0; J < BytesPerElement; ++J)
5100 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5101 return true;
5102 }
5103 return false;
5104 }
5105
5106 // Bytes is a VPERM-like permute vector, except that -1 is used for
5107 // undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5108 // the result come from a contiguous sequence of bytes from one input.
5109 // Set Base to the selector for the first byte if so.
getShuffleInput(const SmallVectorImpl<int> & Bytes,unsigned Start,unsigned BytesPerElement,int & Base)5110 static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5111 unsigned BytesPerElement, int &Base) {
5112 Base = -1;
5113 for (unsigned I = 0; I < BytesPerElement; ++I) {
5114 if (Bytes[Start + I] >= 0) {
5115 unsigned Elem = Bytes[Start + I];
5116 if (Base < 0) {
5117 Base = Elem - I;
5118 // Make sure the bytes would come from one input operand.
5119 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5120 return false;
5121 } else if (unsigned(Base) != Elem - I)
5122 return false;
5123 }
5124 }
5125 return true;
5126 }
5127
5128 // Bytes is a VPERM-like permute vector, except that -1 is used for
5129 // undefined bytes. Return true if it can be performed using VSLDB.
5130 // When returning true, set StartIndex to the shift amount and OpNo0
5131 // and OpNo1 to the VPERM operands that should be used as the first
5132 // and second shift operand respectively.
isShlDoublePermute(const SmallVectorImpl<int> & Bytes,unsigned & StartIndex,unsigned & OpNo0,unsigned & OpNo1)5133 static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5134 unsigned &StartIndex, unsigned &OpNo0,
5135 unsigned &OpNo1) {
5136 int OpNos[] = { -1, -1 };
5137 int Shift = -1;
5138 for (unsigned I = 0; I < 16; ++I) {
5139 int Index = Bytes[I];
5140 if (Index >= 0) {
5141 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5142 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5143 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5144 if (Shift < 0)
5145 Shift = ExpectedShift;
5146 else if (Shift != ExpectedShift)
5147 return false;
5148 // Make sure that the operand mappings are consistent with previous
5149 // elements.
5150 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5151 return false;
5152 OpNos[ModelOpNo] = RealOpNo;
5153 }
5154 }
5155 StartIndex = Shift;
5156 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5157 }
5158
5159 // Create a node that performs P on operands Op0 and Op1, casting the
5160 // operands to the appropriate type. The type of the result is determined by P.
getPermuteNode(SelectionDAG & DAG,const SDLoc & DL,const Permute & P,SDValue Op0,SDValue Op1)5161 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5162 const Permute &P, SDValue Op0, SDValue Op1) {
5163 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5164 // elements of a PACK are twice as wide as the outputs.
5165 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
5166 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
5167 P.Operand);
5168 // Cast both operands to the appropriate type.
5169 MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8),
5170 SystemZ::VectorBytes / InBytes);
5171 Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
5172 Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
5173 SDValue Op;
5174 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
5175 SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
5176 Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
5177 } else if (P.Opcode == SystemZISD::PACK) {
5178 MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
5179 SystemZ::VectorBytes / P.Operand);
5180 Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
5181 } else {
5182 Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
5183 }
5184 return Op;
5185 }
5186
isZeroVector(SDValue N)5187 static bool isZeroVector(SDValue N) {
5188 if (N->getOpcode() == ISD::BITCAST)
5189 N = N->getOperand(0);
5190 if (N->getOpcode() == ISD::SPLAT_VECTOR)
5191 if (auto *Op = dyn_cast<ConstantSDNode>(N->getOperand(0)))
5192 return Op->getZExtValue() == 0;
5193 return ISD::isBuildVectorAllZeros(N.getNode());
5194 }
5195
5196 // Return the index of the zero/undef vector, or UINT32_MAX if not found.
findZeroVectorIdx(SDValue * Ops,unsigned Num)5197 static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
5198 for (unsigned I = 0; I < Num ; I++)
5199 if (isZeroVector(Ops[I]))
5200 return I;
5201 return UINT32_MAX;
5202 }
5203
5204 // Bytes is a VPERM-like permute vector, except that -1 is used for
5205 // undefined bytes. Implement it on operands Ops[0] and Ops[1] using
5206 // VSLDB or VPERM.
getGeneralPermuteNode(SelectionDAG & DAG,const SDLoc & DL,SDValue * Ops,const SmallVectorImpl<int> & Bytes)5207 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5208 SDValue *Ops,
5209 const SmallVectorImpl<int> &Bytes) {
5210 for (unsigned I = 0; I < 2; ++I)
5211 Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
5212
5213 // First see whether VSLDB can be used.
5214 unsigned StartIndex, OpNo0, OpNo1;
5215 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
5216 return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
5217 Ops[OpNo1],
5218 DAG.getTargetConstant(StartIndex, DL, MVT::i32));
5219
5220 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
5221 // eliminate a zero vector by reusing any zero index in the permute vector.
5222 unsigned ZeroVecIdx = findZeroVectorIdx(&Ops[0], 2);
5223 if (ZeroVecIdx != UINT32_MAX) {
5224 bool MaskFirst = true;
5225 int ZeroIdx = -1;
5226 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5227 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5228 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5229 if (OpNo == ZeroVecIdx && I == 0) {
5230 // If the first byte is zero, use mask as first operand.
5231 ZeroIdx = 0;
5232 break;
5233 }
5234 if (OpNo != ZeroVecIdx && Byte == 0) {
5235 // If mask contains a zero, use it by placing that vector first.
5236 ZeroIdx = I + SystemZ::VectorBytes;
5237 MaskFirst = false;
5238 break;
5239 }
5240 }
5241 if (ZeroIdx != -1) {
5242 SDValue IndexNodes[SystemZ::VectorBytes];
5243 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5244 if (Bytes[I] >= 0) {
5245 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5246 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5247 if (OpNo == ZeroVecIdx)
5248 IndexNodes[I] = DAG.getConstant(ZeroIdx, DL, MVT::i32);
5249 else {
5250 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
5251 IndexNodes[I] = DAG.getConstant(BIdx, DL, MVT::i32);
5252 }
5253 } else
5254 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
5255 }
5256 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
5257 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
5258 if (MaskFirst)
5259 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Mask, Src,
5260 Mask);
5261 else
5262 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Src, Mask,
5263 Mask);
5264 }
5265 }
5266
5267 SDValue IndexNodes[SystemZ::VectorBytes];
5268 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
5269 if (Bytes[I] >= 0)
5270 IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
5271 else
5272 IndexNodes[I] = DAG.getUNDEF(MVT::i32);
5273 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
5274 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0],
5275 (!Ops[1].isUndef() ? Ops[1] : Ops[0]), Op2);
5276 }
5277
5278 namespace {
5279 // Describes a general N-operand vector shuffle.
5280 struct GeneralShuffle {
GeneralShuffle__anon968932af0511::GeneralShuffle5281 GeneralShuffle(EVT vt) : VT(vt), UnpackFromEltSize(UINT_MAX) {}
5282 void addUndef();
5283 bool add(SDValue, unsigned);
5284 SDValue getNode(SelectionDAG &, const SDLoc &);
5285 void tryPrepareForUnpack();
unpackWasPrepared__anon968932af0511::GeneralShuffle5286 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
5287 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
5288
5289 // The operands of the shuffle.
5290 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
5291
5292 // Index I is -1 if byte I of the result is undefined. Otherwise the
5293 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
5294 // Bytes[I] / SystemZ::VectorBytes.
5295 SmallVector<int, SystemZ::VectorBytes> Bytes;
5296
5297 // The type of the shuffle result.
5298 EVT VT;
5299
5300 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
5301 unsigned UnpackFromEltSize;
5302 };
5303 }
5304
5305 // Add an extra undefined element to the shuffle.
addUndef()5306 void GeneralShuffle::addUndef() {
5307 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5308 for (unsigned I = 0; I < BytesPerElement; ++I)
5309 Bytes.push_back(-1);
5310 }
5311
5312 // Add an extra element to the shuffle, taking it from element Elem of Op.
5313 // A null Op indicates a vector input whose value will be calculated later;
5314 // there is at most one such input per shuffle and it always has the same
5315 // type as the result. Aborts and returns false if the source vector elements
5316 // of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
5317 // LLVM they become implicitly extended, but this is rare and not optimized.
add(SDValue Op,unsigned Elem)5318 bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
5319 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5320
5321 // The source vector can have wider elements than the result,
5322 // either through an explicit TRUNCATE or because of type legalization.
5323 // We want the least significant part.
5324 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
5325 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
5326
5327 // Return false if the source elements are smaller than their destination
5328 // elements.
5329 if (FromBytesPerElement < BytesPerElement)
5330 return false;
5331
5332 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
5333 (FromBytesPerElement - BytesPerElement));
5334
5335 // Look through things like shuffles and bitcasts.
5336 while (Op.getNode()) {
5337 if (Op.getOpcode() == ISD::BITCAST)
5338 Op = Op.getOperand(0);
5339 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
5340 // See whether the bytes we need come from a contiguous part of one
5341 // operand.
5342 SmallVector<int, SystemZ::VectorBytes> OpBytes;
5343 if (!getVPermMask(Op, OpBytes))
5344 break;
5345 int NewByte;
5346 if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte))
5347 break;
5348 if (NewByte < 0) {
5349 addUndef();
5350 return true;
5351 }
5352 Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
5353 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
5354 } else if (Op.isUndef()) {
5355 addUndef();
5356 return true;
5357 } else
5358 break;
5359 }
5360
5361 // Make sure that the source of the extraction is in Ops.
5362 unsigned OpNo = 0;
5363 for (; OpNo < Ops.size(); ++OpNo)
5364 if (Ops[OpNo] == Op)
5365 break;
5366 if (OpNo == Ops.size())
5367 Ops.push_back(Op);
5368
5369 // Add the element to Bytes.
5370 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
5371 for (unsigned I = 0; I < BytesPerElement; ++I)
5372 Bytes.push_back(Base + I);
5373
5374 return true;
5375 }
5376
5377 // Return SDNodes for the completed shuffle.
getNode(SelectionDAG & DAG,const SDLoc & DL)5378 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
5379 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
5380
5381 if (Ops.size() == 0)
5382 return DAG.getUNDEF(VT);
5383
5384 // Use a single unpack if possible as the last operation.
5385 tryPrepareForUnpack();
5386
5387 // Make sure that there are at least two shuffle operands.
5388 if (Ops.size() == 1)
5389 Ops.push_back(DAG.getUNDEF(MVT::v16i8));
5390
5391 // Create a tree of shuffles, deferring root node until after the loop.
5392 // Try to redistribute the undefined elements of non-root nodes so that
5393 // the non-root shuffles match something like a pack or merge, then adjust
5394 // the parent node's permute vector to compensate for the new order.
5395 // Among other things, this copes with vectors like <2 x i16> that were
5396 // padded with undefined elements during type legalization.
5397 //
5398 // In the best case this redistribution will lead to the whole tree
5399 // using packs and merges. It should rarely be a loss in other cases.
5400 unsigned Stride = 1;
5401 for (; Stride * 2 < Ops.size(); Stride *= 2) {
5402 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
5403 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
5404
5405 // Create a mask for just these two operands.
5406 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
5407 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
5408 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
5409 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
5410 if (OpNo == I)
5411 NewBytes[J] = Byte;
5412 else if (OpNo == I + Stride)
5413 NewBytes[J] = SystemZ::VectorBytes + Byte;
5414 else
5415 NewBytes[J] = -1;
5416 }
5417 // See if it would be better to reorganize NewMask to avoid using VPERM.
5418 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
5419 if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) {
5420 Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
5421 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
5422 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
5423 if (NewBytes[J] >= 0) {
5424 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
5425 "Invalid double permute");
5426 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
5427 } else
5428 assert(NewBytesMap[J] < 0 && "Invalid double permute");
5429 }
5430 } else {
5431 // Just use NewBytes on the operands.
5432 Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
5433 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
5434 if (NewBytes[J] >= 0)
5435 Bytes[J] = I * SystemZ::VectorBytes + J;
5436 }
5437 }
5438 }
5439
5440 // Now we just have 2 inputs. Put the second operand in Ops[1].
5441 if (Stride > 1) {
5442 Ops[1] = Ops[Stride];
5443 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
5444 if (Bytes[I] >= int(SystemZ::VectorBytes))
5445 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
5446 }
5447
5448 // Look for an instruction that can do the permute without resorting
5449 // to VPERM.
5450 unsigned OpNo0, OpNo1;
5451 SDValue Op;
5452 if (unpackWasPrepared() && Ops[1].isUndef())
5453 Op = Ops[0];
5454 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
5455 Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
5456 else
5457 Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
5458
5459 Op = insertUnpackIfPrepared(DAG, DL, Op);
5460
5461 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
5462 }
5463
5464 #ifndef NDEBUG
dumpBytes(const SmallVectorImpl<int> & Bytes,std::string Msg)5465 static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
5466 dbgs() << Msg.c_str() << " { ";
5467 for (unsigned i = 0; i < Bytes.size(); i++)
5468 dbgs() << Bytes[i] << " ";
5469 dbgs() << "}\n";
5470 }
5471 #endif
5472
5473 // If the Bytes vector matches an unpack operation, prepare to do the unpack
5474 // after all else by removing the zero vector and the effect of the unpack on
5475 // Bytes.
tryPrepareForUnpack()5476 void GeneralShuffle::tryPrepareForUnpack() {
5477 uint32_t ZeroVecOpNo = findZeroVectorIdx(&Ops[0], Ops.size());
5478 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
5479 return;
5480
5481 // Only do this if removing the zero vector reduces the depth, otherwise
5482 // the critical path will increase with the final unpack.
5483 if (Ops.size() > 2 &&
5484 Log2_32_Ceil(Ops.size()) == Log2_32_Ceil(Ops.size() - 1))
5485 return;
5486
5487 // Find an unpack that would allow removing the zero vector from Ops.
5488 UnpackFromEltSize = 1;
5489 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
5490 bool MatchUnpack = true;
5491 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
5492 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
5493 unsigned ToEltSize = UnpackFromEltSize * 2;
5494 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
5495 if (!IsZextByte)
5496 SrcBytes.push_back(Bytes[Elt]);
5497 if (Bytes[Elt] != -1) {
5498 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
5499 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
5500 MatchUnpack = false;
5501 break;
5502 }
5503 }
5504 }
5505 if (MatchUnpack) {
5506 if (Ops.size() == 2) {
5507 // Don't use unpack if a single source operand needs rearrangement.
5508 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++)
5509 if (SrcBytes[i] != -1 && SrcBytes[i] % 16 != int(i)) {
5510 UnpackFromEltSize = UINT_MAX;
5511 return;
5512 }
5513 }
5514 break;
5515 }
5516 }
5517 if (UnpackFromEltSize > 4)
5518 return;
5519
5520 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
5521 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
5522 << ".\n";
5523 dumpBytes(Bytes, "Original Bytes vector:"););
5524
5525 // Apply the unpack in reverse to the Bytes array.
5526 unsigned B = 0;
5527 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
5528 Elt += UnpackFromEltSize;
5529 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
5530 Bytes[B] = Bytes[Elt];
5531 }
5532 while (B < SystemZ::VectorBytes)
5533 Bytes[B++] = -1;
5534
5535 // Remove the zero vector from Ops
5536 Ops.erase(&Ops[ZeroVecOpNo]);
5537 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
5538 if (Bytes[I] >= 0) {
5539 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5540 if (OpNo > ZeroVecOpNo)
5541 Bytes[I] -= SystemZ::VectorBytes;
5542 }
5543
5544 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
5545 dbgs() << "\n";);
5546 }
5547
insertUnpackIfPrepared(SelectionDAG & DAG,const SDLoc & DL,SDValue Op)5548 SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
5549 const SDLoc &DL,
5550 SDValue Op) {
5551 if (!unpackWasPrepared())
5552 return Op;
5553 unsigned InBits = UnpackFromEltSize * 8;
5554 EVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBits),
5555 SystemZ::VectorBits / InBits);
5556 SDValue PackedOp = DAG.getNode(ISD::BITCAST, DL, InVT, Op);
5557 unsigned OutBits = InBits * 2;
5558 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(OutBits),
5559 SystemZ::VectorBits / OutBits);
5560 return DAG.getNode(SystemZISD::UNPACKL_HIGH, DL, OutVT, PackedOp);
5561 }
5562
5563 // Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
isScalarToVector(SDValue Op)5564 static bool isScalarToVector(SDValue Op) {
5565 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
5566 if (!Op.getOperand(I).isUndef())
5567 return false;
5568 return true;
5569 }
5570
5571 // Return a vector of type VT that contains Value in the first element.
5572 // The other elements don't matter.
buildScalarToVector(SelectionDAG & DAG,const SDLoc & DL,EVT VT,SDValue Value)5573 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5574 SDValue Value) {
5575 // If we have a constant, replicate it to all elements and let the
5576 // BUILD_VECTOR lowering take care of it.
5577 if (Value.getOpcode() == ISD::Constant ||
5578 Value.getOpcode() == ISD::ConstantFP) {
5579 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
5580 return DAG.getBuildVector(VT, DL, Ops);
5581 }
5582 if (Value.isUndef())
5583 return DAG.getUNDEF(VT);
5584 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
5585 }
5586
5587 // Return a vector of type VT in which Op0 is in element 0 and Op1 is in
5588 // element 1. Used for cases in which replication is cheap.
buildMergeScalars(SelectionDAG & DAG,const SDLoc & DL,EVT VT,SDValue Op0,SDValue Op1)5589 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5590 SDValue Op0, SDValue Op1) {
5591 if (Op0.isUndef()) {
5592 if (Op1.isUndef())
5593 return DAG.getUNDEF(VT);
5594 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
5595 }
5596 if (Op1.isUndef())
5597 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
5598 return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
5599 buildScalarToVector(DAG, DL, VT, Op0),
5600 buildScalarToVector(DAG, DL, VT, Op1));
5601 }
5602
5603 // Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
5604 // vector for them.
joinDwords(SelectionDAG & DAG,const SDLoc & DL,SDValue Op0,SDValue Op1)5605 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
5606 SDValue Op1) {
5607 if (Op0.isUndef() && Op1.isUndef())
5608 return DAG.getUNDEF(MVT::v2i64);
5609 // If one of the two inputs is undefined then replicate the other one,
5610 // in order to avoid using another register unnecessarily.
5611 if (Op0.isUndef())
5612 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
5613 else if (Op1.isUndef())
5614 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
5615 else {
5616 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
5617 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
5618 }
5619 return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
5620 }
5621
5622 // If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
5623 // better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
5624 // the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
5625 // would benefit from this representation and return it if so.
tryBuildVectorShuffle(SelectionDAG & DAG,BuildVectorSDNode * BVN)5626 static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
5627 BuildVectorSDNode *BVN) {
5628 EVT VT = BVN->getValueType(0);
5629 unsigned NumElements = VT.getVectorNumElements();
5630
5631 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
5632 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
5633 // need a BUILD_VECTOR, add an additional placeholder operand for that
5634 // BUILD_VECTOR and store its operands in ResidueOps.
5635 GeneralShuffle GS(VT);
5636 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
5637 bool FoundOne = false;
5638 for (unsigned I = 0; I < NumElements; ++I) {
5639 SDValue Op = BVN->getOperand(I);
5640 if (Op.getOpcode() == ISD::TRUNCATE)
5641 Op = Op.getOperand(0);
5642 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5643 Op.getOperand(1).getOpcode() == ISD::Constant) {
5644 unsigned Elem = Op.getConstantOperandVal(1);
5645 if (!GS.add(Op.getOperand(0), Elem))
5646 return SDValue();
5647 FoundOne = true;
5648 } else if (Op.isUndef()) {
5649 GS.addUndef();
5650 } else {
5651 if (!GS.add(SDValue(), ResidueOps.size()))
5652 return SDValue();
5653 ResidueOps.push_back(BVN->getOperand(I));
5654 }
5655 }
5656
5657 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
5658 if (!FoundOne)
5659 return SDValue();
5660
5661 // Create the BUILD_VECTOR for the remaining elements, if any.
5662 if (!ResidueOps.empty()) {
5663 while (ResidueOps.size() < NumElements)
5664 ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType()));
5665 for (auto &Op : GS.Ops) {
5666 if (!Op.getNode()) {
5667 Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps);
5668 break;
5669 }
5670 }
5671 }
5672 return GS.getNode(DAG, SDLoc(BVN));
5673 }
5674
isVectorElementLoad(SDValue Op) const5675 bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
5676 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Op)->isUnindexed())
5677 return true;
5678 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
5679 return true;
5680 return false;
5681 }
5682
5683 // Combine GPR scalar values Elems into a vector of type VT.
5684 SDValue
buildVector(SelectionDAG & DAG,const SDLoc & DL,EVT VT,SmallVectorImpl<SDValue> & Elems) const5685 SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
5686 SmallVectorImpl<SDValue> &Elems) const {
5687 // See whether there is a single replicated value.
5688 SDValue Single;
5689 unsigned int NumElements = Elems.size();
5690 unsigned int Count = 0;
5691 for (auto Elem : Elems) {
5692 if (!Elem.isUndef()) {
5693 if (!Single.getNode())
5694 Single = Elem;
5695 else if (Elem != Single) {
5696 Single = SDValue();
5697 break;
5698 }
5699 Count += 1;
5700 }
5701 }
5702 // There are three cases here:
5703 //
5704 // - if the only defined element is a loaded one, the best sequence
5705 // is a replicating load.
5706 //
5707 // - otherwise, if the only defined element is an i64 value, we will
5708 // end up with the same VLVGP sequence regardless of whether we short-cut
5709 // for replication or fall through to the later code.
5710 //
5711 // - otherwise, if the only defined element is an i32 or smaller value,
5712 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
5713 // This is only a win if the single defined element is used more than once.
5714 // In other cases we're better off using a single VLVGx.
5715 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Single)))
5716 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
5717
5718 // If all elements are loads, use VLREP/VLEs (below).
5719 bool AllLoads = true;
5720 for (auto Elem : Elems)
5721 if (!isVectorElementLoad(Elem)) {
5722 AllLoads = false;
5723 break;
5724 }
5725
5726 // The best way of building a v2i64 from two i64s is to use VLVGP.
5727 if (VT == MVT::v2i64 && !AllLoads)
5728 return joinDwords(DAG, DL, Elems[0], Elems[1]);
5729
5730 // Use a 64-bit merge high to combine two doubles.
5731 if (VT == MVT::v2f64 && !AllLoads)
5732 return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
5733
5734 // Build v4f32 values directly from the FPRs:
5735 //
5736 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
5737 // V V VMRHF
5738 // <ABxx> <CDxx>
5739 // V VMRHG
5740 // <ABCD>
5741 if (VT == MVT::v4f32 && !AllLoads) {
5742 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
5743 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
5744 // Avoid unnecessary undefs by reusing the other operand.
5745 if (Op01.isUndef())
5746 Op01 = Op23;
5747 else if (Op23.isUndef())
5748 Op23 = Op01;
5749 // Merging identical replications is a no-op.
5750 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
5751 return Op01;
5752 Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
5753 Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
5754 SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH,
5755 DL, MVT::v2i64, Op01, Op23);
5756 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
5757 }
5758
5759 // Collect the constant terms.
5760 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
5761 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
5762
5763 unsigned NumConstants = 0;
5764 for (unsigned I = 0; I < NumElements; ++I) {
5765 SDValue Elem = Elems[I];
5766 if (Elem.getOpcode() == ISD::Constant ||
5767 Elem.getOpcode() == ISD::ConstantFP) {
5768 NumConstants += 1;
5769 Constants[I] = Elem;
5770 Done[I] = true;
5771 }
5772 }
5773 // If there was at least one constant, fill in the other elements of
5774 // Constants with undefs to get a full vector constant and use that
5775 // as the starting point.
5776 SDValue Result;
5777 SDValue ReplicatedVal;
5778 if (NumConstants > 0) {
5779 for (unsigned I = 0; I < NumElements; ++I)
5780 if (!Constants[I].getNode())
5781 Constants[I] = DAG.getUNDEF(Elems[I].getValueType());
5782 Result = DAG.getBuildVector(VT, DL, Constants);
5783 } else {
5784 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
5785 // avoid a false dependency on any previous contents of the vector
5786 // register.
5787
5788 // Use a VLREP if at least one element is a load. Make sure to replicate
5789 // the load with the most elements having its value.
5790 std::map<const SDNode*, unsigned> UseCounts;
5791 SDNode *LoadMaxUses = nullptr;
5792 for (unsigned I = 0; I < NumElements; ++I)
5793 if (isVectorElementLoad(Elems[I])) {
5794 SDNode *Ld = Elems[I].getNode();
5795 UseCounts[Ld]++;
5796 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < UseCounts[Ld])
5797 LoadMaxUses = Ld;
5798 }
5799 if (LoadMaxUses != nullptr) {
5800 ReplicatedVal = SDValue(LoadMaxUses, 0);
5801 Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, ReplicatedVal);
5802 } else {
5803 // Try to use VLVGP.
5804 unsigned I1 = NumElements / 2 - 1;
5805 unsigned I2 = NumElements - 1;
5806 bool Def1 = !Elems[I1].isUndef();
5807 bool Def2 = !Elems[I2].isUndef();
5808 if (Def1 || Def2) {
5809 SDValue Elem1 = Elems[Def1 ? I1 : I2];
5810 SDValue Elem2 = Elems[Def2 ? I2 : I1];
5811 Result = DAG.getNode(ISD::BITCAST, DL, VT,
5812 joinDwords(DAG, DL, Elem1, Elem2));
5813 Done[I1] = true;
5814 Done[I2] = true;
5815 } else
5816 Result = DAG.getUNDEF(VT);
5817 }
5818 }
5819
5820 // Use VLVGx to insert the other elements.
5821 for (unsigned I = 0; I < NumElements; ++I)
5822 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
5823 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
5824 DAG.getConstant(I, DL, MVT::i32));
5825 return Result;
5826 }
5827
lowerBUILD_VECTOR(SDValue Op,SelectionDAG & DAG) const5828 SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
5829 SelectionDAG &DAG) const {
5830 auto *BVN = cast<BuildVectorSDNode>(Op.getNode());
5831 SDLoc DL(Op);
5832 EVT VT = Op.getValueType();
5833
5834 if (BVN->isConstant()) {
5835 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
5836 return Op;
5837
5838 // Fall back to loading it from memory.
5839 return SDValue();
5840 }
5841
5842 // See if we should use shuffles to construct the vector from other vectors.
5843 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
5844 return Res;
5845
5846 // Detect SCALAR_TO_VECTOR conversions.
5847 if (isOperationLegal(ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
5848 return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
5849
5850 // Otherwise use buildVector to build the vector up from GPRs.
5851 unsigned NumElements = Op.getNumOperands();
5852 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
5853 for (unsigned I = 0; I < NumElements; ++I)
5854 Ops[I] = Op.getOperand(I);
5855 return buildVector(DAG, DL, VT, Ops);
5856 }
5857
lowerVECTOR_SHUFFLE(SDValue Op,SelectionDAG & DAG) const5858 SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
5859 SelectionDAG &DAG) const {
5860 auto *VSN = cast<ShuffleVectorSDNode>(Op.getNode());
5861 SDLoc DL(Op);
5862 EVT VT = Op.getValueType();
5863 unsigned NumElements = VT.getVectorNumElements();
5864
5865 if (VSN->isSplat()) {
5866 SDValue Op0 = Op.getOperand(0);
5867 unsigned Index = VSN->getSplatIndex();
5868 assert(Index < VT.getVectorNumElements() &&
5869 "Splat index should be defined and in first operand");
5870 // See whether the value we're splatting is directly available as a scalar.
5871 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
5872 Op0.getOpcode() == ISD::BUILD_VECTOR)
5873 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
5874 // Otherwise keep it as a vector-to-vector operation.
5875 return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
5876 DAG.getTargetConstant(Index, DL, MVT::i32));
5877 }
5878
5879 GeneralShuffle GS(VT);
5880 for (unsigned I = 0; I < NumElements; ++I) {
5881 int Elt = VSN->getMaskElt(I);
5882 if (Elt < 0)
5883 GS.addUndef();
5884 else if (!GS.add(Op.getOperand(unsigned(Elt) / NumElements),
5885 unsigned(Elt) % NumElements))
5886 return SDValue();
5887 }
5888 return GS.getNode(DAG, SDLoc(VSN));
5889 }
5890
lowerSCALAR_TO_VECTOR(SDValue Op,SelectionDAG & DAG) const5891 SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
5892 SelectionDAG &DAG) const {
5893 SDLoc DL(Op);
5894 // Just insert the scalar into element 0 of an undefined vector.
5895 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
5896 Op.getValueType(), DAG.getUNDEF(Op.getValueType()),
5897 Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
5898 }
5899
lowerINSERT_VECTOR_ELT(SDValue Op,SelectionDAG & DAG) const5900 SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
5901 SelectionDAG &DAG) const {
5902 // Handle insertions of floating-point values.
5903 SDLoc DL(Op);
5904 SDValue Op0 = Op.getOperand(0);
5905 SDValue Op1 = Op.getOperand(1);
5906 SDValue Op2 = Op.getOperand(2);
5907 EVT VT = Op.getValueType();
5908
5909 // Insertions into constant indices of a v2f64 can be done using VPDI.
5910 // However, if the inserted value is a bitcast or a constant then it's
5911 // better to use GPRs, as below.
5912 if (VT == MVT::v2f64 &&
5913 Op1.getOpcode() != ISD::BITCAST &&
5914 Op1.getOpcode() != ISD::ConstantFP &&
5915 Op2.getOpcode() == ISD::Constant) {
5916 uint64_t Index = Op2->getAsZExtVal();
5917 unsigned Mask = VT.getVectorNumElements() - 1;
5918 if (Index <= Mask)
5919 return Op;
5920 }
5921
5922 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
5923 MVT IntVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
5924 MVT IntVecVT = MVT::getVectorVT(IntVT, VT.getVectorNumElements());
5925 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
5926 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0),
5927 DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2);
5928 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
5929 }
5930
5931 SDValue
lowerEXTRACT_VECTOR_ELT(SDValue Op,SelectionDAG & DAG) const5932 SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
5933 SelectionDAG &DAG) const {
5934 // Handle extractions of floating-point values.
5935 SDLoc DL(Op);
5936 SDValue Op0 = Op.getOperand(0);
5937 SDValue Op1 = Op.getOperand(1);
5938 EVT VT = Op.getValueType();
5939 EVT VecVT = Op0.getValueType();
5940
5941 // Extractions of constant indices can be done directly.
5942 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Op1)) {
5943 uint64_t Index = CIndexN->getZExtValue();
5944 unsigned Mask = VecVT.getVectorNumElements() - 1;
5945 if (Index <= Mask)
5946 return Op;
5947 }
5948
5949 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
5950 MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
5951 MVT IntVecVT = MVT::getVectorVT(IntVT, VecVT.getVectorNumElements());
5952 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
5953 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
5954 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
5955 }
5956
5957 SDValue SystemZTargetLowering::
lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op,SelectionDAG & DAG) const5958 lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
5959 SDValue PackedOp = Op.getOperand(0);
5960 EVT OutVT = Op.getValueType();
5961 EVT InVT = PackedOp.getValueType();
5962 unsigned ToBits = OutVT.getScalarSizeInBits();
5963 unsigned FromBits = InVT.getScalarSizeInBits();
5964 do {
5965 FromBits *= 2;
5966 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits),
5967 SystemZ::VectorBits / FromBits);
5968 PackedOp =
5969 DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(PackedOp), OutVT, PackedOp);
5970 } while (FromBits != ToBits);
5971 return PackedOp;
5972 }
5973
5974 // Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
5975 SDValue SystemZTargetLowering::
lowerZERO_EXTEND_VECTOR_INREG(SDValue Op,SelectionDAG & DAG) const5976 lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
5977 SDValue PackedOp = Op.getOperand(0);
5978 SDLoc DL(Op);
5979 EVT OutVT = Op.getValueType();
5980 EVT InVT = PackedOp.getValueType();
5981 unsigned InNumElts = InVT.getVectorNumElements();
5982 unsigned OutNumElts = OutVT.getVectorNumElements();
5983 unsigned NumInPerOut = InNumElts / OutNumElts;
5984
5985 SDValue ZeroVec =
5986 DAG.getSplatVector(InVT, DL, DAG.getConstant(0, DL, InVT.getScalarType()));
5987
5988 SmallVector<int, 16> Mask(InNumElts);
5989 unsigned ZeroVecElt = InNumElts;
5990 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
5991 unsigned MaskElt = PackedElt * NumInPerOut;
5992 unsigned End = MaskElt + NumInPerOut - 1;
5993 for (; MaskElt < End; MaskElt++)
5994 Mask[MaskElt] = ZeroVecElt++;
5995 Mask[MaskElt] = PackedElt;
5996 }
5997 SDValue Shuf = DAG.getVectorShuffle(InVT, DL, PackedOp, ZeroVec, Mask);
5998 return DAG.getNode(ISD::BITCAST, DL, OutVT, Shuf);
5999 }
6000
lowerShift(SDValue Op,SelectionDAG & DAG,unsigned ByScalar) const6001 SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6002 unsigned ByScalar) const {
6003 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6004 SDValue Op0 = Op.getOperand(0);
6005 SDValue Op1 = Op.getOperand(1);
6006 SDLoc DL(Op);
6007 EVT VT = Op.getValueType();
6008 unsigned ElemBitSize = VT.getScalarSizeInBits();
6009
6010 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6011 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) {
6012 APInt SplatBits, SplatUndef;
6013 unsigned SplatBitSize;
6014 bool HasAnyUndefs;
6015 // Check for constant splats. Use ElemBitSize as the minimum element
6016 // width and reject splats that need wider elements.
6017 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6018 ElemBitSize, true) &&
6019 SplatBitSize == ElemBitSize) {
6020 SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
6021 DL, MVT::i32);
6022 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6023 }
6024 // Check for variable splats.
6025 BitVector UndefElements;
6026 SDValue Splat = BVN->getSplatValue(&UndefElements);
6027 if (Splat) {
6028 // Since i32 is the smallest legal type, we either need a no-op
6029 // or a truncation.
6030 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
6031 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6032 }
6033 }
6034
6035 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
6036 // and the shift amount is directly available in a GPR.
6037 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) {
6038 if (VSN->isSplat()) {
6039 SDValue VSNOp0 = VSN->getOperand(0);
6040 unsigned Index = VSN->getSplatIndex();
6041 assert(Index < VT.getVectorNumElements() &&
6042 "Splat index should be defined and in first operand");
6043 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6044 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
6045 // Since i32 is the smallest legal type, we either need a no-op
6046 // or a truncation.
6047 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
6048 VSNOp0.getOperand(Index));
6049 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
6050 }
6051 }
6052 }
6053
6054 // Otherwise just treat the current form as legal.
6055 return Op;
6056 }
6057
lowerIS_FPCLASS(SDValue Op,SelectionDAG & DAG) const6058 SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
6059 SelectionDAG &DAG) const {
6060 SDLoc DL(Op);
6061 MVT ResultVT = Op.getSimpleValueType();
6062 SDValue Arg = Op.getOperand(0);
6063 unsigned Check = Op.getConstantOperandVal(1);
6064
6065 unsigned TDCMask = 0;
6066 if (Check & fcSNan)
6067 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
6068 if (Check & fcQNan)
6069 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
6070 if (Check & fcPosInf)
6071 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
6072 if (Check & fcNegInf)
6073 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
6074 if (Check & fcPosNormal)
6075 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
6076 if (Check & fcNegNormal)
6077 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
6078 if (Check & fcPosSubnormal)
6079 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
6080 if (Check & fcNegSubnormal)
6081 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
6082 if (Check & fcPosZero)
6083 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
6084 if (Check & fcNegZero)
6085 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
6086 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, MVT::i64);
6087
6088 SDValue Intr = DAG.getNode(SystemZISD::TDC, DL, ResultVT, Arg, TDCMaskV);
6089 return getCCResult(DAG, Intr);
6090 }
6091
LowerOperation(SDValue Op,SelectionDAG & DAG) const6092 SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
6093 SelectionDAG &DAG) const {
6094 switch (Op.getOpcode()) {
6095 case ISD::FRAMEADDR:
6096 return lowerFRAMEADDR(Op, DAG);
6097 case ISD::RETURNADDR:
6098 return lowerRETURNADDR(Op, DAG);
6099 case ISD::BR_CC:
6100 return lowerBR_CC(Op, DAG);
6101 case ISD::SELECT_CC:
6102 return lowerSELECT_CC(Op, DAG);
6103 case ISD::SETCC:
6104 return lowerSETCC(Op, DAG);
6105 case ISD::STRICT_FSETCC:
6106 return lowerSTRICT_FSETCC(Op, DAG, false);
6107 case ISD::STRICT_FSETCCS:
6108 return lowerSTRICT_FSETCC(Op, DAG, true);
6109 case ISD::GlobalAddress:
6110 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG);
6111 case ISD::GlobalTLSAddress:
6112 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG);
6113 case ISD::BlockAddress:
6114 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG);
6115 case ISD::JumpTable:
6116 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG);
6117 case ISD::ConstantPool:
6118 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG);
6119 case ISD::BITCAST:
6120 return lowerBITCAST(Op, DAG);
6121 case ISD::VASTART:
6122 return lowerVASTART(Op, DAG);
6123 case ISD::VACOPY:
6124 return lowerVACOPY(Op, DAG);
6125 case ISD::DYNAMIC_STACKALLOC:
6126 return lowerDYNAMIC_STACKALLOC(Op, DAG);
6127 case ISD::GET_DYNAMIC_AREA_OFFSET:
6128 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
6129 case ISD::SMUL_LOHI:
6130 return lowerSMUL_LOHI(Op, DAG);
6131 case ISD::UMUL_LOHI:
6132 return lowerUMUL_LOHI(Op, DAG);
6133 case ISD::SDIVREM:
6134 return lowerSDIVREM(Op, DAG);
6135 case ISD::UDIVREM:
6136 return lowerUDIVREM(Op, DAG);
6137 case ISD::SADDO:
6138 case ISD::SSUBO:
6139 case ISD::UADDO:
6140 case ISD::USUBO:
6141 return lowerXALUO(Op, DAG);
6142 case ISD::UADDO_CARRY:
6143 case ISD::USUBO_CARRY:
6144 return lowerUADDSUBO_CARRY(Op, DAG);
6145 case ISD::OR:
6146 return lowerOR(Op, DAG);
6147 case ISD::CTPOP:
6148 return lowerCTPOP(Op, DAG);
6149 case ISD::ATOMIC_FENCE:
6150 return lowerATOMIC_FENCE(Op, DAG);
6151 case ISD::ATOMIC_SWAP:
6152 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
6153 case ISD::ATOMIC_STORE:
6154 return lowerATOMIC_STORE(Op, DAG);
6155 case ISD::ATOMIC_LOAD:
6156 return lowerATOMIC_LOAD(Op, DAG);
6157 case ISD::ATOMIC_LOAD_ADD:
6158 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
6159 case ISD::ATOMIC_LOAD_SUB:
6160 return lowerATOMIC_LOAD_SUB(Op, DAG);
6161 case ISD::ATOMIC_LOAD_AND:
6162 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND);
6163 case ISD::ATOMIC_LOAD_OR:
6164 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR);
6165 case ISD::ATOMIC_LOAD_XOR:
6166 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR);
6167 case ISD::ATOMIC_LOAD_NAND:
6168 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND);
6169 case ISD::ATOMIC_LOAD_MIN:
6170 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN);
6171 case ISD::ATOMIC_LOAD_MAX:
6172 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX);
6173 case ISD::ATOMIC_LOAD_UMIN:
6174 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
6175 case ISD::ATOMIC_LOAD_UMAX:
6176 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
6177 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
6178 return lowerATOMIC_CMP_SWAP(Op, DAG);
6179 case ISD::STACKSAVE:
6180 return lowerSTACKSAVE(Op, DAG);
6181 case ISD::STACKRESTORE:
6182 return lowerSTACKRESTORE(Op, DAG);
6183 case ISD::PREFETCH:
6184 return lowerPREFETCH(Op, DAG);
6185 case ISD::INTRINSIC_W_CHAIN:
6186 return lowerINTRINSIC_W_CHAIN(Op, DAG);
6187 case ISD::INTRINSIC_WO_CHAIN:
6188 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
6189 case ISD::BUILD_VECTOR:
6190 return lowerBUILD_VECTOR(Op, DAG);
6191 case ISD::VECTOR_SHUFFLE:
6192 return lowerVECTOR_SHUFFLE(Op, DAG);
6193 case ISD::SCALAR_TO_VECTOR:
6194 return lowerSCALAR_TO_VECTOR(Op, DAG);
6195 case ISD::INSERT_VECTOR_ELT:
6196 return lowerINSERT_VECTOR_ELT(Op, DAG);
6197 case ISD::EXTRACT_VECTOR_ELT:
6198 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
6199 case ISD::SIGN_EXTEND_VECTOR_INREG:
6200 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
6201 case ISD::ZERO_EXTEND_VECTOR_INREG:
6202 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
6203 case ISD::SHL:
6204 return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
6205 case ISD::SRL:
6206 return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR);
6207 case ISD::SRA:
6208 return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR);
6209 case ISD::ROTL:
6210 return lowerShift(Op, DAG, SystemZISD::VROTL_BY_SCALAR);
6211 case ISD::IS_FPCLASS:
6212 return lowerIS_FPCLASS(Op, DAG);
6213 case ISD::GET_ROUNDING:
6214 return lowerGET_ROUNDING(Op, DAG);
6215 default:
6216 llvm_unreachable("Unexpected node to lower");
6217 }
6218 }
6219
6220 // Lower operations with invalid operand or result types (currently used
6221 // only for 128-bit integer types).
6222 void
LowerOperationWrapper(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const6223 SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
6224 SmallVectorImpl<SDValue> &Results,
6225 SelectionDAG &DAG) const {
6226 switch (N->getOpcode()) {
6227 case ISD::ATOMIC_LOAD: {
6228 SDLoc DL(N);
6229 SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other);
6230 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
6231 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
6232 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_LOAD_128,
6233 DL, Tys, Ops, MVT::i128, MMO);
6234 Results.push_back(lowerGR128ToI128(DAG, Res));
6235 Results.push_back(Res.getValue(1));
6236 break;
6237 }
6238 case ISD::ATOMIC_STORE: {
6239 SDLoc DL(N);
6240 SDVTList Tys = DAG.getVTList(MVT::Other);
6241 SDValue Ops[] = {N->getOperand(0), lowerI128ToGR128(DAG, N->getOperand(1)),
6242 N->getOperand(2)};
6243 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
6244 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
6245 DL, Tys, Ops, MVT::i128, MMO);
6246 // We have to enforce sequential consistency by performing a
6247 // serialization operation after the store.
6248 if (cast<AtomicSDNode>(N)->getSuccessOrdering() ==
6249 AtomicOrdering::SequentiallyConsistent)
6250 Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
6251 MVT::Other, Res), 0);
6252 Results.push_back(Res);
6253 break;
6254 }
6255 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
6256 SDLoc DL(N);
6257 SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other);
6258 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
6259 lowerI128ToGR128(DAG, N->getOperand(2)),
6260 lowerI128ToGR128(DAG, N->getOperand(3)) };
6261 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
6262 SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
6263 DL, Tys, Ops, MVT::i128, MMO);
6264 SDValue Success = emitSETCC(DAG, DL, Res.getValue(1),
6265 SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
6266 Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
6267 Results.push_back(lowerGR128ToI128(DAG, Res));
6268 Results.push_back(Success);
6269 Results.push_back(Res.getValue(2));
6270 break;
6271 }
6272 case ISD::BITCAST: {
6273 SDValue Src = N->getOperand(0);
6274 if (N->getValueType(0) == MVT::i128 && Src.getValueType() == MVT::f128 &&
6275 !useSoftFloat()) {
6276 SDLoc DL(N);
6277 SDValue Lo, Hi;
6278 if (getRepRegClassFor(MVT::f128) == &SystemZ::VR128BitRegClass) {
6279 SDValue VecBC = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Src);
6280 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, VecBC,
6281 DAG.getConstant(1, DL, MVT::i32));
6282 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, VecBC,
6283 DAG.getConstant(0, DL, MVT::i32));
6284 } else {
6285 assert(getRepRegClassFor(MVT::f128) == &SystemZ::FP128BitRegClass &&
6286 "Unrecognized register class for f128.");
6287 SDValue LoFP = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
6288 DL, MVT::f64, Src);
6289 SDValue HiFP = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
6290 DL, MVT::f64, Src);
6291 Lo = DAG.getNode(ISD::BITCAST, DL, MVT::i64, LoFP);
6292 Hi = DAG.getNode(ISD::BITCAST, DL, MVT::i64, HiFP);
6293 }
6294 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi));
6295 }
6296 break;
6297 }
6298 default:
6299 llvm_unreachable("Unexpected node to lower");
6300 }
6301 }
6302
6303 void
ReplaceNodeResults(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const6304 SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
6305 SmallVectorImpl<SDValue> &Results,
6306 SelectionDAG &DAG) const {
6307 return LowerOperationWrapper(N, Results, DAG);
6308 }
6309
getTargetNodeName(unsigned Opcode) const6310 const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
6311 #define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME
6312 switch ((SystemZISD::NodeType)Opcode) {
6313 case SystemZISD::FIRST_NUMBER: break;
6314 OPCODE(RET_GLUE);
6315 OPCODE(CALL);
6316 OPCODE(SIBCALL);
6317 OPCODE(TLS_GDCALL);
6318 OPCODE(TLS_LDCALL);
6319 OPCODE(PCREL_WRAPPER);
6320 OPCODE(PCREL_OFFSET);
6321 OPCODE(ICMP);
6322 OPCODE(FCMP);
6323 OPCODE(STRICT_FCMP);
6324 OPCODE(STRICT_FCMPS);
6325 OPCODE(TM);
6326 OPCODE(BR_CCMASK);
6327 OPCODE(SELECT_CCMASK);
6328 OPCODE(ADJDYNALLOC);
6329 OPCODE(PROBED_ALLOCA);
6330 OPCODE(POPCNT);
6331 OPCODE(SMUL_LOHI);
6332 OPCODE(UMUL_LOHI);
6333 OPCODE(SDIVREM);
6334 OPCODE(UDIVREM);
6335 OPCODE(SADDO);
6336 OPCODE(SSUBO);
6337 OPCODE(UADDO);
6338 OPCODE(USUBO);
6339 OPCODE(ADDCARRY);
6340 OPCODE(SUBCARRY);
6341 OPCODE(GET_CCMASK);
6342 OPCODE(MVC);
6343 OPCODE(NC);
6344 OPCODE(OC);
6345 OPCODE(XC);
6346 OPCODE(CLC);
6347 OPCODE(MEMSET_MVC);
6348 OPCODE(STPCPY);
6349 OPCODE(STRCMP);
6350 OPCODE(SEARCH_STRING);
6351 OPCODE(IPM);
6352 OPCODE(TBEGIN);
6353 OPCODE(TBEGIN_NOFLOAT);
6354 OPCODE(TEND);
6355 OPCODE(BYTE_MASK);
6356 OPCODE(ROTATE_MASK);
6357 OPCODE(REPLICATE);
6358 OPCODE(JOIN_DWORDS);
6359 OPCODE(SPLAT);
6360 OPCODE(MERGE_HIGH);
6361 OPCODE(MERGE_LOW);
6362 OPCODE(SHL_DOUBLE);
6363 OPCODE(PERMUTE_DWORDS);
6364 OPCODE(PERMUTE);
6365 OPCODE(PACK);
6366 OPCODE(PACKS_CC);
6367 OPCODE(PACKLS_CC);
6368 OPCODE(UNPACK_HIGH);
6369 OPCODE(UNPACKL_HIGH);
6370 OPCODE(UNPACK_LOW);
6371 OPCODE(UNPACKL_LOW);
6372 OPCODE(VSHL_BY_SCALAR);
6373 OPCODE(VSRL_BY_SCALAR);
6374 OPCODE(VSRA_BY_SCALAR);
6375 OPCODE(VROTL_BY_SCALAR);
6376 OPCODE(VSUM);
6377 OPCODE(VACC);
6378 OPCODE(VSCBI);
6379 OPCODE(VAC);
6380 OPCODE(VSBI);
6381 OPCODE(VACCC);
6382 OPCODE(VSBCBI);
6383 OPCODE(VICMPE);
6384 OPCODE(VICMPH);
6385 OPCODE(VICMPHL);
6386 OPCODE(VICMPES);
6387 OPCODE(VICMPHS);
6388 OPCODE(VICMPHLS);
6389 OPCODE(VFCMPE);
6390 OPCODE(STRICT_VFCMPE);
6391 OPCODE(STRICT_VFCMPES);
6392 OPCODE(VFCMPH);
6393 OPCODE(STRICT_VFCMPH);
6394 OPCODE(STRICT_VFCMPHS);
6395 OPCODE(VFCMPHE);
6396 OPCODE(STRICT_VFCMPHE);
6397 OPCODE(STRICT_VFCMPHES);
6398 OPCODE(VFCMPES);
6399 OPCODE(VFCMPHS);
6400 OPCODE(VFCMPHES);
6401 OPCODE(VFTCI);
6402 OPCODE(VEXTEND);
6403 OPCODE(STRICT_VEXTEND);
6404 OPCODE(VROUND);
6405 OPCODE(STRICT_VROUND);
6406 OPCODE(VTM);
6407 OPCODE(SCMP128HI);
6408 OPCODE(UCMP128HI);
6409 OPCODE(VFAE_CC);
6410 OPCODE(VFAEZ_CC);
6411 OPCODE(VFEE_CC);
6412 OPCODE(VFEEZ_CC);
6413 OPCODE(VFENE_CC);
6414 OPCODE(VFENEZ_CC);
6415 OPCODE(VISTR_CC);
6416 OPCODE(VSTRC_CC);
6417 OPCODE(VSTRCZ_CC);
6418 OPCODE(VSTRS_CC);
6419 OPCODE(VSTRSZ_CC);
6420 OPCODE(TDC);
6421 OPCODE(ATOMIC_SWAPW);
6422 OPCODE(ATOMIC_LOADW_ADD);
6423 OPCODE(ATOMIC_LOADW_SUB);
6424 OPCODE(ATOMIC_LOADW_AND);
6425 OPCODE(ATOMIC_LOADW_OR);
6426 OPCODE(ATOMIC_LOADW_XOR);
6427 OPCODE(ATOMIC_LOADW_NAND);
6428 OPCODE(ATOMIC_LOADW_MIN);
6429 OPCODE(ATOMIC_LOADW_MAX);
6430 OPCODE(ATOMIC_LOADW_UMIN);
6431 OPCODE(ATOMIC_LOADW_UMAX);
6432 OPCODE(ATOMIC_CMP_SWAPW);
6433 OPCODE(ATOMIC_CMP_SWAP);
6434 OPCODE(ATOMIC_LOAD_128);
6435 OPCODE(ATOMIC_STORE_128);
6436 OPCODE(ATOMIC_CMP_SWAP_128);
6437 OPCODE(LRV);
6438 OPCODE(STRV);
6439 OPCODE(VLER);
6440 OPCODE(VSTER);
6441 OPCODE(PREFETCH);
6442 OPCODE(ADA_ENTRY);
6443 }
6444 return nullptr;
6445 #undef OPCODE
6446 }
6447
6448 // Return true if VT is a vector whose elements are a whole number of bytes
6449 // in width. Also check for presence of vector support.
canTreatAsByteVector(EVT VT) const6450 bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
6451 if (!Subtarget.hasVector())
6452 return false;
6453
6454 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
6455 }
6456
6457 // Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
6458 // producing a result of type ResVT. Op is a possibly bitcast version
6459 // of the input vector and Index is the index (based on type VecVT) that
6460 // should be extracted. Return the new extraction if a simplification
6461 // was possible or if Force is true.
combineExtract(const SDLoc & DL,EVT ResVT,EVT VecVT,SDValue Op,unsigned Index,DAGCombinerInfo & DCI,bool Force) const6462 SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
6463 EVT VecVT, SDValue Op,
6464 unsigned Index,
6465 DAGCombinerInfo &DCI,
6466 bool Force) const {
6467 SelectionDAG &DAG = DCI.DAG;
6468
6469 // The number of bytes being extracted.
6470 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
6471
6472 for (;;) {
6473 unsigned Opcode = Op.getOpcode();
6474 if (Opcode == ISD::BITCAST)
6475 // Look through bitcasts.
6476 Op = Op.getOperand(0);
6477 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
6478 canTreatAsByteVector(Op.getValueType())) {
6479 // Get a VPERM-like permute mask and see whether the bytes covered
6480 // by the extracted element are a contiguous sequence from one
6481 // source operand.
6482 SmallVector<int, SystemZ::VectorBytes> Bytes;
6483 if (!getVPermMask(Op, Bytes))
6484 break;
6485 int First;
6486 if (!getShuffleInput(Bytes, Index * BytesPerElement,
6487 BytesPerElement, First))
6488 break;
6489 if (First < 0)
6490 return DAG.getUNDEF(ResVT);
6491 // Make sure the contiguous sequence starts at a multiple of the
6492 // original element size.
6493 unsigned Byte = unsigned(First) % Bytes.size();
6494 if (Byte % BytesPerElement != 0)
6495 break;
6496 // We can get the extracted value directly from an input.
6497 Index = Byte / BytesPerElement;
6498 Op = Op.getOperand(unsigned(First) / Bytes.size());
6499 Force = true;
6500 } else if (Opcode == ISD::BUILD_VECTOR &&
6501 canTreatAsByteVector(Op.getValueType())) {
6502 // We can only optimize this case if the BUILD_VECTOR elements are
6503 // at least as wide as the extracted value.
6504 EVT OpVT = Op.getValueType();
6505 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
6506 if (OpBytesPerElement < BytesPerElement)
6507 break;
6508 // Make sure that the least-significant bit of the extracted value
6509 // is the least significant bit of an input.
6510 unsigned End = (Index + 1) * BytesPerElement;
6511 if (End % OpBytesPerElement != 0)
6512 break;
6513 // We're extracting the low part of one operand of the BUILD_VECTOR.
6514 Op = Op.getOperand(End / OpBytesPerElement - 1);
6515 if (!Op.getValueType().isInteger()) {
6516 EVT VT = MVT::getIntegerVT(Op.getValueSizeInBits());
6517 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
6518 DCI.AddToWorklist(Op.getNode());
6519 }
6520 EVT VT = MVT::getIntegerVT(ResVT.getSizeInBits());
6521 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
6522 if (VT != ResVT) {
6523 DCI.AddToWorklist(Op.getNode());
6524 Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
6525 }
6526 return Op;
6527 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
6528 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
6529 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
6530 canTreatAsByteVector(Op.getValueType()) &&
6531 canTreatAsByteVector(Op.getOperand(0).getValueType())) {
6532 // Make sure that only the unextended bits are significant.
6533 EVT ExtVT = Op.getValueType();
6534 EVT OpVT = Op.getOperand(0).getValueType();
6535 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
6536 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
6537 unsigned Byte = Index * BytesPerElement;
6538 unsigned SubByte = Byte % ExtBytesPerElement;
6539 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
6540 if (SubByte < MinSubByte ||
6541 SubByte + BytesPerElement > ExtBytesPerElement)
6542 break;
6543 // Get the byte offset of the unextended element
6544 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
6545 // ...then add the byte offset relative to that element.
6546 Byte += SubByte - MinSubByte;
6547 if (Byte % BytesPerElement != 0)
6548 break;
6549 Op = Op.getOperand(0);
6550 Index = Byte / BytesPerElement;
6551 Force = true;
6552 } else
6553 break;
6554 }
6555 if (Force) {
6556 if (Op.getValueType() != VecVT) {
6557 Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
6558 DCI.AddToWorklist(Op.getNode());
6559 }
6560 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
6561 DAG.getConstant(Index, DL, MVT::i32));
6562 }
6563 return SDValue();
6564 }
6565
6566 // Optimize vector operations in scalar value Op on the basis that Op
6567 // is truncated to TruncVT.
combineTruncateExtract(const SDLoc & DL,EVT TruncVT,SDValue Op,DAGCombinerInfo & DCI) const6568 SDValue SystemZTargetLowering::combineTruncateExtract(
6569 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
6570 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
6571 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
6572 // of type TruncVT.
6573 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6574 TruncVT.getSizeInBits() % 8 == 0) {
6575 SDValue Vec = Op.getOperand(0);
6576 EVT VecVT = Vec.getValueType();
6577 if (canTreatAsByteVector(VecVT)) {
6578 if (auto *IndexN = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
6579 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
6580 unsigned TruncBytes = TruncVT.getStoreSize();
6581 if (BytesPerElement % TruncBytes == 0) {
6582 // Calculate the value of Y' in the above description. We are
6583 // splitting the original elements into Scale equal-sized pieces
6584 // and for truncation purposes want the last (least-significant)
6585 // of these pieces for IndexN. This is easiest to do by calculating
6586 // the start index of the following element and then subtracting 1.
6587 unsigned Scale = BytesPerElement / TruncBytes;
6588 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
6589
6590 // Defer the creation of the bitcast from X to combineExtract,
6591 // which might be able to optimize the extraction.
6592 VecVT = MVT::getVectorVT(MVT::getIntegerVT(TruncBytes * 8),
6593 VecVT.getStoreSize() / TruncBytes);
6594 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
6595 return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true);
6596 }
6597 }
6598 }
6599 }
6600 return SDValue();
6601 }
6602
combineZERO_EXTEND(SDNode * N,DAGCombinerInfo & DCI) const6603 SDValue SystemZTargetLowering::combineZERO_EXTEND(
6604 SDNode *N, DAGCombinerInfo &DCI) const {
6605 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
6606 SelectionDAG &DAG = DCI.DAG;
6607 SDValue N0 = N->getOperand(0);
6608 EVT VT = N->getValueType(0);
6609 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
6610 auto *TrueOp = dyn_cast<ConstantSDNode>(N0.getOperand(0));
6611 auto *FalseOp = dyn_cast<ConstantSDNode>(N0.getOperand(1));
6612 if (TrueOp && FalseOp) {
6613 SDLoc DL(N0);
6614 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
6615 DAG.getConstant(FalseOp->getZExtValue(), DL, VT),
6616 N0.getOperand(2), N0.getOperand(3), N0.getOperand(4) };
6617 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
6618 // If N0 has multiple uses, change other uses as well.
6619 if (!N0.hasOneUse()) {
6620 SDValue TruncSelect =
6621 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect);
6622 DCI.CombineTo(N0.getNode(), TruncSelect);
6623 }
6624 return NewSelect;
6625 }
6626 }
6627 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
6628 // of the result is smaller than the size of X and all the truncated bits
6629 // of X are already zero.
6630 if (N0.getOpcode() == ISD::XOR &&
6631 N0.hasOneUse() && N0.getOperand(0).hasOneUse() &&
6632 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
6633 N0.getOperand(1).getOpcode() == ISD::Constant) {
6634 SDValue X = N0.getOperand(0).getOperand(0);
6635 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
6636 KnownBits Known = DAG.computeKnownBits(X);
6637 APInt TruncatedBits = APInt::getBitsSet(X.getValueSizeInBits(),
6638 N0.getValueSizeInBits(),
6639 VT.getSizeInBits());
6640 if (TruncatedBits.isSubsetOf(Known.Zero)) {
6641 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
6642 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
6643 return DAG.getNode(ISD::XOR, SDLoc(N0), VT,
6644 X, DAG.getConstant(Mask, SDLoc(N0), VT));
6645 }
6646 }
6647 }
6648 return SDValue();
6649 }
6650
combineSIGN_EXTEND_INREG(SDNode * N,DAGCombinerInfo & DCI) const6651 SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
6652 SDNode *N, DAGCombinerInfo &DCI) const {
6653 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
6654 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
6655 // into (select_cc LHS, RHS, -1, 0, COND)
6656 SelectionDAG &DAG = DCI.DAG;
6657 SDValue N0 = N->getOperand(0);
6658 EVT VT = N->getValueType(0);
6659 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
6660 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
6661 N0 = N0.getOperand(0);
6662 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
6663 SDLoc DL(N0);
6664 SDValue Ops[] = { N0.getOperand(0), N0.getOperand(1),
6665 DAG.getConstant(-1, DL, VT), DAG.getConstant(0, DL, VT),
6666 N0.getOperand(2) };
6667 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
6668 }
6669 return SDValue();
6670 }
6671
combineSIGN_EXTEND(SDNode * N,DAGCombinerInfo & DCI) const6672 SDValue SystemZTargetLowering::combineSIGN_EXTEND(
6673 SDNode *N, DAGCombinerInfo &DCI) const {
6674 // Convert (sext (ashr (shl X, C1), C2)) to
6675 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
6676 // cheap as narrower ones.
6677 SelectionDAG &DAG = DCI.DAG;
6678 SDValue N0 = N->getOperand(0);
6679 EVT VT = N->getValueType(0);
6680 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
6681 auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
6682 SDValue Inner = N0.getOperand(0);
6683 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
6684 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) {
6685 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
6686 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
6687 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
6688 EVT ShiftVT = N0.getOperand(1).getValueType();
6689 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
6690 Inner.getOperand(0));
6691 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
6692 DAG.getConstant(NewShlAmt, SDLoc(Inner),
6693 ShiftVT));
6694 return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl,
6695 DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT));
6696 }
6697 }
6698 }
6699 return SDValue();
6700 }
6701
combineMERGE(SDNode * N,DAGCombinerInfo & DCI) const6702 SDValue SystemZTargetLowering::combineMERGE(
6703 SDNode *N, DAGCombinerInfo &DCI) const {
6704 SelectionDAG &DAG = DCI.DAG;
6705 unsigned Opcode = N->getOpcode();
6706 SDValue Op0 = N->getOperand(0);
6707 SDValue Op1 = N->getOperand(1);
6708 if (Op0.getOpcode() == ISD::BITCAST)
6709 Op0 = Op0.getOperand(0);
6710 if (ISD::isBuildVectorAllZeros(Op0.getNode())) {
6711 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
6712 // for v4f32.
6713 if (Op1 == N->getOperand(0))
6714 return Op1;
6715 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
6716 EVT VT = Op1.getValueType();
6717 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
6718 if (ElemBytes <= 4) {
6719 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
6720 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
6721 EVT InVT = VT.changeVectorElementTypeToInteger();
6722 EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(ElemBytes * 16),
6723 SystemZ::VectorBytes / ElemBytes / 2);
6724 if (VT != InVT) {
6725 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1);
6726 DCI.AddToWorklist(Op1.getNode());
6727 }
6728 SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
6729 DCI.AddToWorklist(Op.getNode());
6730 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
6731 }
6732 }
6733 return SDValue();
6734 }
6735
combineLOAD(SDNode * N,DAGCombinerInfo & DCI) const6736 SDValue SystemZTargetLowering::combineLOAD(
6737 SDNode *N, DAGCombinerInfo &DCI) const {
6738 SelectionDAG &DAG = DCI.DAG;
6739 EVT LdVT = N->getValueType(0);
6740 SDLoc DL(N);
6741
6742 // Replace an i128 load that is used solely to move its value into GPRs
6743 // by separate loads of both halves.
6744 if (LdVT == MVT::i128) {
6745 LoadSDNode *LD = cast<LoadSDNode>(N);
6746 if (!LD->isSimple() || !ISD::isNormalLoad(LD))
6747 return SDValue();
6748
6749 // Scan through all users.
6750 SmallVector<std::pair<SDNode *, int>, 2> Users;
6751 int UsedElements = 0;
6752 for (SDNode::use_iterator UI = LD->use_begin(), UIEnd = LD->use_end();
6753 UI != UIEnd; ++UI) {
6754 // Skip the uses of the chain.
6755 if (UI.getUse().getResNo() != 0)
6756 continue;
6757
6758 // Verify every user is a TRUNCATE to i64 of the low or high half ...
6759 SDNode *User = *UI;
6760 int Index = 1;
6761 if (User->getOpcode() == ISD::SRL &&
6762 User->getOperand(1).getOpcode() == ISD::Constant &&
6763 User->getConstantOperandVal(1) == 64 && User->hasOneUse()) {
6764 User = *User->use_begin();
6765 Index = 0;
6766 }
6767 if (User->getOpcode() != ISD::TRUNCATE ||
6768 User->getValueType(0) != MVT::i64)
6769 return SDValue();
6770
6771 // ... and no half is extracted twice.
6772 if (UsedElements & (1 << Index))
6773 return SDValue();
6774
6775 UsedElements |= 1 << Index;
6776 Users.push_back(std::make_pair(User, Index));
6777 }
6778
6779 // Rewrite each extraction as an independent load.
6780 SmallVector<SDValue, 2> ArgChains;
6781 for (auto UserAndIndex : Users) {
6782 SDNode *User = UserAndIndex.first;
6783 unsigned Offset = User->getValueType(0).getStoreSize() * UserAndIndex.second;
6784 SDValue Ptr =
6785 DAG.getMemBasePlusOffset(LD->getBasePtr(), TypeSize::getFixed(Offset), DL);
6786 SDValue EltLoad =
6787 DAG.getLoad(User->getValueType(0), DL, LD->getChain(), Ptr,
6788 LD->getPointerInfo().getWithOffset(Offset),
6789 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
6790 LD->getAAInfo());
6791
6792 DCI.CombineTo(User, EltLoad, true);
6793 ArgChains.push_back(EltLoad.getValue(1));
6794 }
6795
6796 // Collect all chains via TokenFactor.
6797 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
6798 ArgChains);
6799 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain);
6800 DCI.AddToWorklist(Chain.getNode());
6801 return SDValue(N, 0);
6802 }
6803
6804 if (LdVT.isVector() || LdVT.isInteger())
6805 return SDValue();
6806 // Transform a scalar load that is REPLICATEd as well as having other
6807 // use(s) to the form where the other use(s) use the first element of the
6808 // REPLICATE instead of the load. Otherwise instruction selection will not
6809 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
6810 // point loads.
6811
6812 SDValue Replicate;
6813 SmallVector<SDNode*, 8> OtherUses;
6814 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
6815 UI != UE; ++UI) {
6816 if (UI->getOpcode() == SystemZISD::REPLICATE) {
6817 if (Replicate)
6818 return SDValue(); // Should never happen
6819 Replicate = SDValue(*UI, 0);
6820 }
6821 else if (UI.getUse().getResNo() == 0)
6822 OtherUses.push_back(*UI);
6823 }
6824 if (!Replicate || OtherUses.empty())
6825 return SDValue();
6826
6827 SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT,
6828 Replicate, DAG.getConstant(0, DL, MVT::i32));
6829 // Update uses of the loaded Value while preserving old chains.
6830 for (SDNode *U : OtherUses) {
6831 SmallVector<SDValue, 8> Ops;
6832 for (SDValue Op : U->ops())
6833 Ops.push_back((Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
6834 DAG.UpdateNodeOperands(U, Ops);
6835 }
6836 return SDValue(N, 0);
6837 }
6838
canLoadStoreByteSwapped(EVT VT) const6839 bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
6840 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
6841 return true;
6842 if (Subtarget.hasVectorEnhancements2())
6843 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
6844 return true;
6845 return false;
6846 }
6847
isVectorElementSwap(ArrayRef<int> M,EVT VT)6848 static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
6849 if (!VT.isVector() || !VT.isSimple() ||
6850 VT.getSizeInBits() != 128 ||
6851 VT.getScalarSizeInBits() % 8 != 0)
6852 return false;
6853
6854 unsigned NumElts = VT.getVectorNumElements();
6855 for (unsigned i = 0; i < NumElts; ++i) {
6856 if (M[i] < 0) continue; // ignore UNDEF indices
6857 if ((unsigned) M[i] != NumElts - 1 - i)
6858 return false;
6859 }
6860
6861 return true;
6862 }
6863
isOnlyUsedByStores(SDValue StoredVal,SelectionDAG & DAG)6864 static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
6865 for (auto *U : StoredVal->uses()) {
6866 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(U)) {
6867 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
6868 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
6869 continue;
6870 } else if (isa<BuildVectorSDNode>(U)) {
6871 SDValue BuildVector = SDValue(U, 0);
6872 if (DAG.isSplatValue(BuildVector, true/*AllowUndefs*/) &&
6873 isOnlyUsedByStores(BuildVector, DAG))
6874 continue;
6875 }
6876 return false;
6877 }
6878 return true;
6879 }
6880
isMovedFromParts(SDValue Val,SDValue & LoPart,SDValue & HiPart)6881 static bool isMovedFromParts(SDValue Val, SDValue &LoPart, SDValue &HiPart) {
6882 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
6883 return false;
6884
6885 SDValue Op0 = Val.getOperand(0);
6886 SDValue Op1 = Val.getOperand(1);
6887
6888 if (Op0.getOpcode() == ISD::SHL)
6889 std::swap(Op0, Op1);
6890 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
6891 Op1.getOperand(1).getOpcode() != ISD::Constant ||
6892 Op1.getConstantOperandVal(1) != 64)
6893 return false;
6894 Op1 = Op1.getOperand(0);
6895
6896 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
6897 Op0.getOperand(0).getValueType() != MVT::i64)
6898 return false;
6899 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
6900 Op1.getOperand(0).getValueType() != MVT::i64)
6901 return false;
6902
6903 LoPart = Op0.getOperand(0);
6904 HiPart = Op1.getOperand(0);
6905 return true;
6906 }
6907
combineSTORE(SDNode * N,DAGCombinerInfo & DCI) const6908 SDValue SystemZTargetLowering::combineSTORE(
6909 SDNode *N, DAGCombinerInfo &DCI) const {
6910 SelectionDAG &DAG = DCI.DAG;
6911 auto *SN = cast<StoreSDNode>(N);
6912 auto &Op1 = N->getOperand(1);
6913 EVT MemVT = SN->getMemoryVT();
6914 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
6915 // for the extraction to be done on a vMiN value, so that we can use VSTE.
6916 // If X has wider elements then convert it to:
6917 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
6918 if (MemVT.isInteger() && SN->isTruncatingStore()) {
6919 if (SDValue Value =
6920 combineTruncateExtract(SDLoc(N), MemVT, SN->getValue(), DCI)) {
6921 DCI.AddToWorklist(Value.getNode());
6922
6923 // Rewrite the store with the new form of stored value.
6924 return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,
6925 SN->getBasePtr(), SN->getMemoryVT(),
6926 SN->getMemOperand());
6927 }
6928 }
6929 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
6930 if (!SN->isTruncatingStore() &&
6931 Op1.getOpcode() == ISD::BSWAP &&
6932 Op1.getNode()->hasOneUse() &&
6933 canLoadStoreByteSwapped(Op1.getValueType())) {
6934
6935 SDValue BSwapOp = Op1.getOperand(0);
6936
6937 if (BSwapOp.getValueType() == MVT::i16)
6938 BSwapOp = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), MVT::i32, BSwapOp);
6939
6940 SDValue Ops[] = {
6941 N->getOperand(0), BSwapOp, N->getOperand(2)
6942 };
6943
6944 return
6945 DAG.getMemIntrinsicNode(SystemZISD::STRV, SDLoc(N), DAG.getVTList(MVT::Other),
6946 Ops, MemVT, SN->getMemOperand());
6947 }
6948 // Combine STORE (element-swap) into VSTER
6949 if (!SN->isTruncatingStore() &&
6950 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
6951 Op1.getNode()->hasOneUse() &&
6952 Subtarget.hasVectorEnhancements2()) {
6953 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op1.getNode());
6954 ArrayRef<int> ShuffleMask = SVN->getMask();
6955 if (isVectorElementSwap(ShuffleMask, Op1.getValueType())) {
6956 SDValue Ops[] = {
6957 N->getOperand(0), Op1.getOperand(0), N->getOperand(2)
6958 };
6959
6960 return DAG.getMemIntrinsicNode(SystemZISD::VSTER, SDLoc(N),
6961 DAG.getVTList(MVT::Other),
6962 Ops, MemVT, SN->getMemOperand());
6963 }
6964 }
6965
6966 // Transform a store of an i128 moved from GPRs into two separate stores.
6967 if (MemVT == MVT::i128 && SN->isSimple() && ISD::isNormalStore(SN)) {
6968 SDValue LoPart, HiPart;
6969 if (isMovedFromParts(Op1, LoPart, HiPart)) {
6970 SDLoc DL(SN);
6971 SDValue Chain0 =
6972 DAG.getStore(SN->getChain(), DL, HiPart, SN->getBasePtr(),
6973 SN->getPointerInfo(), SN->getOriginalAlign(),
6974 SN->getMemOperand()->getFlags(), SN->getAAInfo());
6975 SDValue Chain1 =
6976 DAG.getStore(SN->getChain(), DL, LoPart,
6977 DAG.getObjectPtrOffset(DL, SN->getBasePtr(),
6978 TypeSize::getFixed(8)),
6979 SN->getPointerInfo().getWithOffset(8),
6980 SN->getOriginalAlign(),
6981 SN->getMemOperand()->getFlags(), SN->getAAInfo());
6982
6983 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain0, Chain1);
6984 }
6985 }
6986
6987 // Replicate a reg or immediate with VREP instead of scalar multiply or
6988 // immediate load. It seems best to do this during the first DAGCombine as
6989 // it is straight-forward to handle the zero-extend node in the initial
6990 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
6991 // extracting an i16 element from a v16i8 vector).
6992 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
6993 isOnlyUsedByStores(Op1, DAG)) {
6994 SDValue Word = SDValue();
6995 EVT WordVT;
6996
6997 // Find a replicated immediate and return it if found in Word and its
6998 // type in WordVT.
6999 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
7000 // Some constants are better handled with a scalar store.
7001 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
7002 isInt<16>(C->getSExtValue()) || MemVT.getStoreSize() <= 2)
7003 return;
7004 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, C->getZExtValue()));
7005 if (VCI.isVectorConstantLegal(Subtarget) &&
7006 VCI.Opcode == SystemZISD::REPLICATE) {
7007 Word = DAG.getConstant(VCI.OpVals[0], SDLoc(SN), MVT::i32);
7008 WordVT = VCI.VecVT.getScalarType();
7009 }
7010 };
7011
7012 // Find a replicated register and return it if found in Word and its type
7013 // in WordVT.
7014 auto FindReplicatedReg = [&](SDValue MulOp) {
7015 EVT MulVT = MulOp.getValueType();
7016 if (MulOp->getOpcode() == ISD::MUL &&
7017 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
7018 // Find a zero extended value and its type.
7019 SDValue LHS = MulOp->getOperand(0);
7020 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
7021 WordVT = LHS->getOperand(0).getValueType();
7022 else if (LHS->getOpcode() == ISD::AssertZext)
7023 WordVT = cast<VTSDNode>(LHS->getOperand(1))->getVT();
7024 else
7025 return;
7026 // Find a replicating constant, e.g. 0x00010001.
7027 if (auto *C = dyn_cast<ConstantSDNode>(MulOp->getOperand(1))) {
7028 SystemZVectorConstantInfo VCI(
7029 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
7030 if (VCI.isVectorConstantLegal(Subtarget) &&
7031 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
7032 WordVT == VCI.VecVT.getScalarType())
7033 Word = DAG.getZExtOrTrunc(LHS->getOperand(0), SDLoc(SN), WordVT);
7034 }
7035 }
7036 };
7037
7038 if (isa<BuildVectorSDNode>(Op1) &&
7039 DAG.isSplatValue(Op1, true/*AllowUndefs*/)) {
7040 SDValue SplatVal = Op1->getOperand(0);
7041 if (auto *C = dyn_cast<ConstantSDNode>(SplatVal))
7042 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
7043 else
7044 FindReplicatedReg(SplatVal);
7045 } else {
7046 if (auto *C = dyn_cast<ConstantSDNode>(Op1))
7047 FindReplicatedImm(C, MemVT.getStoreSize());
7048 else
7049 FindReplicatedReg(Op1);
7050 }
7051
7052 if (Word != SDValue()) {
7053 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
7054 "Bad type handling");
7055 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
7056 EVT SplatVT = EVT::getVectorVT(*DAG.getContext(), WordVT, NumElts);
7057 SDValue SplatVal = DAG.getSplatVector(SplatVT, SDLoc(SN), Word);
7058 return DAG.getStore(SN->getChain(), SDLoc(SN), SplatVal,
7059 SN->getBasePtr(), SN->getMemOperand());
7060 }
7061 }
7062
7063 return SDValue();
7064 }
7065
combineVECTOR_SHUFFLE(SDNode * N,DAGCombinerInfo & DCI) const7066 SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
7067 SDNode *N, DAGCombinerInfo &DCI) const {
7068 SelectionDAG &DAG = DCI.DAG;
7069 // Combine element-swap (LOAD) into VLER
7070 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
7071 N->getOperand(0).hasOneUse() &&
7072 Subtarget.hasVectorEnhancements2()) {
7073 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
7074 ArrayRef<int> ShuffleMask = SVN->getMask();
7075 if (isVectorElementSwap(ShuffleMask, N->getValueType(0))) {
7076 SDValue Load = N->getOperand(0);
7077 LoadSDNode *LD = cast<LoadSDNode>(Load);
7078
7079 // Create the element-swapping load.
7080 SDValue Ops[] = {
7081 LD->getChain(), // Chain
7082 LD->getBasePtr() // Ptr
7083 };
7084 SDValue ESLoad =
7085 DAG.getMemIntrinsicNode(SystemZISD::VLER, SDLoc(N),
7086 DAG.getVTList(LD->getValueType(0), MVT::Other),
7087 Ops, LD->getMemoryVT(), LD->getMemOperand());
7088
7089 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
7090 // by the load dead.
7091 DCI.CombineTo(N, ESLoad);
7092
7093 // Next, combine the load away, we give it a bogus result value but a real
7094 // chain result. The result value is dead because the shuffle is dead.
7095 DCI.CombineTo(Load.getNode(), ESLoad, ESLoad.getValue(1));
7096
7097 // Return N so it doesn't get rechecked!
7098 return SDValue(N, 0);
7099 }
7100 }
7101
7102 return SDValue();
7103 }
7104
combineEXTRACT_VECTOR_ELT(SDNode * N,DAGCombinerInfo & DCI) const7105 SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
7106 SDNode *N, DAGCombinerInfo &DCI) const {
7107 SelectionDAG &DAG = DCI.DAG;
7108
7109 if (!Subtarget.hasVector())
7110 return SDValue();
7111
7112 // Look through bitcasts that retain the number of vector elements.
7113 SDValue Op = N->getOperand(0);
7114 if (Op.getOpcode() == ISD::BITCAST &&
7115 Op.getValueType().isVector() &&
7116 Op.getOperand(0).getValueType().isVector() &&
7117 Op.getValueType().getVectorNumElements() ==
7118 Op.getOperand(0).getValueType().getVectorNumElements())
7119 Op = Op.getOperand(0);
7120
7121 // Pull BSWAP out of a vector extraction.
7122 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
7123 EVT VecVT = Op.getValueType();
7124 EVT EltVT = VecVT.getVectorElementType();
7125 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), EltVT,
7126 Op.getOperand(0), N->getOperand(1));
7127 DCI.AddToWorklist(Op.getNode());
7128 Op = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Op);
7129 if (EltVT != N->getValueType(0)) {
7130 DCI.AddToWorklist(Op.getNode());
7131 Op = DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op);
7132 }
7133 return Op;
7134 }
7135
7136 // Try to simplify a vector extraction.
7137 if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
7138 SDValue Op0 = N->getOperand(0);
7139 EVT VecVT = Op0.getValueType();
7140 return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0,
7141 IndexN->getZExtValue(), DCI, false);
7142 }
7143 return SDValue();
7144 }
7145
combineJOIN_DWORDS(SDNode * N,DAGCombinerInfo & DCI) const7146 SDValue SystemZTargetLowering::combineJOIN_DWORDS(
7147 SDNode *N, DAGCombinerInfo &DCI) const {
7148 SelectionDAG &DAG = DCI.DAG;
7149 // (join_dwords X, X) == (replicate X)
7150 if (N->getOperand(0) == N->getOperand(1))
7151 return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),
7152 N->getOperand(0));
7153 return SDValue();
7154 }
7155
MergeInputChains(SDNode * N1,SDNode * N2)7156 static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
7157 SDValue Chain1 = N1->getOperand(0);
7158 SDValue Chain2 = N2->getOperand(0);
7159
7160 // Trivial case: both nodes take the same chain.
7161 if (Chain1 == Chain2)
7162 return Chain1;
7163
7164 // FIXME - we could handle more complex cases via TokenFactor,
7165 // assuming we can verify that this would not create a cycle.
7166 return SDValue();
7167 }
7168
combineFP_ROUND(SDNode * N,DAGCombinerInfo & DCI) const7169 SDValue SystemZTargetLowering::combineFP_ROUND(
7170 SDNode *N, DAGCombinerInfo &DCI) const {
7171
7172 if (!Subtarget.hasVector())
7173 return SDValue();
7174
7175 // (fpround (extract_vector_elt X 0))
7176 // (fpround (extract_vector_elt X 1)) ->
7177 // (extract_vector_elt (VROUND X) 0)
7178 // (extract_vector_elt (VROUND X) 2)
7179 //
7180 // This is a special case since the target doesn't really support v2f32s.
7181 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7182 SelectionDAG &DAG = DCI.DAG;
7183 SDValue Op0 = N->getOperand(OpNo);
7184 if (N->getValueType(0) == MVT::f32 && Op0.hasOneUse() &&
7185 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7186 Op0.getOperand(0).getValueType() == MVT::v2f64 &&
7187 Op0.getOperand(1).getOpcode() == ISD::Constant &&
7188 Op0.getConstantOperandVal(1) == 0) {
7189 SDValue Vec = Op0.getOperand(0);
7190 for (auto *U : Vec->uses()) {
7191 if (U != Op0.getNode() && U->hasOneUse() &&
7192 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7193 U->getOperand(0) == Vec &&
7194 U->getOperand(1).getOpcode() == ISD::Constant &&
7195 U->getConstantOperandVal(1) == 1) {
7196 SDValue OtherRound = SDValue(*U->use_begin(), 0);
7197 if (OtherRound.getOpcode() == N->getOpcode() &&
7198 OtherRound.getOperand(OpNo) == SDValue(U, 0) &&
7199 OtherRound.getValueType() == MVT::f32) {
7200 SDValue VRound, Chain;
7201 if (N->isStrictFPOpcode()) {
7202 Chain = MergeInputChains(N, OtherRound.getNode());
7203 if (!Chain)
7204 continue;
7205 VRound = DAG.getNode(SystemZISD::STRICT_VROUND, SDLoc(N),
7206 {MVT::v4f32, MVT::Other}, {Chain, Vec});
7207 Chain = VRound.getValue(1);
7208 } else
7209 VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
7210 MVT::v4f32, Vec);
7211 DCI.AddToWorklist(VRound.getNode());
7212 SDValue Extract1 =
7213 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32,
7214 VRound, DAG.getConstant(2, SDLoc(U), MVT::i32));
7215 DCI.AddToWorklist(Extract1.getNode());
7216 DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1);
7217 if (Chain)
7218 DAG.ReplaceAllUsesOfValueWith(OtherRound.getValue(1), Chain);
7219 SDValue Extract0 =
7220 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32,
7221 VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
7222 if (Chain)
7223 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
7224 N->getVTList(), Extract0, Chain);
7225 return Extract0;
7226 }
7227 }
7228 }
7229 }
7230 return SDValue();
7231 }
7232
combineFP_EXTEND(SDNode * N,DAGCombinerInfo & DCI) const7233 SDValue SystemZTargetLowering::combineFP_EXTEND(
7234 SDNode *N, DAGCombinerInfo &DCI) const {
7235
7236 if (!Subtarget.hasVector())
7237 return SDValue();
7238
7239 // (fpextend (extract_vector_elt X 0))
7240 // (fpextend (extract_vector_elt X 2)) ->
7241 // (extract_vector_elt (VEXTEND X) 0)
7242 // (extract_vector_elt (VEXTEND X) 1)
7243 //
7244 // This is a special case since the target doesn't really support v2f32s.
7245 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7246 SelectionDAG &DAG = DCI.DAG;
7247 SDValue Op0 = N->getOperand(OpNo);
7248 if (N->getValueType(0) == MVT::f64 && Op0.hasOneUse() &&
7249 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7250 Op0.getOperand(0).getValueType() == MVT::v4f32 &&
7251 Op0.getOperand(1).getOpcode() == ISD::Constant &&
7252 Op0.getConstantOperandVal(1) == 0) {
7253 SDValue Vec = Op0.getOperand(0);
7254 for (auto *U : Vec->uses()) {
7255 if (U != Op0.getNode() && U->hasOneUse() &&
7256 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7257 U->getOperand(0) == Vec &&
7258 U->getOperand(1).getOpcode() == ISD::Constant &&
7259 U->getConstantOperandVal(1) == 2) {
7260 SDValue OtherExtend = SDValue(*U->use_begin(), 0);
7261 if (OtherExtend.getOpcode() == N->getOpcode() &&
7262 OtherExtend.getOperand(OpNo) == SDValue(U, 0) &&
7263 OtherExtend.getValueType() == MVT::f64) {
7264 SDValue VExtend, Chain;
7265 if (N->isStrictFPOpcode()) {
7266 Chain = MergeInputChains(N, OtherExtend.getNode());
7267 if (!Chain)
7268 continue;
7269 VExtend = DAG.getNode(SystemZISD::STRICT_VEXTEND, SDLoc(N),
7270 {MVT::v2f64, MVT::Other}, {Chain, Vec});
7271 Chain = VExtend.getValue(1);
7272 } else
7273 VExtend = DAG.getNode(SystemZISD::VEXTEND, SDLoc(N),
7274 MVT::v2f64, Vec);
7275 DCI.AddToWorklist(VExtend.getNode());
7276 SDValue Extract1 =
7277 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f64,
7278 VExtend, DAG.getConstant(1, SDLoc(U), MVT::i32));
7279 DCI.AddToWorklist(Extract1.getNode());
7280 DAG.ReplaceAllUsesOfValueWith(OtherExtend, Extract1);
7281 if (Chain)
7282 DAG.ReplaceAllUsesOfValueWith(OtherExtend.getValue(1), Chain);
7283 SDValue Extract0 =
7284 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f64,
7285 VExtend, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
7286 if (Chain)
7287 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
7288 N->getVTList(), Extract0, Chain);
7289 return Extract0;
7290 }
7291 }
7292 }
7293 }
7294 return SDValue();
7295 }
7296
combineINT_TO_FP(SDNode * N,DAGCombinerInfo & DCI) const7297 SDValue SystemZTargetLowering::combineINT_TO_FP(
7298 SDNode *N, DAGCombinerInfo &DCI) const {
7299 if (DCI.Level != BeforeLegalizeTypes)
7300 return SDValue();
7301 SelectionDAG &DAG = DCI.DAG;
7302 LLVMContext &Ctx = *DAG.getContext();
7303 unsigned Opcode = N->getOpcode();
7304 EVT OutVT = N->getValueType(0);
7305 Type *OutLLVMTy = OutVT.getTypeForEVT(Ctx);
7306 SDValue Op = N->getOperand(0);
7307 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
7308 unsigned InScalarBits = Op->getValueType(0).getScalarSizeInBits();
7309
7310 // Insert an extension before type-legalization to avoid scalarization, e.g.:
7311 // v2f64 = uint_to_fp v2i16
7312 // =>
7313 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
7314 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
7315 OutScalarBits <= 64) {
7316 unsigned NumElts = cast<FixedVectorType>(OutLLVMTy)->getNumElements();
7317 EVT ExtVT = EVT::getVectorVT(
7318 Ctx, EVT::getIntegerVT(Ctx, OutLLVMTy->getScalarSizeInBits()), NumElts);
7319 unsigned ExtOpcode =
7320 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
7321 SDValue ExtOp = DAG.getNode(ExtOpcode, SDLoc(N), ExtVT, Op);
7322 return DAG.getNode(Opcode, SDLoc(N), OutVT, ExtOp);
7323 }
7324 return SDValue();
7325 }
7326
combineBSWAP(SDNode * N,DAGCombinerInfo & DCI) const7327 SDValue SystemZTargetLowering::combineBSWAP(
7328 SDNode *N, DAGCombinerInfo &DCI) const {
7329 SelectionDAG &DAG = DCI.DAG;
7330 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
7331 if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
7332 N->getOperand(0).hasOneUse() &&
7333 canLoadStoreByteSwapped(N->getValueType(0))) {
7334 SDValue Load = N->getOperand(0);
7335 LoadSDNode *LD = cast<LoadSDNode>(Load);
7336
7337 // Create the byte-swapping load.
7338 SDValue Ops[] = {
7339 LD->getChain(), // Chain
7340 LD->getBasePtr() // Ptr
7341 };
7342 EVT LoadVT = N->getValueType(0);
7343 if (LoadVT == MVT::i16)
7344 LoadVT = MVT::i32;
7345 SDValue BSLoad =
7346 DAG.getMemIntrinsicNode(SystemZISD::LRV, SDLoc(N),
7347 DAG.getVTList(LoadVT, MVT::Other),
7348 Ops, LD->getMemoryVT(), LD->getMemOperand());
7349
7350 // If this is an i16 load, insert the truncate.
7351 SDValue ResVal = BSLoad;
7352 if (N->getValueType(0) == MVT::i16)
7353 ResVal = DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16, BSLoad);
7354
7355 // First, combine the bswap away. This makes the value produced by the
7356 // load dead.
7357 DCI.CombineTo(N, ResVal);
7358
7359 // Next, combine the load away, we give it a bogus result value but a real
7360 // chain result. The result value is dead because the bswap is dead.
7361 DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1));
7362
7363 // Return N so it doesn't get rechecked!
7364 return SDValue(N, 0);
7365 }
7366
7367 // Look through bitcasts that retain the number of vector elements.
7368 SDValue Op = N->getOperand(0);
7369 if (Op.getOpcode() == ISD::BITCAST &&
7370 Op.getValueType().isVector() &&
7371 Op.getOperand(0).getValueType().isVector() &&
7372 Op.getValueType().getVectorNumElements() ==
7373 Op.getOperand(0).getValueType().getVectorNumElements())
7374 Op = Op.getOperand(0);
7375
7376 // Push BSWAP into a vector insertion if at least one side then simplifies.
7377 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
7378 SDValue Vec = Op.getOperand(0);
7379 SDValue Elt = Op.getOperand(1);
7380 SDValue Idx = Op.getOperand(2);
7381
7382 if (DAG.isConstantIntBuildVectorOrConstantInt(Vec) ||
7383 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
7384 DAG.isConstantIntBuildVectorOrConstantInt(Elt) ||
7385 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
7386 (canLoadStoreByteSwapped(N->getValueType(0)) &&
7387 ISD::isNON_EXTLoad(Elt.getNode()) && Elt.hasOneUse())) {
7388 EVT VecVT = N->getValueType(0);
7389 EVT EltVT = N->getValueType(0).getVectorElementType();
7390 if (VecVT != Vec.getValueType()) {
7391 Vec = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Vec);
7392 DCI.AddToWorklist(Vec.getNode());
7393 }
7394 if (EltVT != Elt.getValueType()) {
7395 Elt = DAG.getNode(ISD::BITCAST, SDLoc(N), EltVT, Elt);
7396 DCI.AddToWorklist(Elt.getNode());
7397 }
7398 Vec = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Vec);
7399 DCI.AddToWorklist(Vec.getNode());
7400 Elt = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Elt);
7401 DCI.AddToWorklist(Elt.getNode());
7402 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VecVT,
7403 Vec, Elt, Idx);
7404 }
7405 }
7406
7407 // Push BSWAP into a vector shuffle if at least one side then simplifies.
7408 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Op);
7409 if (SV && Op.hasOneUse()) {
7410 SDValue Op0 = Op.getOperand(0);
7411 SDValue Op1 = Op.getOperand(1);
7412
7413 if (DAG.isConstantIntBuildVectorOrConstantInt(Op0) ||
7414 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
7415 DAG.isConstantIntBuildVectorOrConstantInt(Op1) ||
7416 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
7417 EVT VecVT = N->getValueType(0);
7418 if (VecVT != Op0.getValueType()) {
7419 Op0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op0);
7420 DCI.AddToWorklist(Op0.getNode());
7421 }
7422 if (VecVT != Op1.getValueType()) {
7423 Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op1);
7424 DCI.AddToWorklist(Op1.getNode());
7425 }
7426 Op0 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op0);
7427 DCI.AddToWorklist(Op0.getNode());
7428 Op1 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op1);
7429 DCI.AddToWorklist(Op1.getNode());
7430 return DAG.getVectorShuffle(VecVT, SDLoc(N), Op0, Op1, SV->getMask());
7431 }
7432 }
7433
7434 return SDValue();
7435 }
7436
combineCCMask(SDValue & CCReg,int & CCValid,int & CCMask)7437 static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
7438 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
7439 // set by the CCReg instruction using the CCValid / CCMask masks,
7440 // If the CCReg instruction is itself a ICMP testing the condition
7441 // code set by some other instruction, see whether we can directly
7442 // use that condition code.
7443
7444 // Verify that we have an ICMP against some constant.
7445 if (CCValid != SystemZ::CCMASK_ICMP)
7446 return false;
7447 auto *ICmp = CCReg.getNode();
7448 if (ICmp->getOpcode() != SystemZISD::ICMP)
7449 return false;
7450 auto *CompareLHS = ICmp->getOperand(0).getNode();
7451 auto *CompareRHS = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
7452 if (!CompareRHS)
7453 return false;
7454
7455 // Optimize the case where CompareLHS is a SELECT_CCMASK.
7456 if (CompareLHS->getOpcode() == SystemZISD::SELECT_CCMASK) {
7457 // Verify that we have an appropriate mask for a EQ or NE comparison.
7458 bool Invert = false;
7459 if (CCMask == SystemZ::CCMASK_CMP_NE)
7460 Invert = !Invert;
7461 else if (CCMask != SystemZ::CCMASK_CMP_EQ)
7462 return false;
7463
7464 // Verify that the ICMP compares against one of select values.
7465 auto *TrueVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(0));
7466 if (!TrueVal)
7467 return false;
7468 auto *FalseVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
7469 if (!FalseVal)
7470 return false;
7471 if (CompareRHS->getZExtValue() == FalseVal->getZExtValue())
7472 Invert = !Invert;
7473 else if (CompareRHS->getZExtValue() != TrueVal->getZExtValue())
7474 return false;
7475
7476 // Compute the effective CC mask for the new branch or select.
7477 auto *NewCCValid = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(2));
7478 auto *NewCCMask = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(3));
7479 if (!NewCCValid || !NewCCMask)
7480 return false;
7481 CCValid = NewCCValid->getZExtValue();
7482 CCMask = NewCCMask->getZExtValue();
7483 if (Invert)
7484 CCMask ^= CCValid;
7485
7486 // Return the updated CCReg link.
7487 CCReg = CompareLHS->getOperand(4);
7488 return true;
7489 }
7490
7491 // Optimize the case where CompareRHS is (SRA (SHL (IPM))).
7492 if (CompareLHS->getOpcode() == ISD::SRA) {
7493 auto *SRACount = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
7494 if (!SRACount || SRACount->getZExtValue() != 30)
7495 return false;
7496 auto *SHL = CompareLHS->getOperand(0).getNode();
7497 if (SHL->getOpcode() != ISD::SHL)
7498 return false;
7499 auto *SHLCount = dyn_cast<ConstantSDNode>(SHL->getOperand(1));
7500 if (!SHLCount || SHLCount->getZExtValue() != 30 - SystemZ::IPM_CC)
7501 return false;
7502 auto *IPM = SHL->getOperand(0).getNode();
7503 if (IPM->getOpcode() != SystemZISD::IPM)
7504 return false;
7505
7506 // Avoid introducing CC spills (because SRA would clobber CC).
7507 if (!CompareLHS->hasOneUse())
7508 return false;
7509 // Verify that the ICMP compares against zero.
7510 if (CompareRHS->getZExtValue() != 0)
7511 return false;
7512
7513 // Compute the effective CC mask for the new branch or select.
7514 CCMask = SystemZ::reverseCCMask(CCMask);
7515
7516 // Return the updated CCReg link.
7517 CCReg = IPM->getOperand(0);
7518 return true;
7519 }
7520
7521 return false;
7522 }
7523
combineBR_CCMASK(SDNode * N,DAGCombinerInfo & DCI) const7524 SDValue SystemZTargetLowering::combineBR_CCMASK(
7525 SDNode *N, DAGCombinerInfo &DCI) const {
7526 SelectionDAG &DAG = DCI.DAG;
7527
7528 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
7529 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
7530 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
7531 if (!CCValid || !CCMask)
7532 return SDValue();
7533
7534 int CCValidVal = CCValid->getZExtValue();
7535 int CCMaskVal = CCMask->getZExtValue();
7536 SDValue Chain = N->getOperand(0);
7537 SDValue CCReg = N->getOperand(4);
7538
7539 if (combineCCMask(CCReg, CCValidVal, CCMaskVal))
7540 return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),
7541 Chain,
7542 DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
7543 DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
7544 N->getOperand(3), CCReg);
7545 return SDValue();
7546 }
7547
combineSELECT_CCMASK(SDNode * N,DAGCombinerInfo & DCI) const7548 SDValue SystemZTargetLowering::combineSELECT_CCMASK(
7549 SDNode *N, DAGCombinerInfo &DCI) const {
7550 SelectionDAG &DAG = DCI.DAG;
7551
7552 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
7553 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
7554 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
7555 if (!CCValid || !CCMask)
7556 return SDValue();
7557
7558 int CCValidVal = CCValid->getZExtValue();
7559 int CCMaskVal = CCMask->getZExtValue();
7560 SDValue CCReg = N->getOperand(4);
7561
7562 if (combineCCMask(CCReg, CCValidVal, CCMaskVal))
7563 return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
7564 N->getOperand(0), N->getOperand(1),
7565 DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
7566 DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
7567 CCReg);
7568 return SDValue();
7569 }
7570
7571
combineGET_CCMASK(SDNode * N,DAGCombinerInfo & DCI) const7572 SDValue SystemZTargetLowering::combineGET_CCMASK(
7573 SDNode *N, DAGCombinerInfo &DCI) const {
7574
7575 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
7576 auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
7577 auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
7578 if (!CCValid || !CCMask)
7579 return SDValue();
7580 int CCValidVal = CCValid->getZExtValue();
7581 int CCMaskVal = CCMask->getZExtValue();
7582
7583 SDValue Select = N->getOperand(0);
7584 if (Select->getOpcode() == ISD::TRUNCATE)
7585 Select = Select->getOperand(0);
7586 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
7587 return SDValue();
7588
7589 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));
7590 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));
7591 if (!SelectCCValid || !SelectCCMask)
7592 return SDValue();
7593 int SelectCCValidVal = SelectCCValid->getZExtValue();
7594 int SelectCCMaskVal = SelectCCMask->getZExtValue();
7595
7596 auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));
7597 auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));
7598 if (!TrueVal || !FalseVal)
7599 return SDValue();
7600 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
7601 ;
7602 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
7603 SelectCCMaskVal ^= SelectCCValidVal;
7604 else
7605 return SDValue();
7606
7607 if (SelectCCValidVal & ~CCValidVal)
7608 return SDValue();
7609 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
7610 return SDValue();
7611
7612 return Select->getOperand(4);
7613 }
7614
combineIntDIVREM(SDNode * N,DAGCombinerInfo & DCI) const7615 SDValue SystemZTargetLowering::combineIntDIVREM(
7616 SDNode *N, DAGCombinerInfo &DCI) const {
7617 SelectionDAG &DAG = DCI.DAG;
7618 EVT VT = N->getValueType(0);
7619 // In the case where the divisor is a vector of constants a cheaper
7620 // sequence of instructions can replace the divide. BuildSDIV is called to
7621 // do this during DAG combining, but it only succeeds when it can build a
7622 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
7623 // since it is not Legal but Custom it can only happen before
7624 // legalization. Therefore we must scalarize this early before Combine
7625 // 1. For widened vectors, this is already the result of type legalization.
7626 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
7627 DAG.isConstantIntBuildVectorOrConstantInt(N->getOperand(1)))
7628 return DAG.UnrollVectorOp(N);
7629 return SDValue();
7630 }
7631
combineINTRINSIC(SDNode * N,DAGCombinerInfo & DCI) const7632 SDValue SystemZTargetLowering::combineINTRINSIC(
7633 SDNode *N, DAGCombinerInfo &DCI) const {
7634 SelectionDAG &DAG = DCI.DAG;
7635
7636 unsigned Id = N->getConstantOperandVal(1);
7637 switch (Id) {
7638 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
7639 // or larger is simply a vector load.
7640 case Intrinsic::s390_vll:
7641 case Intrinsic::s390_vlrl:
7642 if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(2)))
7643 if (C->getZExtValue() >= 15)
7644 return DAG.getLoad(N->getValueType(0), SDLoc(N), N->getOperand(0),
7645 N->getOperand(3), MachinePointerInfo());
7646 break;
7647 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
7648 case Intrinsic::s390_vstl:
7649 case Intrinsic::s390_vstrl:
7650 if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(3)))
7651 if (C->getZExtValue() >= 15)
7652 return DAG.getStore(N->getOperand(0), SDLoc(N), N->getOperand(2),
7653 N->getOperand(4), MachinePointerInfo());
7654 break;
7655 }
7656
7657 return SDValue();
7658 }
7659
unwrapAddress(SDValue N) const7660 SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
7661 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
7662 return N->getOperand(0);
7663 return N;
7664 }
7665
PerformDAGCombine(SDNode * N,DAGCombinerInfo & DCI) const7666 SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
7667 DAGCombinerInfo &DCI) const {
7668 switch(N->getOpcode()) {
7669 default: break;
7670 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
7671 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
7672 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
7673 case SystemZISD::MERGE_HIGH:
7674 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
7675 case ISD::LOAD: return combineLOAD(N, DCI);
7676 case ISD::STORE: return combineSTORE(N, DCI);
7677 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
7678 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
7679 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
7680 case ISD::STRICT_FP_ROUND:
7681 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
7682 case ISD::STRICT_FP_EXTEND:
7683 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
7684 case ISD::SINT_TO_FP:
7685 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
7686 case ISD::BSWAP: return combineBSWAP(N, DCI);
7687 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
7688 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
7689 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
7690 case ISD::SDIV:
7691 case ISD::UDIV:
7692 case ISD::SREM:
7693 case ISD::UREM: return combineIntDIVREM(N, DCI);
7694 case ISD::INTRINSIC_W_CHAIN:
7695 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
7696 }
7697
7698 return SDValue();
7699 }
7700
7701 // Return the demanded elements for the OpNo source operand of Op. DemandedElts
7702 // are for Op.
getDemandedSrcElements(SDValue Op,const APInt & DemandedElts,unsigned OpNo)7703 static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
7704 unsigned OpNo) {
7705 EVT VT = Op.getValueType();
7706 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
7707 APInt SrcDemE;
7708 unsigned Opcode = Op.getOpcode();
7709 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
7710 unsigned Id = Op.getConstantOperandVal(0);
7711 switch (Id) {
7712 case Intrinsic::s390_vpksh: // PACKS
7713 case Intrinsic::s390_vpksf:
7714 case Intrinsic::s390_vpksg:
7715 case Intrinsic::s390_vpkshs: // PACKS_CC
7716 case Intrinsic::s390_vpksfs:
7717 case Intrinsic::s390_vpksgs:
7718 case Intrinsic::s390_vpklsh: // PACKLS
7719 case Intrinsic::s390_vpklsf:
7720 case Intrinsic::s390_vpklsg:
7721 case Intrinsic::s390_vpklshs: // PACKLS_CC
7722 case Intrinsic::s390_vpklsfs:
7723 case Intrinsic::s390_vpklsgs:
7724 // VECTOR PACK truncates the elements of two source vectors into one.
7725 SrcDemE = DemandedElts;
7726 if (OpNo == 2)
7727 SrcDemE.lshrInPlace(NumElts / 2);
7728 SrcDemE = SrcDemE.trunc(NumElts / 2);
7729 break;
7730 // VECTOR UNPACK extends half the elements of the source vector.
7731 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
7732 case Intrinsic::s390_vuphh:
7733 case Intrinsic::s390_vuphf:
7734 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
7735 case Intrinsic::s390_vuplhh:
7736 case Intrinsic::s390_vuplhf:
7737 SrcDemE = APInt(NumElts * 2, 0);
7738 SrcDemE.insertBits(DemandedElts, 0);
7739 break;
7740 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
7741 case Intrinsic::s390_vuplhw:
7742 case Intrinsic::s390_vuplf:
7743 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
7744 case Intrinsic::s390_vupllh:
7745 case Intrinsic::s390_vupllf:
7746 SrcDemE = APInt(NumElts * 2, 0);
7747 SrcDemE.insertBits(DemandedElts, NumElts);
7748 break;
7749 case Intrinsic::s390_vpdi: {
7750 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
7751 SrcDemE = APInt(NumElts, 0);
7752 if (!DemandedElts[OpNo - 1])
7753 break;
7754 unsigned Mask = Op.getConstantOperandVal(3);
7755 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
7756 // Demand input element 0 or 1, given by the mask bit value.
7757 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
7758 break;
7759 }
7760 case Intrinsic::s390_vsldb: {
7761 // VECTOR SHIFT LEFT DOUBLE BY BYTE
7762 assert(VT == MVT::v16i8 && "Unexpected type.");
7763 unsigned FirstIdx = Op.getConstantOperandVal(3);
7764 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
7765 unsigned NumSrc0Els = 16 - FirstIdx;
7766 SrcDemE = APInt(NumElts, 0);
7767 if (OpNo == 1) {
7768 APInt DemEls = DemandedElts.trunc(NumSrc0Els);
7769 SrcDemE.insertBits(DemEls, FirstIdx);
7770 } else {
7771 APInt DemEls = DemandedElts.lshr(NumSrc0Els);
7772 SrcDemE.insertBits(DemEls, 0);
7773 }
7774 break;
7775 }
7776 case Intrinsic::s390_vperm:
7777 SrcDemE = APInt(NumElts, 1);
7778 break;
7779 default:
7780 llvm_unreachable("Unhandled intrinsic.");
7781 break;
7782 }
7783 } else {
7784 switch (Opcode) {
7785 case SystemZISD::JOIN_DWORDS:
7786 // Scalar operand.
7787 SrcDemE = APInt(1, 1);
7788 break;
7789 case SystemZISD::SELECT_CCMASK:
7790 SrcDemE = DemandedElts;
7791 break;
7792 default:
7793 llvm_unreachable("Unhandled opcode.");
7794 break;
7795 }
7796 }
7797 return SrcDemE;
7798 }
7799
computeKnownBitsBinOp(const SDValue Op,KnownBits & Known,const APInt & DemandedElts,const SelectionDAG & DAG,unsigned Depth,unsigned OpNo)7800 static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
7801 const APInt &DemandedElts,
7802 const SelectionDAG &DAG, unsigned Depth,
7803 unsigned OpNo) {
7804 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
7805 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
7806 KnownBits LHSKnown =
7807 DAG.computeKnownBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
7808 KnownBits RHSKnown =
7809 DAG.computeKnownBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
7810 Known = LHSKnown.intersectWith(RHSKnown);
7811 }
7812
7813 void
computeKnownBitsForTargetNode(const SDValue Op,KnownBits & Known,const APInt & DemandedElts,const SelectionDAG & DAG,unsigned Depth) const7814 SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
7815 KnownBits &Known,
7816 const APInt &DemandedElts,
7817 const SelectionDAG &DAG,
7818 unsigned Depth) const {
7819 Known.resetAll();
7820
7821 // Intrinsic CC result is returned in the two low bits.
7822 unsigned tmp0, tmp1; // not used
7823 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, tmp0, tmp1)) {
7824 Known.Zero.setBitsFrom(2);
7825 return;
7826 }
7827 EVT VT = Op.getValueType();
7828 if (Op.getResNo() != 0 || VT == MVT::Untyped)
7829 return;
7830 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
7831 "KnownBits does not match VT in bitwidth");
7832 assert ((!VT.isVector() ||
7833 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
7834 "DemandedElts does not match VT number of elements");
7835 unsigned BitWidth = Known.getBitWidth();
7836 unsigned Opcode = Op.getOpcode();
7837 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
7838 bool IsLogical = false;
7839 unsigned Id = Op.getConstantOperandVal(0);
7840 switch (Id) {
7841 case Intrinsic::s390_vpksh: // PACKS
7842 case Intrinsic::s390_vpksf:
7843 case Intrinsic::s390_vpksg:
7844 case Intrinsic::s390_vpkshs: // PACKS_CC
7845 case Intrinsic::s390_vpksfs:
7846 case Intrinsic::s390_vpksgs:
7847 case Intrinsic::s390_vpklsh: // PACKLS
7848 case Intrinsic::s390_vpklsf:
7849 case Intrinsic::s390_vpklsg:
7850 case Intrinsic::s390_vpklshs: // PACKLS_CC
7851 case Intrinsic::s390_vpklsfs:
7852 case Intrinsic::s390_vpklsgs:
7853 case Intrinsic::s390_vpdi:
7854 case Intrinsic::s390_vsldb:
7855 case Intrinsic::s390_vperm:
7856 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 1);
7857 break;
7858 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
7859 case Intrinsic::s390_vuplhh:
7860 case Intrinsic::s390_vuplhf:
7861 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
7862 case Intrinsic::s390_vupllh:
7863 case Intrinsic::s390_vupllf:
7864 IsLogical = true;
7865 [[fallthrough]];
7866 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
7867 case Intrinsic::s390_vuphh:
7868 case Intrinsic::s390_vuphf:
7869 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
7870 case Intrinsic::s390_vuplhw:
7871 case Intrinsic::s390_vuplf: {
7872 SDValue SrcOp = Op.getOperand(1);
7873 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 0);
7874 Known = DAG.computeKnownBits(SrcOp, SrcDemE, Depth + 1);
7875 if (IsLogical) {
7876 Known = Known.zext(BitWidth);
7877 } else
7878 Known = Known.sext(BitWidth);
7879 break;
7880 }
7881 default:
7882 break;
7883 }
7884 } else {
7885 switch (Opcode) {
7886 case SystemZISD::JOIN_DWORDS:
7887 case SystemZISD::SELECT_CCMASK:
7888 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 0);
7889 break;
7890 case SystemZISD::REPLICATE: {
7891 SDValue SrcOp = Op.getOperand(0);
7892 Known = DAG.computeKnownBits(SrcOp, Depth + 1);
7893 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(SrcOp))
7894 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
7895 break;
7896 }
7897 default:
7898 break;
7899 }
7900 }
7901
7902 // Known has the width of the source operand(s). Adjust if needed to match
7903 // the passed bitwidth.
7904 if (Known.getBitWidth() != BitWidth)
7905 Known = Known.anyextOrTrunc(BitWidth);
7906 }
7907
computeNumSignBitsBinOp(SDValue Op,const APInt & DemandedElts,const SelectionDAG & DAG,unsigned Depth,unsigned OpNo)7908 static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
7909 const SelectionDAG &DAG, unsigned Depth,
7910 unsigned OpNo) {
7911 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
7912 unsigned LHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
7913 if (LHS == 1) return 1; // Early out.
7914 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
7915 unsigned RHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
7916 if (RHS == 1) return 1; // Early out.
7917 unsigned Common = std::min(LHS, RHS);
7918 unsigned SrcBitWidth = Op.getOperand(OpNo).getScalarValueSizeInBits();
7919 EVT VT = Op.getValueType();
7920 unsigned VTBits = VT.getScalarSizeInBits();
7921 if (SrcBitWidth > VTBits) { // PACK
7922 unsigned SrcExtraBits = SrcBitWidth - VTBits;
7923 if (Common > SrcExtraBits)
7924 return (Common - SrcExtraBits);
7925 return 1;
7926 }
7927 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
7928 return Common;
7929 }
7930
7931 unsigned
ComputeNumSignBitsForTargetNode(SDValue Op,const APInt & DemandedElts,const SelectionDAG & DAG,unsigned Depth) const7932 SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
7933 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
7934 unsigned Depth) const {
7935 if (Op.getResNo() != 0)
7936 return 1;
7937 unsigned Opcode = Op.getOpcode();
7938 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
7939 unsigned Id = Op.getConstantOperandVal(0);
7940 switch (Id) {
7941 case Intrinsic::s390_vpksh: // PACKS
7942 case Intrinsic::s390_vpksf:
7943 case Intrinsic::s390_vpksg:
7944 case Intrinsic::s390_vpkshs: // PACKS_CC
7945 case Intrinsic::s390_vpksfs:
7946 case Intrinsic::s390_vpksgs:
7947 case Intrinsic::s390_vpklsh: // PACKLS
7948 case Intrinsic::s390_vpklsf:
7949 case Intrinsic::s390_vpklsg:
7950 case Intrinsic::s390_vpklshs: // PACKLS_CC
7951 case Intrinsic::s390_vpklsfs:
7952 case Intrinsic::s390_vpklsgs:
7953 case Intrinsic::s390_vpdi:
7954 case Intrinsic::s390_vsldb:
7955 case Intrinsic::s390_vperm:
7956 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 1);
7957 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
7958 case Intrinsic::s390_vuphh:
7959 case Intrinsic::s390_vuphf:
7960 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
7961 case Intrinsic::s390_vuplhw:
7962 case Intrinsic::s390_vuplf: {
7963 SDValue PackedOp = Op.getOperand(1);
7964 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 1);
7965 unsigned Tmp = DAG.ComputeNumSignBits(PackedOp, SrcDemE, Depth + 1);
7966 EVT VT = Op.getValueType();
7967 unsigned VTBits = VT.getScalarSizeInBits();
7968 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
7969 return Tmp;
7970 }
7971 default:
7972 break;
7973 }
7974 } else {
7975 switch (Opcode) {
7976 case SystemZISD::SELECT_CCMASK:
7977 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 0);
7978 default:
7979 break;
7980 }
7981 }
7982
7983 return 1;
7984 }
7985
7986 bool SystemZTargetLowering::
isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op,const APInt & DemandedElts,const SelectionDAG & DAG,bool PoisonOnly,unsigned Depth) const7987 isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op,
7988 const APInt &DemandedElts, const SelectionDAG &DAG,
7989 bool PoisonOnly, unsigned Depth) const {
7990 switch (Op->getOpcode()) {
7991 case SystemZISD::PCREL_WRAPPER:
7992 case SystemZISD::PCREL_OFFSET:
7993 return true;
7994 }
7995 return false;
7996 }
7997
7998 unsigned
getStackProbeSize(const MachineFunction & MF) const7999 SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
8000 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
8001 unsigned StackAlign = TFI->getStackAlignment();
8002 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
8003 "Unexpected stack alignment");
8004 // The default stack probe size is 4096 if the function has no
8005 // stack-probe-size attribute.
8006 unsigned StackProbeSize =
8007 MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size", 4096);
8008 // Round down to the stack alignment.
8009 StackProbeSize &= ~(StackAlign - 1);
8010 return StackProbeSize ? StackProbeSize : StackAlign;
8011 }
8012
8013 //===----------------------------------------------------------------------===//
8014 // Custom insertion
8015 //===----------------------------------------------------------------------===//
8016
8017 // Force base value Base into a register before MI. Return the register.
forceReg(MachineInstr & MI,MachineOperand & Base,const SystemZInstrInfo * TII)8018 static Register forceReg(MachineInstr &MI, MachineOperand &Base,
8019 const SystemZInstrInfo *TII) {
8020 MachineBasicBlock *MBB = MI.getParent();
8021 MachineFunction &MF = *MBB->getParent();
8022 MachineRegisterInfo &MRI = MF.getRegInfo();
8023
8024 if (Base.isReg()) {
8025 // Copy Base into a new virtual register to help register coalescing in
8026 // cases with multiple uses.
8027 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8028 BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::COPY), Reg)
8029 .add(Base);
8030 return Reg;
8031 }
8032
8033 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8034 BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LA), Reg)
8035 .add(Base)
8036 .addImm(0)
8037 .addReg(0);
8038 return Reg;
8039 }
8040
8041 // The CC operand of MI might be missing a kill marker because there
8042 // were multiple uses of CC, and ISel didn't know which to mark.
8043 // Figure out whether MI should have had a kill marker.
checkCCKill(MachineInstr & MI,MachineBasicBlock * MBB)8044 static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
8045 // Scan forward through BB for a use/def of CC.
8046 MachineBasicBlock::iterator miI(std::next(MachineBasicBlock::iterator(MI)));
8047 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
8048 const MachineInstr& mi = *miI;
8049 if (mi.readsRegister(SystemZ::CC))
8050 return false;
8051 if (mi.definesRegister(SystemZ::CC))
8052 break; // Should have kill-flag - update below.
8053 }
8054
8055 // If we hit the end of the block, check whether CC is live into a
8056 // successor.
8057 if (miI == MBB->end()) {
8058 for (const MachineBasicBlock *Succ : MBB->successors())
8059 if (Succ->isLiveIn(SystemZ::CC))
8060 return false;
8061 }
8062
8063 return true;
8064 }
8065
8066 // Return true if it is OK for this Select pseudo-opcode to be cascaded
8067 // together with other Select pseudo-opcodes into a single basic-block with
8068 // a conditional jump around it.
isSelectPseudo(MachineInstr & MI)8069 static bool isSelectPseudo(MachineInstr &MI) {
8070 switch (MI.getOpcode()) {
8071 case SystemZ::Select32:
8072 case SystemZ::Select64:
8073 case SystemZ::Select128:
8074 case SystemZ::SelectF32:
8075 case SystemZ::SelectF64:
8076 case SystemZ::SelectF128:
8077 case SystemZ::SelectVR32:
8078 case SystemZ::SelectVR64:
8079 case SystemZ::SelectVR128:
8080 return true;
8081
8082 default:
8083 return false;
8084 }
8085 }
8086
8087 // Helper function, which inserts PHI functions into SinkMBB:
8088 // %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
8089 // where %FalseValue(i) and %TrueValue(i) are taken from Selects.
createPHIsForSelects(SmallVector<MachineInstr *,8> & Selects,MachineBasicBlock * TrueMBB,MachineBasicBlock * FalseMBB,MachineBasicBlock * SinkMBB)8090 static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
8091 MachineBasicBlock *TrueMBB,
8092 MachineBasicBlock *FalseMBB,
8093 MachineBasicBlock *SinkMBB) {
8094 MachineFunction *MF = TrueMBB->getParent();
8095 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
8096
8097 MachineInstr *FirstMI = Selects.front();
8098 unsigned CCValid = FirstMI->getOperand(3).getImm();
8099 unsigned CCMask = FirstMI->getOperand(4).getImm();
8100
8101 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
8102
8103 // As we are creating the PHIs, we have to be careful if there is more than
8104 // one. Later Selects may reference the results of earlier Selects, but later
8105 // PHIs have to reference the individual true/false inputs from earlier PHIs.
8106 // That also means that PHI construction must work forward from earlier to
8107 // later, and that the code must maintain a mapping from earlier PHI's
8108 // destination registers, and the registers that went into the PHI.
8109 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
8110
8111 for (auto *MI : Selects) {
8112 Register DestReg = MI->getOperand(0).getReg();
8113 Register TrueReg = MI->getOperand(1).getReg();
8114 Register FalseReg = MI->getOperand(2).getReg();
8115
8116 // If this Select we are generating is the opposite condition from
8117 // the jump we generated, then we have to swap the operands for the
8118 // PHI that is going to be generated.
8119 if (MI->getOperand(4).getImm() == (CCValid ^ CCMask))
8120 std::swap(TrueReg, FalseReg);
8121
8122 if (RegRewriteTable.contains(TrueReg))
8123 TrueReg = RegRewriteTable[TrueReg].first;
8124
8125 if (RegRewriteTable.contains(FalseReg))
8126 FalseReg = RegRewriteTable[FalseReg].second;
8127
8128 DebugLoc DL = MI->getDebugLoc();
8129 BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)
8130 .addReg(TrueReg).addMBB(TrueMBB)
8131 .addReg(FalseReg).addMBB(FalseMBB);
8132
8133 // Add this PHI to the rewrite table.
8134 RegRewriteTable[DestReg] = std::make_pair(TrueReg, FalseReg);
8135 }
8136
8137 MF->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
8138 }
8139
8140 // Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
8141 MachineBasicBlock *
emitSelect(MachineInstr & MI,MachineBasicBlock * MBB) const8142 SystemZTargetLowering::emitSelect(MachineInstr &MI,
8143 MachineBasicBlock *MBB) const {
8144 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
8145 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8146
8147 unsigned CCValid = MI.getOperand(3).getImm();
8148 unsigned CCMask = MI.getOperand(4).getImm();
8149
8150 // If we have a sequence of Select* pseudo instructions using the
8151 // same condition code value, we want to expand all of them into
8152 // a single pair of basic blocks using the same condition.
8153 SmallVector<MachineInstr*, 8> Selects;
8154 SmallVector<MachineInstr*, 8> DbgValues;
8155 Selects.push_back(&MI);
8156 unsigned Count = 0;
8157 for (MachineInstr &NextMI : llvm::make_range(
8158 std::next(MachineBasicBlock::iterator(MI)), MBB->end())) {
8159 if (isSelectPseudo(NextMI)) {
8160 assert(NextMI.getOperand(3).getImm() == CCValid &&
8161 "Bad CCValid operands since CC was not redefined.");
8162 if (NextMI.getOperand(4).getImm() == CCMask ||
8163 NextMI.getOperand(4).getImm() == (CCValid ^ CCMask)) {
8164 Selects.push_back(&NextMI);
8165 continue;
8166 }
8167 break;
8168 }
8169 if (NextMI.definesRegister(SystemZ::CC) || NextMI.usesCustomInsertionHook())
8170 break;
8171 bool User = false;
8172 for (auto *SelMI : Selects)
8173 if (NextMI.readsVirtualRegister(SelMI->getOperand(0).getReg())) {
8174 User = true;
8175 break;
8176 }
8177 if (NextMI.isDebugInstr()) {
8178 if (User) {
8179 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
8180 DbgValues.push_back(&NextMI);
8181 }
8182 } else if (User || ++Count > 20)
8183 break;
8184 }
8185
8186 MachineInstr *LastMI = Selects.back();
8187 bool CCKilled =
8188 (LastMI->killsRegister(SystemZ::CC) || checkCCKill(*LastMI, MBB));
8189 MachineBasicBlock *StartMBB = MBB;
8190 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(LastMI, MBB);
8191 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(StartMBB);
8192
8193 // Unless CC was killed in the last Select instruction, mark it as
8194 // live-in to both FalseMBB and JoinMBB.
8195 if (!CCKilled) {
8196 FalseMBB->addLiveIn(SystemZ::CC);
8197 JoinMBB->addLiveIn(SystemZ::CC);
8198 }
8199
8200 // StartMBB:
8201 // BRC CCMask, JoinMBB
8202 // # fallthrough to FalseMBB
8203 MBB = StartMBB;
8204 BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))
8205 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
8206 MBB->addSuccessor(JoinMBB);
8207 MBB->addSuccessor(FalseMBB);
8208
8209 // FalseMBB:
8210 // # fallthrough to JoinMBB
8211 MBB = FalseMBB;
8212 MBB->addSuccessor(JoinMBB);
8213
8214 // JoinMBB:
8215 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
8216 // ...
8217 MBB = JoinMBB;
8218 createPHIsForSelects(Selects, StartMBB, FalseMBB, MBB);
8219 for (auto *SelMI : Selects)
8220 SelMI->eraseFromParent();
8221
8222 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
8223 for (auto *DbgMI : DbgValues)
8224 MBB->splice(InsertPos, StartMBB, DbgMI);
8225
8226 return JoinMBB;
8227 }
8228
8229 // Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
8230 // StoreOpcode is the store to use and Invert says whether the store should
8231 // happen when the condition is false rather than true. If a STORE ON
8232 // CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
emitCondStore(MachineInstr & MI,MachineBasicBlock * MBB,unsigned StoreOpcode,unsigned STOCOpcode,bool Invert) const8233 MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
8234 MachineBasicBlock *MBB,
8235 unsigned StoreOpcode,
8236 unsigned STOCOpcode,
8237 bool Invert) const {
8238 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8239
8240 Register SrcReg = MI.getOperand(0).getReg();
8241 MachineOperand Base = MI.getOperand(1);
8242 int64_t Disp = MI.getOperand(2).getImm();
8243 Register IndexReg = MI.getOperand(3).getReg();
8244 unsigned CCValid = MI.getOperand(4).getImm();
8245 unsigned CCMask = MI.getOperand(5).getImm();
8246 DebugLoc DL = MI.getDebugLoc();
8247
8248 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp);
8249
8250 // ISel pattern matching also adds a load memory operand of the same
8251 // address, so take special care to find the storing memory operand.
8252 MachineMemOperand *MMO = nullptr;
8253 for (auto *I : MI.memoperands())
8254 if (I->isStore()) {
8255 MMO = I;
8256 break;
8257 }
8258
8259 // Use STOCOpcode if possible. We could use different store patterns in
8260 // order to avoid matching the index register, but the performance trade-offs
8261 // might be more complicated in that case.
8262 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
8263 if (Invert)
8264 CCMask ^= CCValid;
8265
8266 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))
8267 .addReg(SrcReg)
8268 .add(Base)
8269 .addImm(Disp)
8270 .addImm(CCValid)
8271 .addImm(CCMask)
8272 .addMemOperand(MMO);
8273
8274 MI.eraseFromParent();
8275 return MBB;
8276 }
8277
8278 // Get the condition needed to branch around the store.
8279 if (!Invert)
8280 CCMask ^= CCValid;
8281
8282 MachineBasicBlock *StartMBB = MBB;
8283 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
8284 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(StartMBB);
8285
8286 // Unless CC was killed in the CondStore instruction, mark it as
8287 // live-in to both FalseMBB and JoinMBB.
8288 if (!MI.killsRegister(SystemZ::CC) && !checkCCKill(MI, JoinMBB)) {
8289 FalseMBB->addLiveIn(SystemZ::CC);
8290 JoinMBB->addLiveIn(SystemZ::CC);
8291 }
8292
8293 // StartMBB:
8294 // BRC CCMask, JoinMBB
8295 // # fallthrough to FalseMBB
8296 MBB = StartMBB;
8297 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8298 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
8299 MBB->addSuccessor(JoinMBB);
8300 MBB->addSuccessor(FalseMBB);
8301
8302 // FalseMBB:
8303 // store %SrcReg, %Disp(%Index,%Base)
8304 // # fallthrough to JoinMBB
8305 MBB = FalseMBB;
8306 BuildMI(MBB, DL, TII->get(StoreOpcode))
8307 .addReg(SrcReg)
8308 .add(Base)
8309 .addImm(Disp)
8310 .addReg(IndexReg)
8311 .addMemOperand(MMO);
8312 MBB->addSuccessor(JoinMBB);
8313
8314 MI.eraseFromParent();
8315 return JoinMBB;
8316 }
8317
8318 // Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
8319 MachineBasicBlock *
emitICmp128Hi(MachineInstr & MI,MachineBasicBlock * MBB,bool Unsigned) const8320 SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
8321 MachineBasicBlock *MBB,
8322 bool Unsigned) const {
8323 MachineFunction &MF = *MBB->getParent();
8324 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8325 MachineRegisterInfo &MRI = MF.getRegInfo();
8326
8327 // Synthetic instruction to compare 128-bit values.
8328 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
8329 Register Op0 = MI.getOperand(0).getReg();
8330 Register Op1 = MI.getOperand(1).getReg();
8331
8332 MachineBasicBlock *StartMBB = MBB;
8333 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
8334 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(StartMBB);
8335
8336 // StartMBB:
8337 //
8338 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
8339 // Swap the inputs to get:
8340 // CC 1 if high(Op0) > high(Op1)
8341 // CC 2 if high(Op0) < high(Op1)
8342 // CC 0 if high(Op0) == high(Op1)
8343 //
8344 // If CC != 0, we'd done, so jump over the next instruction.
8345 //
8346 // VEC[L]G Op1, Op0
8347 // JNE JoinMBB
8348 // # fallthrough to HiEqMBB
8349 MBB = StartMBB;
8350 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
8351 BuildMI(MBB, MI.getDebugLoc(), TII->get(HiOpcode))
8352 .addReg(Op1).addReg(Op0);
8353 BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))
8354 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE).addMBB(JoinMBB);
8355 MBB->addSuccessor(JoinMBB);
8356 MBB->addSuccessor(HiEqMBB);
8357
8358 // HiEqMBB:
8359 //
8360 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
8361 // Since we already know the high parts are equal, the CC
8362 // result will only depend on the low parts:
8363 // CC 1 if low(Op0) > low(Op1)
8364 // CC 3 if low(Op0) <= low(Op1)
8365 //
8366 // VCHLGS Tmp, Op0, Op1
8367 // # fallthrough to JoinMBB
8368 MBB = HiEqMBB;
8369 Register Temp = MRI.createVirtualRegister(&SystemZ::VR128BitRegClass);
8370 BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::VCHLGS), Temp)
8371 .addReg(Op0).addReg(Op1);
8372 MBB->addSuccessor(JoinMBB);
8373
8374 // Mark CC as live-in to JoinMBB.
8375 JoinMBB->addLiveIn(SystemZ::CC);
8376
8377 MI.eraseFromParent();
8378 return JoinMBB;
8379 }
8380
8381 // Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
8382 // ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
8383 // the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
8384 // whether the field should be inverted after performing BinOpcode (e.g. for
8385 // NAND).
emitAtomicLoadBinary(MachineInstr & MI,MachineBasicBlock * MBB,unsigned BinOpcode,bool Invert) const8386 MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
8387 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
8388 bool Invert) const {
8389 MachineFunction &MF = *MBB->getParent();
8390 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8391 MachineRegisterInfo &MRI = MF.getRegInfo();
8392
8393 // Extract the operands. Base can be a register or a frame index.
8394 // Src2 can be a register or immediate.
8395 Register Dest = MI.getOperand(0).getReg();
8396 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
8397 int64_t Disp = MI.getOperand(2).getImm();
8398 MachineOperand Src2 = earlyUseOperand(MI.getOperand(3));
8399 Register BitShift = MI.getOperand(4).getReg();
8400 Register NegBitShift = MI.getOperand(5).getReg();
8401 unsigned BitSize = MI.getOperand(6).getImm();
8402 DebugLoc DL = MI.getDebugLoc();
8403
8404 // Get the right opcodes for the displacement.
8405 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
8406 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
8407 assert(LOpcode && CSOpcode && "Displacement out of range");
8408
8409 // Create virtual registers for temporary results.
8410 Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8411 Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8412 Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8413 Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8414 Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8415
8416 // Insert a basic block for the main loop.
8417 MachineBasicBlock *StartMBB = MBB;
8418 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
8419 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
8420
8421 // StartMBB:
8422 // ...
8423 // %OrigVal = L Disp(%Base)
8424 // # fall through to LoopMBB
8425 MBB = StartMBB;
8426 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
8427 MBB->addSuccessor(LoopMBB);
8428
8429 // LoopMBB:
8430 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
8431 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
8432 // %RotatedNewVal = OP %RotatedOldVal, %Src2
8433 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
8434 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
8435 // JNE LoopMBB
8436 // # fall through to DoneMBB
8437 MBB = LoopMBB;
8438 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
8439 .addReg(OrigVal).addMBB(StartMBB)
8440 .addReg(Dest).addMBB(LoopMBB);
8441 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
8442 .addReg(OldVal).addReg(BitShift).addImm(0);
8443 if (Invert) {
8444 // Perform the operation normally and then invert every bit of the field.
8445 Register Tmp = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8446 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2);
8447 // XILF with the upper BitSize bits set.
8448 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)
8449 .addReg(Tmp).addImm(-1U << (32 - BitSize));
8450 } else if (BinOpcode)
8451 // A simply binary operation.
8452 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal)
8453 .addReg(RotatedOldVal)
8454 .add(Src2);
8455 else
8456 // Use RISBG to rotate Src2 into position and use it to replace the
8457 // field in RotatedOldVal.
8458 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)
8459 .addReg(RotatedOldVal).addReg(Src2.getReg())
8460 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize);
8461 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
8462 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
8463 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
8464 .addReg(OldVal)
8465 .addReg(NewVal)
8466 .add(Base)
8467 .addImm(Disp);
8468 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8469 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
8470 MBB->addSuccessor(LoopMBB);
8471 MBB->addSuccessor(DoneMBB);
8472
8473 MI.eraseFromParent();
8474 return DoneMBB;
8475 }
8476
8477 // Implement EmitInstrWithCustomInserter for subword pseudo
8478 // ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
8479 // instruction that should be used to compare the current field with the
8480 // minimum or maximum value. KeepOldMask is the BRC condition-code mask
8481 // for when the current field should be kept.
emitAtomicLoadMinMax(MachineInstr & MI,MachineBasicBlock * MBB,unsigned CompareOpcode,unsigned KeepOldMask) const8482 MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
8483 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
8484 unsigned KeepOldMask) const {
8485 MachineFunction &MF = *MBB->getParent();
8486 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8487 MachineRegisterInfo &MRI = MF.getRegInfo();
8488
8489 // Extract the operands. Base can be a register or a frame index.
8490 Register Dest = MI.getOperand(0).getReg();
8491 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
8492 int64_t Disp = MI.getOperand(2).getImm();
8493 Register Src2 = MI.getOperand(3).getReg();
8494 Register BitShift = MI.getOperand(4).getReg();
8495 Register NegBitShift = MI.getOperand(5).getReg();
8496 unsigned BitSize = MI.getOperand(6).getImm();
8497 DebugLoc DL = MI.getDebugLoc();
8498
8499 // Get the right opcodes for the displacement.
8500 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
8501 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
8502 assert(LOpcode && CSOpcode && "Displacement out of range");
8503
8504 // Create virtual registers for temporary results.
8505 Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8506 Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8507 Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8508 Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8509 Register RotatedAltVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8510 Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
8511
8512 // Insert 3 basic blocks for the loop.
8513 MachineBasicBlock *StartMBB = MBB;
8514 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
8515 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
8516 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(LoopMBB);
8517 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(UseAltMBB);
8518
8519 // StartMBB:
8520 // ...
8521 // %OrigVal = L Disp(%Base)
8522 // # fall through to LoopMBB
8523 MBB = StartMBB;
8524 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
8525 MBB->addSuccessor(LoopMBB);
8526
8527 // LoopMBB:
8528 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
8529 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
8530 // CompareOpcode %RotatedOldVal, %Src2
8531 // BRC KeepOldMask, UpdateMBB
8532 MBB = LoopMBB;
8533 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
8534 .addReg(OrigVal).addMBB(StartMBB)
8535 .addReg(Dest).addMBB(UpdateMBB);
8536 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
8537 .addReg(OldVal).addReg(BitShift).addImm(0);
8538 BuildMI(MBB, DL, TII->get(CompareOpcode))
8539 .addReg(RotatedOldVal).addReg(Src2);
8540 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8541 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB);
8542 MBB->addSuccessor(UpdateMBB);
8543 MBB->addSuccessor(UseAltMBB);
8544
8545 // UseAltMBB:
8546 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
8547 // # fall through to UpdateMBB
8548 MBB = UseAltMBB;
8549 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
8550 .addReg(RotatedOldVal).addReg(Src2)
8551 .addImm(32).addImm(31 + BitSize).addImm(0);
8552 MBB->addSuccessor(UpdateMBB);
8553
8554 // UpdateMBB:
8555 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
8556 // [ %RotatedAltVal, UseAltMBB ]
8557 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
8558 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
8559 // JNE LoopMBB
8560 // # fall through to DoneMBB
8561 MBB = UpdateMBB;
8562 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
8563 .addReg(RotatedOldVal).addMBB(LoopMBB)
8564 .addReg(RotatedAltVal).addMBB(UseAltMBB);
8565 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
8566 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
8567 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
8568 .addReg(OldVal)
8569 .addReg(NewVal)
8570 .add(Base)
8571 .addImm(Disp);
8572 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8573 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
8574 MBB->addSuccessor(LoopMBB);
8575 MBB->addSuccessor(DoneMBB);
8576
8577 MI.eraseFromParent();
8578 return DoneMBB;
8579 }
8580
8581 // Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
8582 // instruction MI.
8583 MachineBasicBlock *
emitAtomicCmpSwapW(MachineInstr & MI,MachineBasicBlock * MBB) const8584 SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
8585 MachineBasicBlock *MBB) const {
8586 MachineFunction &MF = *MBB->getParent();
8587 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8588 MachineRegisterInfo &MRI = MF.getRegInfo();
8589
8590 // Extract the operands. Base can be a register or a frame index.
8591 Register Dest = MI.getOperand(0).getReg();
8592 MachineOperand Base = earlyUseOperand(MI.getOperand(1));
8593 int64_t Disp = MI.getOperand(2).getImm();
8594 Register CmpVal = MI.getOperand(3).getReg();
8595 Register OrigSwapVal = MI.getOperand(4).getReg();
8596 Register BitShift = MI.getOperand(5).getReg();
8597 Register NegBitShift = MI.getOperand(6).getReg();
8598 int64_t BitSize = MI.getOperand(7).getImm();
8599 DebugLoc DL = MI.getDebugLoc();
8600
8601 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
8602
8603 // Get the right opcodes for the displacement and zero-extension.
8604 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
8605 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
8606 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
8607 assert(LOpcode && CSOpcode && "Displacement out of range");
8608
8609 // Create virtual registers for temporary results.
8610 Register OrigOldVal = MRI.createVirtualRegister(RC);
8611 Register OldVal = MRI.createVirtualRegister(RC);
8612 Register SwapVal = MRI.createVirtualRegister(RC);
8613 Register StoreVal = MRI.createVirtualRegister(RC);
8614 Register OldValRot = MRI.createVirtualRegister(RC);
8615 Register RetryOldVal = MRI.createVirtualRegister(RC);
8616 Register RetrySwapVal = MRI.createVirtualRegister(RC);
8617
8618 // Insert 2 basic blocks for the loop.
8619 MachineBasicBlock *StartMBB = MBB;
8620 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
8621 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
8622 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(LoopMBB);
8623
8624 // StartMBB:
8625 // ...
8626 // %OrigOldVal = L Disp(%Base)
8627 // # fall through to LoopMBB
8628 MBB = StartMBB;
8629 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)
8630 .add(Base)
8631 .addImm(Disp)
8632 .addReg(0);
8633 MBB->addSuccessor(LoopMBB);
8634
8635 // LoopMBB:
8636 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
8637 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
8638 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
8639 // ^^ The low BitSize bits contain the field
8640 // of interest.
8641 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
8642 // ^^ Replace the upper 32-BitSize bits of the
8643 // swap value with those that we loaded and rotated.
8644 // %Dest = LL[CH] %OldValRot
8645 // CR %Dest, %CmpVal
8646 // JNE DoneMBB
8647 // # Fall through to SetMBB
8648 MBB = LoopMBB;
8649 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
8650 .addReg(OrigOldVal).addMBB(StartMBB)
8651 .addReg(RetryOldVal).addMBB(SetMBB);
8652 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
8653 .addReg(OrigSwapVal).addMBB(StartMBB)
8654 .addReg(RetrySwapVal).addMBB(SetMBB);
8655 BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
8656 .addReg(OldVal).addReg(BitShift).addImm(BitSize);
8657 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
8658 .addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);
8659 BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
8660 .addReg(OldValRot);
8661 BuildMI(MBB, DL, TII->get(SystemZ::CR))
8662 .addReg(Dest).addReg(CmpVal);
8663 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8664 .addImm(SystemZ::CCMASK_ICMP)
8665 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
8666 MBB->addSuccessor(DoneMBB);
8667 MBB->addSuccessor(SetMBB);
8668
8669 // SetMBB:
8670 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
8671 // ^^ Rotate the new field to its proper position.
8672 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
8673 // JNE LoopMBB
8674 // # fall through to ExitMBB
8675 MBB = SetMBB;
8676 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
8677 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
8678 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
8679 .addReg(OldVal)
8680 .addReg(StoreVal)
8681 .add(Base)
8682 .addImm(Disp);
8683 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8684 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
8685 MBB->addSuccessor(LoopMBB);
8686 MBB->addSuccessor(DoneMBB);
8687
8688 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
8689 // to the block after the loop. At this point, CC may have been defined
8690 // either by the CR in LoopMBB or by the CS in SetMBB.
8691 if (!MI.registerDefIsDead(SystemZ::CC))
8692 DoneMBB->addLiveIn(SystemZ::CC);
8693
8694 MI.eraseFromParent();
8695 return DoneMBB;
8696 }
8697
8698 // Emit a move from two GR64s to a GR128.
8699 MachineBasicBlock *
emitPair128(MachineInstr & MI,MachineBasicBlock * MBB) const8700 SystemZTargetLowering::emitPair128(MachineInstr &MI,
8701 MachineBasicBlock *MBB) const {
8702 MachineFunction &MF = *MBB->getParent();
8703 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8704 MachineRegisterInfo &MRI = MF.getRegInfo();
8705 DebugLoc DL = MI.getDebugLoc();
8706
8707 Register Dest = MI.getOperand(0).getReg();
8708 Register Hi = MI.getOperand(1).getReg();
8709 Register Lo = MI.getOperand(2).getReg();
8710 Register Tmp1 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
8711 Register Tmp2 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
8712
8713 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Tmp1);
8714 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Tmp2)
8715 .addReg(Tmp1).addReg(Hi).addImm(SystemZ::subreg_h64);
8716 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
8717 .addReg(Tmp2).addReg(Lo).addImm(SystemZ::subreg_l64);
8718
8719 MI.eraseFromParent();
8720 return MBB;
8721 }
8722
8723 // Emit an extension from a GR64 to a GR128. ClearEven is true
8724 // if the high register of the GR128 value must be cleared or false if
8725 // it's "don't care".
emitExt128(MachineInstr & MI,MachineBasicBlock * MBB,bool ClearEven) const8726 MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
8727 MachineBasicBlock *MBB,
8728 bool ClearEven) const {
8729 MachineFunction &MF = *MBB->getParent();
8730 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8731 MachineRegisterInfo &MRI = MF.getRegInfo();
8732 DebugLoc DL = MI.getDebugLoc();
8733
8734 Register Dest = MI.getOperand(0).getReg();
8735 Register Src = MI.getOperand(1).getReg();
8736 Register In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
8737
8738 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128);
8739 if (ClearEven) {
8740 Register NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
8741 Register Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
8742
8743 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64)
8744 .addImm(0);
8745 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128)
8746 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64);
8747 In128 = NewIn128;
8748 }
8749 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
8750 .addReg(In128).addReg(Src).addImm(SystemZ::subreg_l64);
8751
8752 MI.eraseFromParent();
8753 return MBB;
8754 }
8755
8756 MachineBasicBlock *
emitMemMemWrapper(MachineInstr & MI,MachineBasicBlock * MBB,unsigned Opcode,bool IsMemset) const8757 SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
8758 MachineBasicBlock *MBB,
8759 unsigned Opcode, bool IsMemset) const {
8760 MachineFunction &MF = *MBB->getParent();
8761 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
8762 MachineRegisterInfo &MRI = MF.getRegInfo();
8763 DebugLoc DL = MI.getDebugLoc();
8764
8765 MachineOperand DestBase = earlyUseOperand(MI.getOperand(0));
8766 uint64_t DestDisp = MI.getOperand(1).getImm();
8767 MachineOperand SrcBase = MachineOperand::CreateReg(0U, false);
8768 uint64_t SrcDisp;
8769
8770 // Fold the displacement Disp if it is out of range.
8771 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
8772 if (!isUInt<12>(Disp)) {
8773 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8774 unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
8775 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode), Reg)
8776 .add(Base).addImm(Disp).addReg(0);
8777 Base = MachineOperand::CreateReg(Reg, false);
8778 Disp = 0;
8779 }
8780 };
8781
8782 if (!IsMemset) {
8783 SrcBase = earlyUseOperand(MI.getOperand(2));
8784 SrcDisp = MI.getOperand(3).getImm();
8785 } else {
8786 SrcBase = DestBase;
8787 SrcDisp = DestDisp++;
8788 foldDisplIfNeeded(DestBase, DestDisp);
8789 }
8790
8791 MachineOperand &LengthMO = MI.getOperand(IsMemset ? 2 : 4);
8792 bool IsImmForm = LengthMO.isImm();
8793 bool IsRegForm = !IsImmForm;
8794
8795 // Build and insert one Opcode of Length, with special treatment for memset.
8796 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
8797 MachineBasicBlock::iterator InsPos,
8798 MachineOperand DBase, uint64_t DDisp,
8799 MachineOperand SBase, uint64_t SDisp,
8800 unsigned Length) -> void {
8801 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
8802 if (IsMemset) {
8803 MachineOperand ByteMO = earlyUseOperand(MI.getOperand(3));
8804 if (ByteMO.isImm())
8805 BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI))
8806 .add(SBase).addImm(SDisp).add(ByteMO);
8807 else
8808 BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC))
8809 .add(ByteMO).add(SBase).addImm(SDisp).addReg(0);
8810 if (--Length == 0)
8811 return;
8812 }
8813 BuildMI(*MBB, InsPos, DL, TII->get(Opcode))
8814 .add(DBase).addImm(DDisp).addImm(Length)
8815 .add(SBase).addImm(SDisp)
8816 .setMemRefs(MI.memoperands());
8817 };
8818
8819 bool NeedsLoop = false;
8820 uint64_t ImmLength = 0;
8821 Register LenAdjReg = SystemZ::NoRegister;
8822 if (IsImmForm) {
8823 ImmLength = LengthMO.getImm();
8824 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
8825 if (ImmLength == 0) {
8826 MI.eraseFromParent();
8827 return MBB;
8828 }
8829 if (Opcode == SystemZ::CLC) {
8830 if (ImmLength > 3 * 256)
8831 // A two-CLC sequence is a clear win over a loop, not least because
8832 // it needs only one branch. A three-CLC sequence needs the same
8833 // number of branches as a loop (i.e. 2), but is shorter. That
8834 // brings us to lengths greater than 768 bytes. It seems relatively
8835 // likely that a difference will be found within the first 768 bytes,
8836 // so we just optimize for the smallest number of branch
8837 // instructions, in order to avoid polluting the prediction buffer
8838 // too much.
8839 NeedsLoop = true;
8840 } else if (ImmLength > 6 * 256)
8841 // The heuristic we use is to prefer loops for anything that would
8842 // require 7 or more MVCs. With these kinds of sizes there isn't much
8843 // to choose between straight-line code and looping code, since the
8844 // time will be dominated by the MVCs themselves.
8845 NeedsLoop = true;
8846 } else {
8847 NeedsLoop = true;
8848 LenAdjReg = LengthMO.getReg();
8849 }
8850
8851 // When generating more than one CLC, all but the last will need to
8852 // branch to the end when a difference is found.
8853 MachineBasicBlock *EndMBB =
8854 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
8855 ? SystemZ::splitBlockAfter(MI, MBB)
8856 : nullptr);
8857
8858 if (NeedsLoop) {
8859 Register StartCountReg =
8860 MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
8861 if (IsImmForm) {
8862 TII->loadImmediate(*MBB, MI, StartCountReg, ImmLength / 256);
8863 ImmLength &= 255;
8864 } else {
8865 BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg)
8866 .addReg(LenAdjReg)
8867 .addReg(0)
8868 .addImm(8);
8869 }
8870
8871 bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
8872 auto loadZeroAddress = [&]() -> MachineOperand {
8873 Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
8874 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);
8875 return MachineOperand::CreateReg(Reg, false);
8876 };
8877 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
8878 DestBase = loadZeroAddress();
8879 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
8880 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
8881
8882 MachineBasicBlock *StartMBB = nullptr;
8883 MachineBasicBlock *LoopMBB = nullptr;
8884 MachineBasicBlock *NextMBB = nullptr;
8885 MachineBasicBlock *DoneMBB = nullptr;
8886 MachineBasicBlock *AllDoneMBB = nullptr;
8887
8888 Register StartSrcReg = forceReg(MI, SrcBase, TII);
8889 Register StartDestReg =
8890 (HaveSingleBase ? StartSrcReg : forceReg(MI, DestBase, TII));
8891
8892 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
8893 Register ThisSrcReg = MRI.createVirtualRegister(RC);
8894 Register ThisDestReg =
8895 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RC));
8896 Register NextSrcReg = MRI.createVirtualRegister(RC);
8897 Register NextDestReg =
8898 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RC));
8899 RC = &SystemZ::GR64BitRegClass;
8900 Register ThisCountReg = MRI.createVirtualRegister(RC);
8901 Register NextCountReg = MRI.createVirtualRegister(RC);
8902
8903 if (IsRegForm) {
8904 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
8905 StartMBB = SystemZ::emitBlockAfter(MBB);
8906 LoopMBB = SystemZ::emitBlockAfter(StartMBB);
8907 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
8908 DoneMBB = SystemZ::emitBlockAfter(NextMBB);
8909
8910 // MBB:
8911 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
8912 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
8913 .addReg(LenAdjReg).addImm(IsMemset ? -2 : -1);
8914 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8915 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
8916 .addMBB(AllDoneMBB);
8917 MBB->addSuccessor(AllDoneMBB);
8918 if (!IsMemset)
8919 MBB->addSuccessor(StartMBB);
8920 else {
8921 // MemsetOneCheckMBB:
8922 // # Jump to MemsetOneMBB for a memset of length 1, or
8923 // # fall thru to StartMBB.
8924 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
8925 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(&*MF.rbegin());
8926 MBB->addSuccessor(MemsetOneCheckMBB);
8927 MBB = MemsetOneCheckMBB;
8928 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
8929 .addReg(LenAdjReg).addImm(-1);
8930 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8931 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
8932 .addMBB(MemsetOneMBB);
8933 MBB->addSuccessor(MemsetOneMBB, {10, 100});
8934 MBB->addSuccessor(StartMBB, {90, 100});
8935
8936 // MemsetOneMBB:
8937 // # Jump back to AllDoneMBB after a single MVI or STC.
8938 MBB = MemsetOneMBB;
8939 insertMemMemOp(MBB, MBB->end(),
8940 MachineOperand::CreateReg(StartDestReg, false), DestDisp,
8941 MachineOperand::CreateReg(StartSrcReg, false), SrcDisp,
8942 1);
8943 BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB);
8944 MBB->addSuccessor(AllDoneMBB);
8945 }
8946
8947 // StartMBB:
8948 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
8949 MBB = StartMBB;
8950 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
8951 .addReg(StartCountReg).addImm(0);
8952 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
8953 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
8954 .addMBB(DoneMBB);
8955 MBB->addSuccessor(DoneMBB);
8956 MBB->addSuccessor(LoopMBB);
8957 }
8958 else {
8959 StartMBB = MBB;
8960 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
8961 LoopMBB = SystemZ::emitBlockAfter(StartMBB);
8962 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
8963
8964 // StartMBB:
8965 // # fall through to LoopMBB
8966 MBB->addSuccessor(LoopMBB);
8967
8968 DestBase = MachineOperand::CreateReg(NextDestReg, false);
8969 SrcBase = MachineOperand::CreateReg(NextSrcReg, false);
8970 if (EndMBB && !ImmLength)
8971 // If the loop handled the whole CLC range, DoneMBB will be empty with
8972 // CC live-through into EndMBB, so add it as live-in.
8973 DoneMBB->addLiveIn(SystemZ::CC);
8974 }
8975
8976 // LoopMBB:
8977 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
8978 // [ %NextDestReg, NextMBB ]
8979 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
8980 // [ %NextSrcReg, NextMBB ]
8981 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
8982 // [ %NextCountReg, NextMBB ]
8983 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
8984 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
8985 // ( JLH EndMBB )
8986 //
8987 // The prefetch is used only for MVC. The JLH is used only for CLC.
8988 MBB = LoopMBB;
8989 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)
8990 .addReg(StartDestReg).addMBB(StartMBB)
8991 .addReg(NextDestReg).addMBB(NextMBB);
8992 if (!HaveSingleBase)
8993 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg)
8994 .addReg(StartSrcReg).addMBB(StartMBB)
8995 .addReg(NextSrcReg).addMBB(NextMBB);
8996 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg)
8997 .addReg(StartCountReg).addMBB(StartMBB)
8998 .addReg(NextCountReg).addMBB(NextMBB);
8999 if (Opcode == SystemZ::MVC)
9000 BuildMI(MBB, DL, TII->get(SystemZ::PFD))
9001 .addImm(SystemZ::PFD_WRITE)
9002 .addReg(ThisDestReg).addImm(DestDisp - IsMemset + 768).addReg(0);
9003 insertMemMemOp(MBB, MBB->end(),
9004 MachineOperand::CreateReg(ThisDestReg, false), DestDisp,
9005 MachineOperand::CreateReg(ThisSrcReg, false), SrcDisp, 256);
9006 if (EndMBB) {
9007 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
9008 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
9009 .addMBB(EndMBB);
9010 MBB->addSuccessor(EndMBB);
9011 MBB->addSuccessor(NextMBB);
9012 }
9013
9014 // NextMBB:
9015 // %NextDestReg = LA 256(%ThisDestReg)
9016 // %NextSrcReg = LA 256(%ThisSrcReg)
9017 // %NextCountReg = AGHI %ThisCountReg, -1
9018 // CGHI %NextCountReg, 0
9019 // JLH LoopMBB
9020 // # fall through to DoneMBB
9021 //
9022 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
9023 MBB = NextMBB;
9024 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)
9025 .addReg(ThisDestReg).addImm(256).addReg(0);
9026 if (!HaveSingleBase)
9027 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg)
9028 .addReg(ThisSrcReg).addImm(256).addReg(0);
9029 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg)
9030 .addReg(ThisCountReg).addImm(-1);
9031 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
9032 .addReg(NextCountReg).addImm(0);
9033 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
9034 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
9035 .addMBB(LoopMBB);
9036 MBB->addSuccessor(LoopMBB);
9037 MBB->addSuccessor(DoneMBB);
9038
9039 MBB = DoneMBB;
9040 if (IsRegForm) {
9041 // DoneMBB:
9042 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
9043 // # Use EXecute Relative Long for the remainder of the bytes. The target
9044 // instruction of the EXRL will have a length field of 1 since 0 is an
9045 // illegal value. The number of bytes processed becomes (%LenAdjReg &
9046 // 0xff) + 1.
9047 // # Fall through to AllDoneMBB.
9048 Register RemSrcReg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9049 Register RemDestReg = HaveSingleBase ? RemSrcReg
9050 : MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9051 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)
9052 .addReg(StartDestReg).addMBB(StartMBB)
9053 .addReg(NextDestReg).addMBB(NextMBB);
9054 if (!HaveSingleBase)
9055 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
9056 .addReg(StartSrcReg).addMBB(StartMBB)
9057 .addReg(NextSrcReg).addMBB(NextMBB);
9058 if (IsMemset)
9059 insertMemMemOp(MBB, MBB->end(),
9060 MachineOperand::CreateReg(RemDestReg, false), DestDisp,
9061 MachineOperand::CreateReg(RemSrcReg, false), SrcDisp, 1);
9062 MachineInstrBuilder EXRL_MIB =
9063 BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
9064 .addImm(Opcode)
9065 .addReg(LenAdjReg)
9066 .addReg(RemDestReg).addImm(DestDisp)
9067 .addReg(RemSrcReg).addImm(SrcDisp);
9068 MBB->addSuccessor(AllDoneMBB);
9069 MBB = AllDoneMBB;
9070 if (Opcode != SystemZ::MVC) {
9071 EXRL_MIB.addReg(SystemZ::CC, RegState::ImplicitDefine);
9072 if (EndMBB)
9073 MBB->addLiveIn(SystemZ::CC);
9074 }
9075 }
9076 MF.getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
9077 }
9078
9079 // Handle any remaining bytes with straight-line code.
9080 while (ImmLength > 0) {
9081 uint64_t ThisLength = std::min(ImmLength, uint64_t(256));
9082 // The previous iteration might have created out-of-range displacements.
9083 // Apply them using LA/LAY if so.
9084 foldDisplIfNeeded(DestBase, DestDisp);
9085 foldDisplIfNeeded(SrcBase, SrcDisp);
9086 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
9087 DestDisp += ThisLength;
9088 SrcDisp += ThisLength;
9089 ImmLength -= ThisLength;
9090 // If there's another CLC to go, branch to the end if a difference
9091 // was found.
9092 if (EndMBB && ImmLength > 0) {
9093 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
9094 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
9095 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
9096 .addMBB(EndMBB);
9097 MBB->addSuccessor(EndMBB);
9098 MBB->addSuccessor(NextMBB);
9099 MBB = NextMBB;
9100 }
9101 }
9102 if (EndMBB) {
9103 MBB->addSuccessor(EndMBB);
9104 MBB = EndMBB;
9105 MBB->addLiveIn(SystemZ::CC);
9106 }
9107
9108 MI.eraseFromParent();
9109 return MBB;
9110 }
9111
9112 // Decompose string pseudo-instruction MI into a loop that continually performs
9113 // Opcode until CC != 3.
emitStringWrapper(MachineInstr & MI,MachineBasicBlock * MBB,unsigned Opcode) const9114 MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
9115 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
9116 MachineFunction &MF = *MBB->getParent();
9117 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9118 MachineRegisterInfo &MRI = MF.getRegInfo();
9119 DebugLoc DL = MI.getDebugLoc();
9120
9121 uint64_t End1Reg = MI.getOperand(0).getReg();
9122 uint64_t Start1Reg = MI.getOperand(1).getReg();
9123 uint64_t Start2Reg = MI.getOperand(2).getReg();
9124 uint64_t CharReg = MI.getOperand(3).getReg();
9125
9126 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
9127 uint64_t This1Reg = MRI.createVirtualRegister(RC);
9128 uint64_t This2Reg = MRI.createVirtualRegister(RC);
9129 uint64_t End2Reg = MRI.createVirtualRegister(RC);
9130
9131 MachineBasicBlock *StartMBB = MBB;
9132 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
9133 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
9134
9135 // StartMBB:
9136 // # fall through to LoopMBB
9137 MBB->addSuccessor(LoopMBB);
9138
9139 // LoopMBB:
9140 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
9141 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
9142 // R0L = %CharReg
9143 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
9144 // JO LoopMBB
9145 // # fall through to DoneMBB
9146 //
9147 // The load of R0L can be hoisted by post-RA LICM.
9148 MBB = LoopMBB;
9149
9150 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg)
9151 .addReg(Start1Reg).addMBB(StartMBB)
9152 .addReg(End1Reg).addMBB(LoopMBB);
9153 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg)
9154 .addReg(Start2Reg).addMBB(StartMBB)
9155 .addReg(End2Reg).addMBB(LoopMBB);
9156 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg);
9157 BuildMI(MBB, DL, TII->get(Opcode))
9158 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define)
9159 .addReg(This1Reg).addReg(This2Reg);
9160 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
9161 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB);
9162 MBB->addSuccessor(LoopMBB);
9163 MBB->addSuccessor(DoneMBB);
9164
9165 DoneMBB->addLiveIn(SystemZ::CC);
9166
9167 MI.eraseFromParent();
9168 return DoneMBB;
9169 }
9170
9171 // Update TBEGIN instruction with final opcode and register clobbers.
emitTransactionBegin(MachineInstr & MI,MachineBasicBlock * MBB,unsigned Opcode,bool NoFloat) const9172 MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
9173 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
9174 bool NoFloat) const {
9175 MachineFunction &MF = *MBB->getParent();
9176 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9177 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9178
9179 // Update opcode.
9180 MI.setDesc(TII->get(Opcode));
9181
9182 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
9183 // Make sure to add the corresponding GRSM bits if they are missing.
9184 uint64_t Control = MI.getOperand(2).getImm();
9185 static const unsigned GPRControlBit[16] = {
9186 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
9187 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
9188 };
9189 Control |= GPRControlBit[15];
9190 if (TFI->hasFP(MF))
9191 Control |= GPRControlBit[11];
9192 MI.getOperand(2).setImm(Control);
9193
9194 // Add GPR clobbers.
9195 for (int I = 0; I < 16; I++) {
9196 if ((Control & GPRControlBit[I]) == 0) {
9197 unsigned Reg = SystemZMC::GR64Regs[I];
9198 MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
9199 }
9200 }
9201
9202 // Add FPR/VR clobbers.
9203 if (!NoFloat && (Control & 4) != 0) {
9204 if (Subtarget.hasVector()) {
9205 for (unsigned Reg : SystemZMC::VR128Regs) {
9206 MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
9207 }
9208 } else {
9209 for (unsigned Reg : SystemZMC::FP64Regs) {
9210 MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
9211 }
9212 }
9213 }
9214
9215 return MBB;
9216 }
9217
emitLoadAndTestCmp0(MachineInstr & MI,MachineBasicBlock * MBB,unsigned Opcode) const9218 MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
9219 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
9220 MachineFunction &MF = *MBB->getParent();
9221 MachineRegisterInfo *MRI = &MF.getRegInfo();
9222 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9223 DebugLoc DL = MI.getDebugLoc();
9224
9225 Register SrcReg = MI.getOperand(0).getReg();
9226
9227 // Create new virtual register of the same class as source.
9228 const TargetRegisterClass *RC = MRI->getRegClass(SrcReg);
9229 Register DstReg = MRI->createVirtualRegister(RC);
9230
9231 // Replace pseudo with a normal load-and-test that models the def as
9232 // well.
9233 BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg)
9234 .addReg(SrcReg)
9235 .setMIFlags(MI.getFlags());
9236 MI.eraseFromParent();
9237
9238 return MBB;
9239 }
9240
emitProbedAlloca(MachineInstr & MI,MachineBasicBlock * MBB) const9241 MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
9242 MachineInstr &MI, MachineBasicBlock *MBB) const {
9243 MachineFunction &MF = *MBB->getParent();
9244 MachineRegisterInfo *MRI = &MF.getRegInfo();
9245 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9246 DebugLoc DL = MI.getDebugLoc();
9247 const unsigned ProbeSize = getStackProbeSize(MF);
9248 Register DstReg = MI.getOperand(0).getReg();
9249 Register SizeReg = MI.getOperand(2).getReg();
9250
9251 MachineBasicBlock *StartMBB = MBB;
9252 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
9253 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(StartMBB);
9254 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(LoopTestMBB);
9255 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(LoopBodyMBB);
9256 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(TailTestMBB);
9257
9258 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(MachinePointerInfo(),
9259 MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, 8, Align(1));
9260
9261 Register PHIReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9262 Register IncReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
9263
9264 // LoopTestMBB
9265 // BRC TailTestMBB
9266 // # fallthrough to LoopBodyMBB
9267 StartMBB->addSuccessor(LoopTestMBB);
9268 MBB = LoopTestMBB;
9269 BuildMI(MBB, DL, TII->get(SystemZ::PHI), PHIReg)
9270 .addReg(SizeReg)
9271 .addMBB(StartMBB)
9272 .addReg(IncReg)
9273 .addMBB(LoopBodyMBB);
9274 BuildMI(MBB, DL, TII->get(SystemZ::CLGFI))
9275 .addReg(PHIReg)
9276 .addImm(ProbeSize);
9277 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
9278 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_LT)
9279 .addMBB(TailTestMBB);
9280 MBB->addSuccessor(LoopBodyMBB);
9281 MBB->addSuccessor(TailTestMBB);
9282
9283 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
9284 // J LoopTestMBB
9285 MBB = LoopBodyMBB;
9286 BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), IncReg)
9287 .addReg(PHIReg)
9288 .addImm(ProbeSize);
9289 BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), SystemZ::R15D)
9290 .addReg(SystemZ::R15D)
9291 .addImm(ProbeSize);
9292 BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D)
9293 .addReg(SystemZ::R15D).addImm(ProbeSize - 8).addReg(0)
9294 .setMemRefs(VolLdMMO);
9295 BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(LoopTestMBB);
9296 MBB->addSuccessor(LoopTestMBB);
9297
9298 // TailTestMBB
9299 // BRC DoneMBB
9300 // # fallthrough to TailMBB
9301 MBB = TailTestMBB;
9302 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
9303 .addReg(PHIReg)
9304 .addImm(0);
9305 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
9306 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
9307 .addMBB(DoneMBB);
9308 MBB->addSuccessor(TailMBB);
9309 MBB->addSuccessor(DoneMBB);
9310
9311 // TailMBB
9312 // # fallthrough to DoneMBB
9313 MBB = TailMBB;
9314 BuildMI(MBB, DL, TII->get(SystemZ::SLGR), SystemZ::R15D)
9315 .addReg(SystemZ::R15D)
9316 .addReg(PHIReg);
9317 BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D)
9318 .addReg(SystemZ::R15D).addImm(-8).addReg(PHIReg)
9319 .setMemRefs(VolLdMMO);
9320 MBB->addSuccessor(DoneMBB);
9321
9322 // DoneMBB
9323 MBB = DoneMBB;
9324 BuildMI(*MBB, MBB->begin(), DL, TII->get(TargetOpcode::COPY), DstReg)
9325 .addReg(SystemZ::R15D);
9326
9327 MI.eraseFromParent();
9328 return DoneMBB;
9329 }
9330
9331 SDValue SystemZTargetLowering::
getBackchainAddress(SDValue SP,SelectionDAG & DAG) const9332 getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
9333 MachineFunction &MF = DAG.getMachineFunction();
9334 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
9335 SDLoc DL(SP);
9336 return DAG.getNode(ISD::ADD, DL, MVT::i64, SP,
9337 DAG.getIntPtrConstant(TFL->getBackchainOffset(MF), DL));
9338 }
9339
EmitInstrWithCustomInserter(MachineInstr & MI,MachineBasicBlock * MBB) const9340 MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
9341 MachineInstr &MI, MachineBasicBlock *MBB) const {
9342 switch (MI.getOpcode()) {
9343 case SystemZ::Select32:
9344 case SystemZ::Select64:
9345 case SystemZ::Select128:
9346 case SystemZ::SelectF32:
9347 case SystemZ::SelectF64:
9348 case SystemZ::SelectF128:
9349 case SystemZ::SelectVR32:
9350 case SystemZ::SelectVR64:
9351 case SystemZ::SelectVR128:
9352 return emitSelect(MI, MBB);
9353
9354 case SystemZ::CondStore8Mux:
9355 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false);
9356 case SystemZ::CondStore8MuxInv:
9357 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true);
9358 case SystemZ::CondStore16Mux:
9359 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false);
9360 case SystemZ::CondStore16MuxInv:
9361 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true);
9362 case SystemZ::CondStore32Mux:
9363 return emitCondStore(MI, MBB, SystemZ::STMux, SystemZ::STOCMux, false);
9364 case SystemZ::CondStore32MuxInv:
9365 return emitCondStore(MI, MBB, SystemZ::STMux, SystemZ::STOCMux, true);
9366 case SystemZ::CondStore8:
9367 return emitCondStore(MI, MBB, SystemZ::STC, 0, false);
9368 case SystemZ::CondStore8Inv:
9369 return emitCondStore(MI, MBB, SystemZ::STC, 0, true);
9370 case SystemZ::CondStore16:
9371 return emitCondStore(MI, MBB, SystemZ::STH, 0, false);
9372 case SystemZ::CondStore16Inv:
9373 return emitCondStore(MI, MBB, SystemZ::STH, 0, true);
9374 case SystemZ::CondStore32:
9375 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false);
9376 case SystemZ::CondStore32Inv:
9377 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true);
9378 case SystemZ::CondStore64:
9379 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false);
9380 case SystemZ::CondStore64Inv:
9381 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true);
9382 case SystemZ::CondStoreF32:
9383 return emitCondStore(MI, MBB, SystemZ::STE, 0, false);
9384 case SystemZ::CondStoreF32Inv:
9385 return emitCondStore(MI, MBB, SystemZ::STE, 0, true);
9386 case SystemZ::CondStoreF64:
9387 return emitCondStore(MI, MBB, SystemZ::STD, 0, false);
9388 case SystemZ::CondStoreF64Inv:
9389 return emitCondStore(MI, MBB, SystemZ::STD, 0, true);
9390
9391 case SystemZ::SCmp128Hi:
9392 return emitICmp128Hi(MI, MBB, false);
9393 case SystemZ::UCmp128Hi:
9394 return emitICmp128Hi(MI, MBB, true);
9395
9396 case SystemZ::PAIR128:
9397 return emitPair128(MI, MBB);
9398 case SystemZ::AEXT128:
9399 return emitExt128(MI, MBB, false);
9400 case SystemZ::ZEXT128:
9401 return emitExt128(MI, MBB, true);
9402
9403 case SystemZ::ATOMIC_SWAPW:
9404 return emitAtomicLoadBinary(MI, MBB, 0);
9405
9406 case SystemZ::ATOMIC_LOADW_AR:
9407 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR);
9408 case SystemZ::ATOMIC_LOADW_AFI:
9409 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI);
9410
9411 case SystemZ::ATOMIC_LOADW_SR:
9412 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR);
9413
9414 case SystemZ::ATOMIC_LOADW_NR:
9415 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR);
9416 case SystemZ::ATOMIC_LOADW_NILH:
9417 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH);
9418
9419 case SystemZ::ATOMIC_LOADW_OR:
9420 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR);
9421 case SystemZ::ATOMIC_LOADW_OILH:
9422 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH);
9423
9424 case SystemZ::ATOMIC_LOADW_XR:
9425 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR);
9426 case SystemZ::ATOMIC_LOADW_XILF:
9427 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF);
9428
9429 case SystemZ::ATOMIC_LOADW_NRi:
9430 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, true);
9431 case SystemZ::ATOMIC_LOADW_NILHi:
9432 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, true);
9433
9434 case SystemZ::ATOMIC_LOADW_MIN:
9435 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_LE);
9436 case SystemZ::ATOMIC_LOADW_MAX:
9437 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_GE);
9438 case SystemZ::ATOMIC_LOADW_UMIN:
9439 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_LE);
9440 case SystemZ::ATOMIC_LOADW_UMAX:
9441 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_GE);
9442
9443 case SystemZ::ATOMIC_CMP_SWAPW:
9444 return emitAtomicCmpSwapW(MI, MBB);
9445 case SystemZ::MVCImm:
9446 case SystemZ::MVCReg:
9447 return emitMemMemWrapper(MI, MBB, SystemZ::MVC);
9448 case SystemZ::NCImm:
9449 return emitMemMemWrapper(MI, MBB, SystemZ::NC);
9450 case SystemZ::OCImm:
9451 return emitMemMemWrapper(MI, MBB, SystemZ::OC);
9452 case SystemZ::XCImm:
9453 case SystemZ::XCReg:
9454 return emitMemMemWrapper(MI, MBB, SystemZ::XC);
9455 case SystemZ::CLCImm:
9456 case SystemZ::CLCReg:
9457 return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
9458 case SystemZ::MemsetImmImm:
9459 case SystemZ::MemsetImmReg:
9460 case SystemZ::MemsetRegImm:
9461 case SystemZ::MemsetRegReg:
9462 return emitMemMemWrapper(MI, MBB, SystemZ::MVC, true/*IsMemset*/);
9463 case SystemZ::CLSTLoop:
9464 return emitStringWrapper(MI, MBB, SystemZ::CLST);
9465 case SystemZ::MVSTLoop:
9466 return emitStringWrapper(MI, MBB, SystemZ::MVST);
9467 case SystemZ::SRSTLoop:
9468 return emitStringWrapper(MI, MBB, SystemZ::SRST);
9469 case SystemZ::TBEGIN:
9470 return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, false);
9471 case SystemZ::TBEGIN_nofloat:
9472 return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, true);
9473 case SystemZ::TBEGINC:
9474 return emitTransactionBegin(MI, MBB, SystemZ::TBEGINC, true);
9475 case SystemZ::LTEBRCompare_Pseudo:
9476 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTEBR);
9477 case SystemZ::LTDBRCompare_Pseudo:
9478 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTDBR);
9479 case SystemZ::LTXBRCompare_Pseudo:
9480 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTXBR);
9481
9482 case SystemZ::PROBED_ALLOCA:
9483 return emitProbedAlloca(MI, MBB);
9484
9485 case TargetOpcode::STACKMAP:
9486 case TargetOpcode::PATCHPOINT:
9487 return emitPatchPoint(MI, MBB);
9488
9489 default:
9490 llvm_unreachable("Unexpected instr type to insert");
9491 }
9492 }
9493
9494 // This is only used by the isel schedulers, and is needed only to prevent
9495 // compiler from crashing when list-ilp is used.
9496 const TargetRegisterClass *
getRepRegClassFor(MVT VT) const9497 SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
9498 if (VT == MVT::Untyped)
9499 return &SystemZ::ADDR128BitRegClass;
9500 return TargetLowering::getRepRegClassFor(VT);
9501 }
9502
lowerGET_ROUNDING(SDValue Op,SelectionDAG & DAG) const9503 SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
9504 SelectionDAG &DAG) const {
9505 SDLoc dl(Op);
9506 /*
9507 The rounding method is in FPC Byte 3 bits 6-7, and has the following
9508 settings:
9509 00 Round to nearest
9510 01 Round to 0
9511 10 Round to +inf
9512 11 Round to -inf
9513
9514 FLT_ROUNDS, on the other hand, expects the following:
9515 -1 Undefined
9516 0 Round to 0
9517 1 Round to nearest
9518 2 Round to +inf
9519 3 Round to -inf
9520 */
9521
9522 // Save FPC to register.
9523 SDValue Chain = Op.getOperand(0);
9524 SDValue EFPC(
9525 DAG.getMachineNode(SystemZ::EFPC, dl, {MVT::i32, MVT::Other}, Chain), 0);
9526 Chain = EFPC.getValue(1);
9527
9528 // Transform as necessary
9529 SDValue CWD1 = DAG.getNode(ISD::AND, dl, MVT::i32, EFPC,
9530 DAG.getConstant(3, dl, MVT::i32));
9531 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
9532 SDValue CWD2 = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1,
9533 DAG.getNode(ISD::SRL, dl, MVT::i32, CWD1,
9534 DAG.getConstant(1, dl, MVT::i32)));
9535
9536 SDValue RetVal = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD2,
9537 DAG.getConstant(1, dl, MVT::i32));
9538 RetVal = DAG.getZExtOrTrunc(RetVal, dl, Op.getValueType());
9539
9540 return DAG.getMergeValues({RetVal, Chain}, dl);
9541 }
9542