1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the SystemZ target.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "SystemZTargetMachine.h"
14 #include "SystemZISelLowering.h"
15 #include "llvm/Analysis/AliasAnalysis.h"
16 #include "llvm/CodeGen/SelectionDAGISel.h"
17 #include "llvm/Support/Debug.h"
18 #include "llvm/Support/KnownBits.h"
19 #include "llvm/Support/raw_ostream.h"
20
21 using namespace llvm;
22
23 #define DEBUG_TYPE "systemz-isel"
24
25 namespace {
26 // Used to build addressing modes.
27 struct SystemZAddressingMode {
28 // The shape of the address.
29 enum AddrForm {
30 // base+displacement
31 FormBD,
32
33 // base+displacement+index for load and store operands
34 FormBDXNormal,
35
36 // base+displacement+index for load address operands
37 FormBDXLA,
38
39 // base+displacement+index+ADJDYNALLOC
40 FormBDXDynAlloc
41 };
42 AddrForm Form;
43
44 // The type of displacement. The enum names here correspond directly
45 // to the definitions in SystemZOperand.td. We could split them into
46 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
47 enum DispRange {
48 Disp12Only,
49 Disp12Pair,
50 Disp20Only,
51 Disp20Only128,
52 Disp20Pair
53 };
54 DispRange DR;
55
56 // The parts of the address. The address is equivalent to:
57 //
58 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
59 SDValue Base;
60 int64_t Disp;
61 SDValue Index;
62 bool IncludesDynAlloc;
63
SystemZAddressingMode__anon7a3e01970111::SystemZAddressingMode64 SystemZAddressingMode(AddrForm form, DispRange dr)
65 : Form(form), DR(dr), Base(), Disp(0), Index(),
66 IncludesDynAlloc(false) {}
67
68 // True if the address can have an index register.
hasIndexField__anon7a3e01970111::SystemZAddressingMode69 bool hasIndexField() { return Form != FormBD; }
70
71 // True if the address can (and must) include ADJDYNALLOC.
isDynAlloc__anon7a3e01970111::SystemZAddressingMode72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
dump__anon7a3e01970111::SystemZAddressingMode74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
76
77 errs() << " Base ";
78 if (Base.getNode())
79 Base.getNode()->dump(DAG);
80 else
81 errs() << "null\n";
82
83 if (hasIndexField()) {
84 errs() << " Index ";
85 if (Index.getNode())
86 Index.getNode()->dump(DAG);
87 else
88 errs() << "null\n";
89 }
90
91 errs() << " Disp " << Disp;
92 if (IncludesDynAlloc)
93 errs() << " + ADJDYNALLOC";
94 errs() << '\n';
95 }
96 };
97
98 // Return a mask with Count low bits set.
allOnes(unsigned int Count)99 static uint64_t allOnes(unsigned int Count) {
100 assert(Count <= 64);
101 if (Count > 63)
102 return UINT64_MAX;
103 return (uint64_t(1) << Count) - 1;
104 }
105
106 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108 // Rotate (I5). The combined operand value is effectively:
109 //
110 // (or (rotl Input, Rotate), ~Mask)
111 //
112 // for RNSBG and:
113 //
114 // (and (rotl Input, Rotate), Mask)
115 //
116 // otherwise. The output value has BitSize bits, although Input may be
117 // narrower (in which case the upper bits are don't care), or wider (in which
118 // case the result will be truncated as part of the operation).
119 struct RxSBGOperands {
RxSBGOperands__anon7a3e01970111::RxSBGOperands120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123 Rotate(0) {}
124
125 unsigned Opcode;
126 unsigned BitSize;
127 uint64_t Mask;
128 SDValue Input;
129 unsigned Start;
130 unsigned End;
131 unsigned Rotate;
132 };
133
134 class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
136
137 // Used by SystemZOperands.td to create integer constants.
getImm(const SDNode * Node,uint64_t Imm) const138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140 }
141
getTargetMachine() const142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
144 }
145
getInstrInfo() const146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
148 }
149
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
168
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
174
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182 // PC-relative address matching routines used by SystemZOperands.td.
selectPCRelAddress(SDValue Addr,SDValue & Target) const183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
186 return true;
187 }
188 return false;
189 }
190
191 // BD matching routines used by SystemZOperands.td.
selectBDAddr12Only(SDValue Addr,SDValue & Base,SDValue & Disp) const192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194 }
selectBDAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197 }
selectBDAddr20Only(SDValue Addr,SDValue & Base,SDValue & Disp) const198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200 }
selectBDAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203 }
204
205 // MVI matching routines used by SystemZOperands.td.
selectMVIAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208 }
selectMVIAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211 }
212
213 // BDX matching routines used by SystemZOperands.td.
selectBDXAddr12Only(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
219 }
selectBDXAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
225 }
selectDynAlloc12Only(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
231 }
selectBDXAddr20Only(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
237 }
selectBDXAddr20Only128(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
243 }
selectBDXAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
249 }
selectLAAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
255 }
selectLAAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
261 }
262
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
269
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
272 // set Op to that Y.
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
292
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297 // If Op0 is null, then Node is a constant that can be loaded using:
298 //
299 // (Opcode UpperVal LowerVal)
300 //
301 // If Op0 is nonnull, then Node can be implemented using:
302 //
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
306
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308 SDNode *Node);
309
310 // Try to use gather instruction Opcode to implement vector insertion N.
311 bool tryGather(SDNode *N, unsigned Opcode);
312
313 // Try to use scatter instruction Opcode to implement store Store.
314 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
315
316 // Change a chain of {load; op; store} of the same value into a simple op
317 // through memory of that value, if the uses of the modified value and its
318 // address are suitable.
319 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
320
321 // Return true if Load and Store are loads and stores of the same size
322 // and are guaranteed not to overlap. Such operations can be implemented
323 // using block (SS-format) instructions.
324 //
325 // Partial overlap would lead to incorrect code, since the block operations
326 // are logically bytewise, even though they have a fast path for the
327 // non-overlapping case. We also need to avoid full overlap (i.e. two
328 // addresses that might be equal at run time) because although that case
329 // would be handled correctly, it might be implemented by millicode.
330 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
331
332 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
333 // from Y to X.
334 bool storeLoadCanUseMVC(SDNode *N) const;
335
336 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
337 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
338 // to X.
339 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
340
341 // Return true if N (a load or a store) fullfills the alignment
342 // requirements for a PC-relative access.
343 bool storeLoadIsAligned(SDNode *N) const;
344
345 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
346 SDValue expandSelectBoolean(SDNode *Node);
347
348 public:
SystemZDAGToDAGISel(SystemZTargetMachine & TM,CodeGenOpt::Level OptLevel)349 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
350 : SelectionDAGISel(TM, OptLevel) {}
351
runOnMachineFunction(MachineFunction & MF)352 bool runOnMachineFunction(MachineFunction &MF) override {
353 const Function &F = MF.getFunction();
354 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
355 if (F.hasFnAttribute("mnop-mcount"))
356 report_fatal_error("mnop-mcount only supported with fentry-call");
357 if (F.hasFnAttribute("mrecord-mcount"))
358 report_fatal_error("mrecord-mcount only supported with fentry-call");
359 }
360
361 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
362 return SelectionDAGISel::runOnMachineFunction(MF);
363 }
364
365 // Override MachineFunctionPass.
getPassName() const366 StringRef getPassName() const override {
367 return "SystemZ DAG->DAG Pattern Instruction Selection";
368 }
369
370 // Override SelectionDAGISel.
371 void Select(SDNode *Node) override;
372 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
373 std::vector<SDValue> &OutOps) override;
374 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
375 void PreprocessISelDAG() override;
376
377 // Include the pieces autogenerated from the target description.
378 #include "SystemZGenDAGISel.inc"
379 };
380 } // end anonymous namespace
381
createSystemZISelDag(SystemZTargetMachine & TM,CodeGenOpt::Level OptLevel)382 FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
383 CodeGenOpt::Level OptLevel) {
384 return new SystemZDAGToDAGISel(TM, OptLevel);
385 }
386
387 // Return true if Val should be selected as a displacement for an address
388 // with range DR. Here we're interested in the range of both the instruction
389 // described by DR and of any pairing instruction.
selectDisp(SystemZAddressingMode::DispRange DR,int64_t Val)390 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
391 switch (DR) {
392 case SystemZAddressingMode::Disp12Only:
393 return isUInt<12>(Val);
394
395 case SystemZAddressingMode::Disp12Pair:
396 case SystemZAddressingMode::Disp20Only:
397 case SystemZAddressingMode::Disp20Pair:
398 return isInt<20>(Val);
399
400 case SystemZAddressingMode::Disp20Only128:
401 return isInt<20>(Val) && isInt<20>(Val + 8);
402 }
403 llvm_unreachable("Unhandled displacement range");
404 }
405
406 // Change the base or index in AM to Value, where IsBase selects
407 // between the base and index.
changeComponent(SystemZAddressingMode & AM,bool IsBase,SDValue Value)408 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
409 SDValue Value) {
410 if (IsBase)
411 AM.Base = Value;
412 else
413 AM.Index = Value;
414 }
415
416 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
417 // where IsBase selects between the base and index. Try to fold the
418 // ADJDYNALLOC into AM.
expandAdjDynAlloc(SystemZAddressingMode & AM,bool IsBase,SDValue Value)419 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
420 SDValue Value) {
421 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
422 changeComponent(AM, IsBase, Value);
423 AM.IncludesDynAlloc = true;
424 return true;
425 }
426 return false;
427 }
428
429 // The base of AM is equivalent to Base + Index. Try to use Index as
430 // the index register.
expandIndex(SystemZAddressingMode & AM,SDValue Base,SDValue Index)431 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
432 SDValue Index) {
433 if (AM.hasIndexField() && !AM.Index.getNode()) {
434 AM.Base = Base;
435 AM.Index = Index;
436 return true;
437 }
438 return false;
439 }
440
441 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
442 // between the base and index. Try to fold Op1 into AM's displacement.
expandDisp(SystemZAddressingMode & AM,bool IsBase,SDValue Op0,uint64_t Op1)443 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
444 SDValue Op0, uint64_t Op1) {
445 // First try adjusting the displacement.
446 int64_t TestDisp = AM.Disp + Op1;
447 if (selectDisp(AM.DR, TestDisp)) {
448 changeComponent(AM, IsBase, Op0);
449 AM.Disp = TestDisp;
450 return true;
451 }
452
453 // We could consider forcing the displacement into a register and
454 // using it as an index, but it would need to be carefully tuned.
455 return false;
456 }
457
expandAddress(SystemZAddressingMode & AM,bool IsBase) const458 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
459 bool IsBase) const {
460 SDValue N = IsBase ? AM.Base : AM.Index;
461 unsigned Opcode = N.getOpcode();
462 if (Opcode == ISD::TRUNCATE) {
463 N = N.getOperand(0);
464 Opcode = N.getOpcode();
465 }
466 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
467 SDValue Op0 = N.getOperand(0);
468 SDValue Op1 = N.getOperand(1);
469
470 unsigned Op0Code = Op0->getOpcode();
471 unsigned Op1Code = Op1->getOpcode();
472
473 if (Op0Code == SystemZISD::ADJDYNALLOC)
474 return expandAdjDynAlloc(AM, IsBase, Op1);
475 if (Op1Code == SystemZISD::ADJDYNALLOC)
476 return expandAdjDynAlloc(AM, IsBase, Op0);
477
478 if (Op0Code == ISD::Constant)
479 return expandDisp(AM, IsBase, Op1,
480 cast<ConstantSDNode>(Op0)->getSExtValue());
481 if (Op1Code == ISD::Constant)
482 return expandDisp(AM, IsBase, Op0,
483 cast<ConstantSDNode>(Op1)->getSExtValue());
484
485 if (IsBase && expandIndex(AM, Op0, Op1))
486 return true;
487 }
488 if (Opcode == SystemZISD::PCREL_OFFSET) {
489 SDValue Full = N.getOperand(0);
490 SDValue Base = N.getOperand(1);
491 SDValue Anchor = Base.getOperand(0);
492 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
493 cast<GlobalAddressSDNode>(Anchor)->getOffset());
494 return expandDisp(AM, IsBase, Base, Offset);
495 }
496 return false;
497 }
498
499 // Return true if an instruction with displacement range DR should be
500 // used for displacement value Val. selectDisp(DR, Val) must already hold.
isValidDisp(SystemZAddressingMode::DispRange DR,int64_t Val)501 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
502 assert(selectDisp(DR, Val) && "Invalid displacement");
503 switch (DR) {
504 case SystemZAddressingMode::Disp12Only:
505 case SystemZAddressingMode::Disp20Only:
506 case SystemZAddressingMode::Disp20Only128:
507 return true;
508
509 case SystemZAddressingMode::Disp12Pair:
510 // Use the other instruction if the displacement is too large.
511 return isUInt<12>(Val);
512
513 case SystemZAddressingMode::Disp20Pair:
514 // Use the other instruction if the displacement is small enough.
515 return !isUInt<12>(Val);
516 }
517 llvm_unreachable("Unhandled displacement range");
518 }
519
520 // Return true if Base + Disp + Index should be performed by LA(Y).
shouldUseLA(SDNode * Base,int64_t Disp,SDNode * Index)521 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
522 // Don't use LA(Y) for constants.
523 if (!Base)
524 return false;
525
526 // Always use LA(Y) for frame addresses, since we know that the destination
527 // register is almost always (perhaps always) going to be different from
528 // the frame register.
529 if (Base->getOpcode() == ISD::FrameIndex)
530 return true;
531
532 if (Disp) {
533 // Always use LA(Y) if there is a base, displacement and index.
534 if (Index)
535 return true;
536
537 // Always use LA if the displacement is small enough. It should always
538 // be no worse than AGHI (and better if it avoids a move).
539 if (isUInt<12>(Disp))
540 return true;
541
542 // For similar reasons, always use LAY if the constant is too big for AGHI.
543 // LAY should be no worse than AGFI.
544 if (!isInt<16>(Disp))
545 return true;
546 } else {
547 // Don't use LA for plain registers.
548 if (!Index)
549 return false;
550
551 // Don't use LA for plain addition if the index operand is only used
552 // once. It should be a natural two-operand addition in that case.
553 if (Index->hasOneUse())
554 return false;
555
556 // Prefer addition if the second operation is sign-extended, in the
557 // hope of using AGF.
558 unsigned IndexOpcode = Index->getOpcode();
559 if (IndexOpcode == ISD::SIGN_EXTEND ||
560 IndexOpcode == ISD::SIGN_EXTEND_INREG)
561 return false;
562 }
563
564 // Don't use LA for two-operand addition if either operand is only
565 // used once. The addition instructions are better in that case.
566 if (Base->hasOneUse())
567 return false;
568
569 return true;
570 }
571
572 // Return true if Addr is suitable for AM, updating AM if so.
selectAddress(SDValue Addr,SystemZAddressingMode & AM) const573 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
574 SystemZAddressingMode &AM) const {
575 // Start out assuming that the address will need to be loaded separately,
576 // then try to extend it as much as we can.
577 AM.Base = Addr;
578
579 // First try treating the address as a constant.
580 if (Addr.getOpcode() == ISD::Constant &&
581 expandDisp(AM, true, SDValue(),
582 cast<ConstantSDNode>(Addr)->getSExtValue()))
583 ;
584 // Also see if it's a bare ADJDYNALLOC.
585 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
586 expandAdjDynAlloc(AM, true, SDValue()))
587 ;
588 else
589 // Otherwise try expanding each component.
590 while (expandAddress(AM, true) ||
591 (AM.Index.getNode() && expandAddress(AM, false)))
592 continue;
593
594 // Reject cases where it isn't profitable to use LA(Y).
595 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
596 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
597 return false;
598
599 // Reject cases where the other instruction in a pair should be used.
600 if (!isValidDisp(AM.DR, AM.Disp))
601 return false;
602
603 // Make sure that ADJDYNALLOC is included where necessary.
604 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
605 return false;
606
607 LLVM_DEBUG(AM.dump(CurDAG));
608 return true;
609 }
610
611 // Insert a node into the DAG at least before Pos. This will reposition
612 // the node as needed, and will assign it a node ID that is <= Pos's ID.
613 // Note that this does *not* preserve the uniqueness of node IDs!
614 // The selection DAG must no longer depend on their uniqueness when this
615 // function is used.
insertDAGNode(SelectionDAG * DAG,SDNode * Pos,SDValue N)616 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
617 if (N->getNodeId() == -1 ||
618 (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) >
619 SelectionDAGISel::getUninvalidatedNodeId(Pos))) {
620 DAG->RepositionNode(Pos->getIterator(), N.getNode());
621 // Mark Node as invalid for pruning as after this it may be a successor to a
622 // selected node but otherwise be in the same position of Pos.
623 // Conservatively mark it with the same -abs(Id) to assure node id
624 // invariant is preserved.
625 N->setNodeId(Pos->getNodeId());
626 SelectionDAGISel::InvalidateNodeId(N.getNode());
627 }
628 }
629
getAddressOperands(const SystemZAddressingMode & AM,EVT VT,SDValue & Base,SDValue & Disp) const630 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
631 EVT VT, SDValue &Base,
632 SDValue &Disp) const {
633 Base = AM.Base;
634 if (!Base.getNode())
635 // Register 0 means "no base". This is mostly useful for shifts.
636 Base = CurDAG->getRegister(0, VT);
637 else if (Base.getOpcode() == ISD::FrameIndex) {
638 // Lower a FrameIndex to a TargetFrameIndex.
639 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
640 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
641 } else if (Base.getValueType() != VT) {
642 // Truncate values from i64 to i32, for shifts.
643 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
644 "Unexpected truncation");
645 SDLoc DL(Base);
646 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
647 insertDAGNode(CurDAG, Base.getNode(), Trunc);
648 Base = Trunc;
649 }
650
651 // Lower the displacement to a TargetConstant.
652 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
653 }
654
getAddressOperands(const SystemZAddressingMode & AM,EVT VT,SDValue & Base,SDValue & Disp,SDValue & Index) const655 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
656 EVT VT, SDValue &Base,
657 SDValue &Disp,
658 SDValue &Index) const {
659 getAddressOperands(AM, VT, Base, Disp);
660
661 Index = AM.Index;
662 if (!Index.getNode())
663 // Register 0 means "no index".
664 Index = CurDAG->getRegister(0, VT);
665 }
666
selectBDAddr(SystemZAddressingMode::DispRange DR,SDValue Addr,SDValue & Base,SDValue & Disp) const667 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
668 SDValue Addr, SDValue &Base,
669 SDValue &Disp) const {
670 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
671 if (!selectAddress(Addr, AM))
672 return false;
673
674 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
675 return true;
676 }
677
selectMVIAddr(SystemZAddressingMode::DispRange DR,SDValue Addr,SDValue & Base,SDValue & Disp) const678 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
679 SDValue Addr, SDValue &Base,
680 SDValue &Disp) const {
681 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
682 if (!selectAddress(Addr, AM) || AM.Index.getNode())
683 return false;
684
685 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
686 return true;
687 }
688
selectBDXAddr(SystemZAddressingMode::AddrForm Form,SystemZAddressingMode::DispRange DR,SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const689 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
690 SystemZAddressingMode::DispRange DR,
691 SDValue Addr, SDValue &Base,
692 SDValue &Disp, SDValue &Index) const {
693 SystemZAddressingMode AM(Form, DR);
694 if (!selectAddress(Addr, AM))
695 return false;
696
697 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
698 return true;
699 }
700
selectBDVAddr12Only(SDValue Addr,SDValue Elem,SDValue & Base,SDValue & Disp,SDValue & Index) const701 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
702 SDValue &Base,
703 SDValue &Disp,
704 SDValue &Index) const {
705 SDValue Regs[2];
706 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
707 Regs[0].getNode() && Regs[1].getNode()) {
708 for (unsigned int I = 0; I < 2; ++I) {
709 Base = Regs[I];
710 Index = Regs[1 - I];
711 // We can't tell here whether the index vector has the right type
712 // for the access; the caller needs to do that instead.
713 if (Index.getOpcode() == ISD::ZERO_EXTEND)
714 Index = Index.getOperand(0);
715 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
716 Index.getOperand(1) == Elem) {
717 Index = Index.getOperand(0);
718 return true;
719 }
720 }
721 }
722 return false;
723 }
724
detectOrAndInsertion(SDValue & Op,uint64_t InsertMask) const725 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
726 uint64_t InsertMask) const {
727 // We're only interested in cases where the insertion is into some operand
728 // of Op, rather than into Op itself. The only useful case is an AND.
729 if (Op.getOpcode() != ISD::AND)
730 return false;
731
732 // We need a constant mask.
733 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
734 if (!MaskNode)
735 return false;
736
737 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
738 uint64_t AndMask = MaskNode->getZExtValue();
739 if (InsertMask & AndMask)
740 return false;
741
742 // It's only an insertion if all bits are covered or are known to be zero.
743 // The inner check covers all cases but is more expensive.
744 uint64_t Used = allOnes(Op.getValueSizeInBits());
745 if (Used != (AndMask | InsertMask)) {
746 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
747 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
748 return false;
749 }
750
751 Op = Op.getOperand(0);
752 return true;
753 }
754
refineRxSBGMask(RxSBGOperands & RxSBG,uint64_t Mask) const755 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
756 uint64_t Mask) const {
757 const SystemZInstrInfo *TII = getInstrInfo();
758 if (RxSBG.Rotate != 0)
759 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
760 Mask &= RxSBG.Mask;
761 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
762 RxSBG.Mask = Mask;
763 return true;
764 }
765 return false;
766 }
767
768 // Return true if any bits of (RxSBG.Input & Mask) are significant.
maskMatters(RxSBGOperands & RxSBG,uint64_t Mask)769 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
770 // Rotate the mask in the same way as RxSBG.Input is rotated.
771 if (RxSBG.Rotate != 0)
772 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
773 return (Mask & RxSBG.Mask) != 0;
774 }
775
expandRxSBG(RxSBGOperands & RxSBG) const776 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
777 SDValue N = RxSBG.Input;
778 unsigned Opcode = N.getOpcode();
779 switch (Opcode) {
780 case ISD::TRUNCATE: {
781 if (RxSBG.Opcode == SystemZ::RNSBG)
782 return false;
783 uint64_t BitSize = N.getValueSizeInBits();
784 uint64_t Mask = allOnes(BitSize);
785 if (!refineRxSBGMask(RxSBG, Mask))
786 return false;
787 RxSBG.Input = N.getOperand(0);
788 return true;
789 }
790 case ISD::AND: {
791 if (RxSBG.Opcode == SystemZ::RNSBG)
792 return false;
793
794 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
795 if (!MaskNode)
796 return false;
797
798 SDValue Input = N.getOperand(0);
799 uint64_t Mask = MaskNode->getZExtValue();
800 if (!refineRxSBGMask(RxSBG, Mask)) {
801 // If some bits of Input are already known zeros, those bits will have
802 // been removed from the mask. See if adding them back in makes the
803 // mask suitable.
804 KnownBits Known = CurDAG->computeKnownBits(Input);
805 Mask |= Known.Zero.getZExtValue();
806 if (!refineRxSBGMask(RxSBG, Mask))
807 return false;
808 }
809 RxSBG.Input = Input;
810 return true;
811 }
812
813 case ISD::OR: {
814 if (RxSBG.Opcode != SystemZ::RNSBG)
815 return false;
816
817 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
818 if (!MaskNode)
819 return false;
820
821 SDValue Input = N.getOperand(0);
822 uint64_t Mask = ~MaskNode->getZExtValue();
823 if (!refineRxSBGMask(RxSBG, Mask)) {
824 // If some bits of Input are already known ones, those bits will have
825 // been removed from the mask. See if adding them back in makes the
826 // mask suitable.
827 KnownBits Known = CurDAG->computeKnownBits(Input);
828 Mask &= ~Known.One.getZExtValue();
829 if (!refineRxSBGMask(RxSBG, Mask))
830 return false;
831 }
832 RxSBG.Input = Input;
833 return true;
834 }
835
836 case ISD::ROTL: {
837 // Any 64-bit rotate left can be merged into the RxSBG.
838 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
839 return false;
840 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
841 if (!CountNode)
842 return false;
843
844 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
845 RxSBG.Input = N.getOperand(0);
846 return true;
847 }
848
849 case ISD::ANY_EXTEND:
850 // Bits above the extended operand are don't-care.
851 RxSBG.Input = N.getOperand(0);
852 return true;
853
854 case ISD::ZERO_EXTEND:
855 if (RxSBG.Opcode != SystemZ::RNSBG) {
856 // Restrict the mask to the extended operand.
857 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
858 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
859 return false;
860
861 RxSBG.Input = N.getOperand(0);
862 return true;
863 }
864 LLVM_FALLTHROUGH;
865
866 case ISD::SIGN_EXTEND: {
867 // Check that the extension bits are don't-care (i.e. are masked out
868 // by the final mask).
869 unsigned BitSize = N.getValueSizeInBits();
870 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
871 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
872 // In the case where only the sign bit is active, increase Rotate with
873 // the extension width.
874 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
875 RxSBG.Rotate += (BitSize - InnerBitSize);
876 else
877 return false;
878 }
879
880 RxSBG.Input = N.getOperand(0);
881 return true;
882 }
883
884 case ISD::SHL: {
885 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
886 if (!CountNode)
887 return false;
888
889 uint64_t Count = CountNode->getZExtValue();
890 unsigned BitSize = N.getValueSizeInBits();
891 if (Count < 1 || Count >= BitSize)
892 return false;
893
894 if (RxSBG.Opcode == SystemZ::RNSBG) {
895 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
896 // count bits from RxSBG.Input are ignored.
897 if (maskMatters(RxSBG, allOnes(Count)))
898 return false;
899 } else {
900 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
901 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
902 return false;
903 }
904
905 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
906 RxSBG.Input = N.getOperand(0);
907 return true;
908 }
909
910 case ISD::SRL:
911 case ISD::SRA: {
912 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
913 if (!CountNode)
914 return false;
915
916 uint64_t Count = CountNode->getZExtValue();
917 unsigned BitSize = N.getValueSizeInBits();
918 if (Count < 1 || Count >= BitSize)
919 return false;
920
921 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
922 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
923 // count bits from RxSBG.Input are ignored.
924 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
925 return false;
926 } else {
927 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
928 // which is similar to SLL above.
929 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
930 return false;
931 }
932
933 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
934 RxSBG.Input = N.getOperand(0);
935 return true;
936 }
937 default:
938 return false;
939 }
940 }
941
getUNDEF(const SDLoc & DL,EVT VT) const942 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
943 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
944 return SDValue(N, 0);
945 }
946
convertTo(const SDLoc & DL,EVT VT,SDValue N) const947 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
948 SDValue N) const {
949 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
950 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
951 DL, VT, getUNDEF(DL, MVT::i64), N);
952 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
953 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
954 assert(N.getValueType() == VT && "Unexpected value types");
955 return N;
956 }
957
tryRISBGZero(SDNode * N)958 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
959 SDLoc DL(N);
960 EVT VT = N->getValueType(0);
961 if (!VT.isInteger() || VT.getSizeInBits() > 64)
962 return false;
963 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
964 unsigned Count = 0;
965 while (expandRxSBG(RISBG))
966 // The widening or narrowing is expected to be free.
967 // Counting widening or narrowing as a saved operation will result in
968 // preferring an R*SBG over a simple shift/logical instruction.
969 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
970 RISBG.Input.getOpcode() != ISD::TRUNCATE)
971 Count += 1;
972 if (Count == 0)
973 return false;
974
975 // Prefer to use normal shift instructions over RISBG, since they can handle
976 // all cases and are sometimes shorter.
977 if (Count == 1 && N->getOpcode() != ISD::AND)
978 return false;
979
980 // Prefer register extensions like LLC over RISBG. Also prefer to start
981 // out with normal ANDs if one instruction would be enough. We can convert
982 // these ANDs into an RISBG later if a three-address instruction is useful.
983 if (RISBG.Rotate == 0) {
984 bool PreferAnd = false;
985 // Prefer AND for any 32-bit and-immediate operation.
986 if (VT == MVT::i32)
987 PreferAnd = true;
988 // As well as for any 64-bit operation that can be implemented via LLC(R),
989 // LLH(R), LLGT(R), or one of the and-immediate instructions.
990 else if (RISBG.Mask == 0xff ||
991 RISBG.Mask == 0xffff ||
992 RISBG.Mask == 0x7fffffff ||
993 SystemZ::isImmLF(~RISBG.Mask) ||
994 SystemZ::isImmHF(~RISBG.Mask))
995 PreferAnd = true;
996 // And likewise for the LLZRGF instruction, which doesn't have a register
997 // to register version.
998 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
999 if (Load->getMemoryVT() == MVT::i32 &&
1000 (Load->getExtensionType() == ISD::EXTLOAD ||
1001 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1002 RISBG.Mask == 0xffffff00 &&
1003 Subtarget->hasLoadAndZeroRightmostByte())
1004 PreferAnd = true;
1005 }
1006 if (PreferAnd) {
1007 // Replace the current node with an AND. Note that the current node
1008 // might already be that same AND, in which case it is already CSE'd
1009 // with it, and we must not call ReplaceNode.
1010 SDValue In = convertTo(DL, VT, RISBG.Input);
1011 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1012 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1013 if (N != New.getNode()) {
1014 insertDAGNode(CurDAG, N, Mask);
1015 insertDAGNode(CurDAG, N, New);
1016 ReplaceNode(N, New.getNode());
1017 N = New.getNode();
1018 }
1019 // Now, select the machine opcode to implement this operation.
1020 if (!N->isMachineOpcode())
1021 SelectCode(N);
1022 return true;
1023 }
1024 }
1025
1026 unsigned Opcode = SystemZ::RISBG;
1027 // Prefer RISBGN if available, since it does not clobber CC.
1028 if (Subtarget->hasMiscellaneousExtensions())
1029 Opcode = SystemZ::RISBGN;
1030 EVT OpcodeVT = MVT::i64;
1031 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1032 // We can only use the 32-bit instructions if all source bits are
1033 // in the low 32 bits without wrapping, both after rotation (because
1034 // of the smaller range for Start and End) and before rotation
1035 // (because the input value is truncated).
1036 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1037 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1038 ((RISBG.End + RISBG.Rotate) & 63) >=
1039 ((RISBG.Start + RISBG.Rotate) & 63)) {
1040 Opcode = SystemZ::RISBMux;
1041 OpcodeVT = MVT::i32;
1042 RISBG.Start &= 31;
1043 RISBG.End &= 31;
1044 }
1045 SDValue Ops[5] = {
1046 getUNDEF(DL, OpcodeVT),
1047 convertTo(DL, OpcodeVT, RISBG.Input),
1048 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1049 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1050 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1051 };
1052 SDValue New = convertTo(
1053 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1054 ReplaceNode(N, New.getNode());
1055 return true;
1056 }
1057
tryRxSBG(SDNode * N,unsigned Opcode)1058 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1059 SDLoc DL(N);
1060 EVT VT = N->getValueType(0);
1061 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1062 return false;
1063 // Try treating each operand of N as the second operand of the RxSBG
1064 // and see which goes deepest.
1065 RxSBGOperands RxSBG[] = {
1066 RxSBGOperands(Opcode, N->getOperand(0)),
1067 RxSBGOperands(Opcode, N->getOperand(1))
1068 };
1069 unsigned Count[] = { 0, 0 };
1070 for (unsigned I = 0; I < 2; ++I)
1071 while (expandRxSBG(RxSBG[I]))
1072 // The widening or narrowing is expected to be free.
1073 // Counting widening or narrowing as a saved operation will result in
1074 // preferring an R*SBG over a simple shift/logical instruction.
1075 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1076 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1077 Count[I] += 1;
1078
1079 // Do nothing if neither operand is suitable.
1080 if (Count[0] == 0 && Count[1] == 0)
1081 return false;
1082
1083 // Pick the deepest second operand.
1084 unsigned I = Count[0] > Count[1] ? 0 : 1;
1085 SDValue Op0 = N->getOperand(I ^ 1);
1086
1087 // Prefer IC for character insertions from memory.
1088 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1089 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1090 if (Load->getMemoryVT() == MVT::i8)
1091 return false;
1092
1093 // See whether we can avoid an AND in the first operand by converting
1094 // ROSBG to RISBG.
1095 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1096 Opcode = SystemZ::RISBG;
1097 // Prefer RISBGN if available, since it does not clobber CC.
1098 if (Subtarget->hasMiscellaneousExtensions())
1099 Opcode = SystemZ::RISBGN;
1100 }
1101
1102 SDValue Ops[5] = {
1103 convertTo(DL, MVT::i64, Op0),
1104 convertTo(DL, MVT::i64, RxSBG[I].Input),
1105 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1106 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1107 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1108 };
1109 SDValue New = convertTo(
1110 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1111 ReplaceNode(N, New.getNode());
1112 return true;
1113 }
1114
splitLargeImmediate(unsigned Opcode,SDNode * Node,SDValue Op0,uint64_t UpperVal,uint64_t LowerVal)1115 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1116 SDValue Op0, uint64_t UpperVal,
1117 uint64_t LowerVal) {
1118 EVT VT = Node->getValueType(0);
1119 SDLoc DL(Node);
1120 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1121 if (Op0.getNode())
1122 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1123
1124 {
1125 // When we haven't passed in Op0, Upper will be a constant. In order to
1126 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1127 // SelectCode first and end up with an opaque machine node. This means that
1128 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1129 // SelectCode.
1130 //
1131 // Note that in the case where Op0 is passed in we could just call
1132 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1133 // the handle at all, but it's fine to do it here.
1134 //
1135 // TODO: This is a pretty hacky way to do this. Can we do something that
1136 // doesn't require a two paragraph explanation?
1137 HandleSDNode Handle(Upper);
1138 SelectCode(Upper.getNode());
1139 Upper = Handle.getValue();
1140 }
1141
1142 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1143 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1144
1145 ReplaceNode(Node, Or.getNode());
1146
1147 SelectCode(Or.getNode());
1148 }
1149
loadVectorConstant(const SystemZVectorConstantInfo & VCI,SDNode * Node)1150 void SystemZDAGToDAGISel::loadVectorConstant(
1151 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1152 assert((VCI.Opcode == SystemZISD::BYTE_MASK ||
1153 VCI.Opcode == SystemZISD::REPLICATE ||
1154 VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1155 "Bad opcode!");
1156 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1157 EVT VT = Node->getValueType(0);
1158 SDLoc DL(Node);
1159 SmallVector<SDValue, 2> Ops;
1160 for (unsigned OpVal : VCI.OpVals)
1161 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1162 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1163
1164 if (VCI.VecVT == VT.getSimpleVT())
1165 ReplaceNode(Node, Op.getNode());
1166 else if (VT.getSizeInBits() == 128) {
1167 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1168 ReplaceNode(Node, BitCast.getNode());
1169 SelectCode(BitCast.getNode());
1170 } else { // float or double
1171 unsigned SubRegIdx =
1172 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1173 ReplaceNode(
1174 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1175 }
1176 SelectCode(Op.getNode());
1177 }
1178
tryGather(SDNode * N,unsigned Opcode)1179 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1180 SDValue ElemV = N->getOperand(2);
1181 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1182 if (!ElemN)
1183 return false;
1184
1185 unsigned Elem = ElemN->getZExtValue();
1186 EVT VT = N->getValueType(0);
1187 if (Elem >= VT.getVectorNumElements())
1188 return false;
1189
1190 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1191 if (!Load || !Load->hasNUsesOfValue(1, 0))
1192 return false;
1193 if (Load->getMemoryVT().getSizeInBits() !=
1194 Load->getValueType(0).getSizeInBits())
1195 return false;
1196
1197 SDValue Base, Disp, Index;
1198 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1199 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1200 return false;
1201
1202 SDLoc DL(Load);
1203 SDValue Ops[] = {
1204 N->getOperand(0), Base, Disp, Index,
1205 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1206 };
1207 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1208 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1209 ReplaceNode(N, Res);
1210 return true;
1211 }
1212
tryScatter(StoreSDNode * Store,unsigned Opcode)1213 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1214 SDValue Value = Store->getValue();
1215 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1216 return false;
1217 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1218 return false;
1219
1220 SDValue ElemV = Value.getOperand(1);
1221 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1222 if (!ElemN)
1223 return false;
1224
1225 SDValue Vec = Value.getOperand(0);
1226 EVT VT = Vec.getValueType();
1227 unsigned Elem = ElemN->getZExtValue();
1228 if (Elem >= VT.getVectorNumElements())
1229 return false;
1230
1231 SDValue Base, Disp, Index;
1232 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1233 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1234 return false;
1235
1236 SDLoc DL(Store);
1237 SDValue Ops[] = {
1238 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1239 Store->getChain()
1240 };
1241 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1242 return true;
1243 }
1244
1245 // Check whether or not the chain ending in StoreNode is suitable for doing
1246 // the {load; op; store} to modify transformation.
isFusableLoadOpStorePattern(StoreSDNode * StoreNode,SDValue StoredVal,SelectionDAG * CurDAG,LoadSDNode * & LoadNode,SDValue & InputChain)1247 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
1248 SDValue StoredVal, SelectionDAG *CurDAG,
1249 LoadSDNode *&LoadNode,
1250 SDValue &InputChain) {
1251 // Is the stored value result 0 of the operation?
1252 if (StoredVal.getResNo() != 0)
1253 return false;
1254
1255 // Are there other uses of the loaded value than the operation?
1256 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1257 return false;
1258
1259 // Is the store non-extending and non-indexed?
1260 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1261 return false;
1262
1263 SDValue Load = StoredVal->getOperand(0);
1264 // Is the stored value a non-extending and non-indexed load?
1265 if (!ISD::isNormalLoad(Load.getNode()))
1266 return false;
1267
1268 // Return LoadNode by reference.
1269 LoadNode = cast<LoadSDNode>(Load);
1270
1271 // Is store the only read of the loaded value?
1272 if (!Load.hasOneUse())
1273 return false;
1274
1275 // Is the address of the store the same as the load?
1276 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1277 LoadNode->getOffset() != StoreNode->getOffset())
1278 return false;
1279
1280 // Check if the chain is produced by the load or is a TokenFactor with
1281 // the load output chain as an operand. Return InputChain by reference.
1282 SDValue Chain = StoreNode->getChain();
1283
1284 bool ChainCheck = false;
1285 if (Chain == Load.getValue(1)) {
1286 ChainCheck = true;
1287 InputChain = LoadNode->getChain();
1288 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1289 SmallVector<SDValue, 4> ChainOps;
1290 SmallVector<const SDNode *, 4> LoopWorklist;
1291 SmallPtrSet<const SDNode *, 16> Visited;
1292 const unsigned int Max = 1024;
1293 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1294 SDValue Op = Chain.getOperand(i);
1295 if (Op == Load.getValue(1)) {
1296 ChainCheck = true;
1297 // Drop Load, but keep its chain. No cycle check necessary.
1298 ChainOps.push_back(Load.getOperand(0));
1299 continue;
1300 }
1301 LoopWorklist.push_back(Op.getNode());
1302 ChainOps.push_back(Op);
1303 }
1304
1305 if (ChainCheck) {
1306 // Add the other operand of StoredVal to worklist.
1307 for (SDValue Op : StoredVal->ops())
1308 if (Op.getNode() != LoadNode)
1309 LoopWorklist.push_back(Op.getNode());
1310
1311 // Check if Load is reachable from any of the nodes in the worklist.
1312 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1313 true))
1314 return false;
1315
1316 // Make a new TokenFactor with all the other input chains except
1317 // for the load.
1318 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1319 MVT::Other, ChainOps);
1320 }
1321 }
1322 if (!ChainCheck)
1323 return false;
1324
1325 return true;
1326 }
1327
1328 // Change a chain of {load; op; store} of the same value into a simple op
1329 // through memory of that value, if the uses of the modified value and its
1330 // address are suitable.
1331 //
1332 // The tablegen pattern memory operand pattern is currently not able to match
1333 // the case where the CC on the original operation are used.
1334 //
1335 // See the equivalent routine in X86ISelDAGToDAG for further comments.
tryFoldLoadStoreIntoMemOperand(SDNode * Node)1336 bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1337 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1338 SDValue StoredVal = StoreNode->getOperand(1);
1339 unsigned Opc = StoredVal->getOpcode();
1340 SDLoc DL(StoreNode);
1341
1342 // Before we try to select anything, make sure this is memory operand size
1343 // and opcode we can handle. Note that this must match the code below that
1344 // actually lowers the opcodes.
1345 EVT MemVT = StoreNode->getMemoryVT();
1346 unsigned NewOpc = 0;
1347 bool NegateOperand = false;
1348 switch (Opc) {
1349 default:
1350 return false;
1351 case SystemZISD::SSUBO:
1352 NegateOperand = true;
1353 LLVM_FALLTHROUGH;
1354 case SystemZISD::SADDO:
1355 if (MemVT == MVT::i32)
1356 NewOpc = SystemZ::ASI;
1357 else if (MemVT == MVT::i64)
1358 NewOpc = SystemZ::AGSI;
1359 else
1360 return false;
1361 break;
1362 case SystemZISD::USUBO:
1363 NegateOperand = true;
1364 LLVM_FALLTHROUGH;
1365 case SystemZISD::UADDO:
1366 if (MemVT == MVT::i32)
1367 NewOpc = SystemZ::ALSI;
1368 else if (MemVT == MVT::i64)
1369 NewOpc = SystemZ::ALGSI;
1370 else
1371 return false;
1372 break;
1373 }
1374
1375 LoadSDNode *LoadNode = nullptr;
1376 SDValue InputChain;
1377 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1378 InputChain))
1379 return false;
1380
1381 SDValue Operand = StoredVal.getOperand(1);
1382 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1383 if (!OperandC)
1384 return false;
1385 auto OperandV = OperandC->getAPIntValue();
1386 if (NegateOperand)
1387 OperandV = -OperandV;
1388 if (OperandV.getMinSignedBits() > 8)
1389 return false;
1390 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1391
1392 SDValue Base, Disp;
1393 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1394 return false;
1395
1396 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1397 MachineSDNode *Result =
1398 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1399 CurDAG->setNodeMemRefs(
1400 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1401
1402 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1403 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1404 CurDAG->RemoveDeadNode(Node);
1405 return true;
1406 }
1407
canUseBlockOperation(StoreSDNode * Store,LoadSDNode * Load) const1408 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1409 LoadSDNode *Load) const {
1410 // Check that the two memory operands have the same size.
1411 if (Load->getMemoryVT() != Store->getMemoryVT())
1412 return false;
1413
1414 // Volatility stops an access from being decomposed.
1415 if (Load->isVolatile() || Store->isVolatile())
1416 return false;
1417
1418 // There's no chance of overlap if the load is invariant.
1419 if (Load->isInvariant() && Load->isDereferenceable())
1420 return true;
1421
1422 // Otherwise we need to check whether there's an alias.
1423 const Value *V1 = Load->getMemOperand()->getValue();
1424 const Value *V2 = Store->getMemOperand()->getValue();
1425 if (!V1 || !V2)
1426 return false;
1427
1428 // Reject equality.
1429 uint64_t Size = Load->getMemoryVT().getStoreSize();
1430 int64_t End1 = Load->getSrcValueOffset() + Size;
1431 int64_t End2 = Store->getSrcValueOffset() + Size;
1432 if (V1 == V2 && End1 == End2)
1433 return false;
1434
1435 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1436 MemoryLocation(V2, End2, Store->getAAInfo()));
1437 }
1438
storeLoadCanUseMVC(SDNode * N) const1439 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1440 auto *Store = cast<StoreSDNode>(N);
1441 auto *Load = cast<LoadSDNode>(Store->getValue());
1442
1443 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1444 // instructions.
1445 uint64_t Size = Load->getMemoryVT().getStoreSize();
1446 if (Size > 1 && Size <= 8) {
1447 // Prefer LHRL, LRL and LGRL.
1448 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1449 return false;
1450 // Prefer STHRL, STRL and STGRL.
1451 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1452 return false;
1453 }
1454
1455 return canUseBlockOperation(Store, Load);
1456 }
1457
storeLoadCanUseBlockBinary(SDNode * N,unsigned I) const1458 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1459 unsigned I) const {
1460 auto *StoreA = cast<StoreSDNode>(N);
1461 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1462 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1463 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1464 canUseBlockOperation(StoreA, LoadB);
1465 }
1466
storeLoadIsAligned(SDNode * N) const1467 bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1468
1469 auto *MemAccess = cast<LSBaseSDNode>(N);
1470 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1471 SDValue BasePtr = MemAccess->getBasePtr();
1472 MachineMemOperand *MMO = MemAccess->getMemOperand();
1473 assert(MMO && "Expected a memory operand.");
1474
1475 // The memory access must have a proper alignment and no index register.
1476 if (MemAccess->getAlignment() < StoreSize ||
1477 !MemAccess->getOffset().isUndef())
1478 return false;
1479
1480 // The MMO must not have an unaligned offset.
1481 if (MMO->getOffset() % StoreSize != 0)
1482 return false;
1483
1484 // An access to GOT or the Constant Pool is aligned.
1485 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1486 if ((PSV->isGOT() || PSV->isConstantPool()))
1487 return true;
1488
1489 // Check the alignment of a Global Address.
1490 if (BasePtr.getNumOperands())
1491 if (GlobalAddressSDNode *GA =
1492 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1493 // The immediate offset must be aligned.
1494 if (GA->getOffset() % StoreSize != 0)
1495 return false;
1496
1497 // The alignment of the symbol itself must be at least the store size.
1498 const GlobalValue *GV = GA->getGlobal();
1499 const DataLayout &DL = GV->getParent()->getDataLayout();
1500 if (GV->getPointerAlignment(DL).value() < StoreSize)
1501 return false;
1502 }
1503
1504 return true;
1505 }
1506
Select(SDNode * Node)1507 void SystemZDAGToDAGISel::Select(SDNode *Node) {
1508 // If we have a custom node, we already have selected!
1509 if (Node->isMachineOpcode()) {
1510 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1511 Node->setNodeId(-1);
1512 return;
1513 }
1514
1515 unsigned Opcode = Node->getOpcode();
1516 switch (Opcode) {
1517 case ISD::OR:
1518 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1519 if (tryRxSBG(Node, SystemZ::ROSBG))
1520 return;
1521 goto or_xor;
1522
1523 case ISD::XOR:
1524 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1525 if (tryRxSBG(Node, SystemZ::RXSBG))
1526 return;
1527 // Fall through.
1528 or_xor:
1529 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1530 // split the operation into two. If both operands here happen to be
1531 // constant, leave this to common code to optimize.
1532 if (Node->getValueType(0) == MVT::i64 &&
1533 Node->getOperand(0).getOpcode() != ISD::Constant)
1534 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1535 uint64_t Val = Op1->getZExtValue();
1536 // Don't split the operation if we can match one of the combined
1537 // logical operations provided by miscellaneous-extensions-3.
1538 if (Subtarget->hasMiscellaneousExtensions3()) {
1539 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1540 // Check whether this expression matches NAND/NOR/NXOR.
1541 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1542 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1543 ChildOpcode == ISD::XOR)
1544 break;
1545 // Check whether this expression matches OR-with-complement
1546 // (or matches an alternate pattern for NXOR).
1547 if (ChildOpcode == ISD::XOR) {
1548 auto Op0 = Node->getOperand(0);
1549 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1550 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1551 break;
1552 }
1553 }
1554 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1555 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1556 Val - uint32_t(Val), uint32_t(Val));
1557 return;
1558 }
1559 }
1560 break;
1561
1562 case ISD::AND:
1563 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1564 if (tryRxSBG(Node, SystemZ::RNSBG))
1565 return;
1566 LLVM_FALLTHROUGH;
1567 case ISD::ROTL:
1568 case ISD::SHL:
1569 case ISD::SRL:
1570 case ISD::ZERO_EXTEND:
1571 if (tryRISBGZero(Node))
1572 return;
1573 break;
1574
1575 case ISD::Constant:
1576 // If this is a 64-bit constant that is out of the range of LLILF,
1577 // LLIHF and LGFI, split it into two 32-bit pieces.
1578 if (Node->getValueType(0) == MVT::i64) {
1579 uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1580 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1581 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1582 uint32_t(Val));
1583 return;
1584 }
1585 }
1586 break;
1587
1588 case SystemZISD::SELECT_CCMASK: {
1589 SDValue Op0 = Node->getOperand(0);
1590 SDValue Op1 = Node->getOperand(1);
1591 // Prefer to put any load first, so that it can be matched as a
1592 // conditional load. Likewise for constants in range for LOCHI.
1593 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1594 (Subtarget->hasLoadStoreOnCond2() &&
1595 Node->getValueType(0).isInteger() &&
1596 Op1.getOpcode() == ISD::Constant &&
1597 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1598 !(Op0.getOpcode() == ISD::Constant &&
1599 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1600 SDValue CCValid = Node->getOperand(2);
1601 SDValue CCMask = Node->getOperand(3);
1602 uint64_t ConstCCValid =
1603 cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1604 uint64_t ConstCCMask =
1605 cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1606 // Invert the condition.
1607 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1608 SDLoc(Node), CCMask.getValueType());
1609 SDValue Op4 = Node->getOperand(4);
1610 SDNode *UpdatedNode =
1611 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1612 if (UpdatedNode != Node) {
1613 // In case this node already exists then replace Node with it.
1614 ReplaceNode(Node, UpdatedNode);
1615 Node = UpdatedNode;
1616 }
1617 }
1618 break;
1619 }
1620
1621 case ISD::INSERT_VECTOR_ELT: {
1622 EVT VT = Node->getValueType(0);
1623 unsigned ElemBitSize = VT.getScalarSizeInBits();
1624 if (ElemBitSize == 32) {
1625 if (tryGather(Node, SystemZ::VGEF))
1626 return;
1627 } else if (ElemBitSize == 64) {
1628 if (tryGather(Node, SystemZ::VGEG))
1629 return;
1630 }
1631 break;
1632 }
1633
1634 case ISD::BUILD_VECTOR: {
1635 auto *BVN = cast<BuildVectorSDNode>(Node);
1636 SystemZVectorConstantInfo VCI(BVN);
1637 if (VCI.isVectorConstantLegal(*Subtarget)) {
1638 loadVectorConstant(VCI, Node);
1639 return;
1640 }
1641 break;
1642 }
1643
1644 case ISD::ConstantFP: {
1645 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1646 if (Imm.isZero() || Imm.isNegZero())
1647 break;
1648 SystemZVectorConstantInfo VCI(Imm);
1649 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1650 assert(Success && "Expected legal FP immediate");
1651 loadVectorConstant(VCI, Node);
1652 return;
1653 }
1654
1655 case ISD::STORE: {
1656 if (tryFoldLoadStoreIntoMemOperand(Node))
1657 return;
1658 auto *Store = cast<StoreSDNode>(Node);
1659 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1660 if (ElemBitSize == 32) {
1661 if (tryScatter(Store, SystemZ::VSCEF))
1662 return;
1663 } else if (ElemBitSize == 64) {
1664 if (tryScatter(Store, SystemZ::VSCEG))
1665 return;
1666 }
1667 break;
1668 }
1669 }
1670
1671 SelectCode(Node);
1672 }
1673
1674 bool SystemZDAGToDAGISel::
SelectInlineAsmMemoryOperand(const SDValue & Op,unsigned ConstraintID,std::vector<SDValue> & OutOps)1675 SelectInlineAsmMemoryOperand(const SDValue &Op,
1676 unsigned ConstraintID,
1677 std::vector<SDValue> &OutOps) {
1678 SystemZAddressingMode::AddrForm Form;
1679 SystemZAddressingMode::DispRange DispRange;
1680 SDValue Base, Disp, Index;
1681
1682 switch(ConstraintID) {
1683 default:
1684 llvm_unreachable("Unexpected asm memory constraint");
1685 case InlineAsm::Constraint_i:
1686 case InlineAsm::Constraint_Q:
1687 // Accept an address with a short displacement, but no index.
1688 Form = SystemZAddressingMode::FormBD;
1689 DispRange = SystemZAddressingMode::Disp12Only;
1690 break;
1691 case InlineAsm::Constraint_R:
1692 // Accept an address with a short displacement and an index.
1693 Form = SystemZAddressingMode::FormBDXNormal;
1694 DispRange = SystemZAddressingMode::Disp12Only;
1695 break;
1696 case InlineAsm::Constraint_S:
1697 // Accept an address with a long displacement, but no index.
1698 Form = SystemZAddressingMode::FormBD;
1699 DispRange = SystemZAddressingMode::Disp20Only;
1700 break;
1701 case InlineAsm::Constraint_T:
1702 case InlineAsm::Constraint_m:
1703 case InlineAsm::Constraint_o:
1704 // Accept an address with a long displacement and an index.
1705 // m works the same as T, as this is the most general case.
1706 // We don't really have any special handling of "offsettable"
1707 // memory addresses, so just treat o the same as m.
1708 Form = SystemZAddressingMode::FormBDXNormal;
1709 DispRange = SystemZAddressingMode::Disp20Only;
1710 break;
1711 }
1712
1713 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1714 const TargetRegisterClass *TRC =
1715 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1716 SDLoc DL(Base);
1717 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1718
1719 // Make sure that the base address doesn't go into %r0.
1720 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1721 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1722 Base.getOpcode() != ISD::Register) {
1723 Base =
1724 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1725 DL, Base.getValueType(),
1726 Base, RC), 0);
1727 }
1728
1729 // Make sure that the index register isn't assigned to %r0 either.
1730 if (Index.getOpcode() != ISD::Register) {
1731 Index =
1732 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1733 DL, Index.getValueType(),
1734 Index, RC), 0);
1735 }
1736
1737 OutOps.push_back(Base);
1738 OutOps.push_back(Disp);
1739 OutOps.push_back(Index);
1740 return false;
1741 }
1742
1743 return true;
1744 }
1745
1746 // IsProfitableToFold - Returns true if is profitable to fold the specific
1747 // operand node N of U during instruction selection that starts at Root.
1748 bool
IsProfitableToFold(SDValue N,SDNode * U,SDNode * Root) const1749 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1750 SDNode *Root) const {
1751 // We want to avoid folding a LOAD into an ICMP node if as a result
1752 // we would be forced to spill the condition code into a GPR.
1753 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1754 if (!N.hasOneUse() || !U->hasOneUse())
1755 return false;
1756
1757 // The user of the CC value will usually be a CopyToReg into the
1758 // physical CC register, which in turn is glued and chained to the
1759 // actual instruction that uses the CC value. Bail out if we have
1760 // anything else than that.
1761 SDNode *CCUser = *U->use_begin();
1762 SDNode *CCRegUser = nullptr;
1763 if (CCUser->getOpcode() == ISD::CopyToReg ||
1764 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1765 for (auto *U : CCUser->uses()) {
1766 if (CCRegUser == nullptr)
1767 CCRegUser = U;
1768 else if (CCRegUser != U)
1769 return false;
1770 }
1771 }
1772 if (CCRegUser == nullptr)
1773 return false;
1774
1775 // If the actual instruction is a branch, the only thing that remains to be
1776 // checked is whether the CCUser chain is a predecessor of the load.
1777 if (CCRegUser->isMachineOpcode() &&
1778 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1779 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1780
1781 // Otherwise, the instruction may have multiple operands, and we need to
1782 // verify that none of them are a predecessor of the load. This is exactly
1783 // the same check that would be done by common code if the CC setter were
1784 // glued to the CC user, so simply invoke that check here.
1785 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1786 return false;
1787 }
1788
1789 return true;
1790 }
1791
1792 namespace {
1793 // Represents a sequence for extracting a 0/1 value from an IPM result:
1794 // (((X ^ XORValue) + AddValue) >> Bit)
1795 struct IPMConversion {
IPMConversion__anon7a3e01970211::IPMConversion1796 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1797 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1798
1799 int64_t XORValue;
1800 int64_t AddValue;
1801 unsigned Bit;
1802 };
1803 } // end anonymous namespace
1804
1805 // Return a sequence for getting a 1 from an IPM result when CC has a
1806 // value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1807 // The handling of CC values outside CCValid doesn't matter.
getIPMConversion(unsigned CCValid,unsigned CCMask)1808 static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1809 // Deal with cases where the result can be taken directly from a bit
1810 // of the IPM result.
1811 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1812 return IPMConversion(0, 0, SystemZ::IPM_CC);
1813 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1814 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1815
1816 // Deal with cases where we can add a value to force the sign bit
1817 // to contain the right value. Putting the bit in 31 means we can
1818 // use SRL rather than RISBG(L), and also makes it easier to get a
1819 // 0/-1 value, so it has priority over the other tests below.
1820 //
1821 // These sequences rely on the fact that the upper two bits of the
1822 // IPM result are zero.
1823 uint64_t TopBit = uint64_t(1) << 31;
1824 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1825 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1826 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1827 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1828 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1829 | SystemZ::CCMASK_1
1830 | SystemZ::CCMASK_2)))
1831 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1832 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1833 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1834 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1835 | SystemZ::CCMASK_2
1836 | SystemZ::CCMASK_3)))
1837 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1838
1839 // Next try inverting the value and testing a bit. 0/1 could be
1840 // handled this way too, but we dealt with that case above.
1841 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1842 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1843
1844 // Handle cases where adding a value forces a non-sign bit to contain
1845 // the right value.
1846 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1847 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1848 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1849 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1850
1851 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1852 // can be done by inverting the low CC bit and applying one of the
1853 // sign-based extractions above.
1854 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1855 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1856 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1857 return IPMConversion(1 << SystemZ::IPM_CC,
1858 TopBit - (3 << SystemZ::IPM_CC), 31);
1859 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1860 | SystemZ::CCMASK_1
1861 | SystemZ::CCMASK_3)))
1862 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1863 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1864 | SystemZ::CCMASK_2
1865 | SystemZ::CCMASK_3)))
1866 return IPMConversion(1 << SystemZ::IPM_CC,
1867 TopBit - (1 << SystemZ::IPM_CC), 31);
1868
1869 llvm_unreachable("Unexpected CC combination");
1870 }
1871
expandSelectBoolean(SDNode * Node)1872 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1873 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1874 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1875 if (!TrueOp || !FalseOp)
1876 return SDValue();
1877 if (FalseOp->getZExtValue() != 0)
1878 return SDValue();
1879 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1880 return SDValue();
1881
1882 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1883 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1884 if (!CCValidOp || !CCMaskOp)
1885 return SDValue();
1886 int CCValid = CCValidOp->getZExtValue();
1887 int CCMask = CCMaskOp->getZExtValue();
1888
1889 SDLoc DL(Node);
1890 SDValue CCReg = Node->getOperand(4);
1891 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1892 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1893
1894 if (IPM.XORValue)
1895 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1896 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1897
1898 if (IPM.AddValue)
1899 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1900 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1901
1902 EVT VT = Node->getValueType(0);
1903 if (VT == MVT::i32 && IPM.Bit == 31) {
1904 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1905 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1906 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1907 } else {
1908 if (VT != MVT::i32)
1909 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1910
1911 if (TrueOp->getSExtValue() == 1) {
1912 // The SHR/AND sequence should get optimized to an RISBG.
1913 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1914 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1915 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1916 CurDAG->getConstant(1, DL, VT));
1917 } else {
1918 // Sign-extend from IPM.Bit using a pair of shifts.
1919 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
1920 int SraAmt = VT.getSizeInBits() - 1;
1921 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
1922 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
1923 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
1924 CurDAG->getConstant(SraAmt, DL, MVT::i32));
1925 }
1926 }
1927
1928 return Result;
1929 }
1930
PreprocessISelDAG()1931 void SystemZDAGToDAGISel::PreprocessISelDAG() {
1932 // If we have conditional immediate loads, we always prefer
1933 // using those over an IPM sequence.
1934 if (Subtarget->hasLoadStoreOnCond2())
1935 return;
1936
1937 bool MadeChange = false;
1938
1939 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
1940 E = CurDAG->allnodes_end();
1941 I != E;) {
1942 SDNode *N = &*I++;
1943 if (N->use_empty())
1944 continue;
1945
1946 SDValue Res;
1947 switch (N->getOpcode()) {
1948 default: break;
1949 case SystemZISD::SELECT_CCMASK:
1950 Res = expandSelectBoolean(N);
1951 break;
1952 }
1953
1954 if (Res) {
1955 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
1956 LLVM_DEBUG(N->dump(CurDAG));
1957 LLVM_DEBUG(dbgs() << "\nNew: ");
1958 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
1959 LLVM_DEBUG(dbgs() << "\n");
1960
1961 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1962 MadeChange = true;
1963 }
1964 }
1965
1966 if (MadeChange)
1967 CurDAG->RemoveDeadNodes();
1968 }
1969