1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the SystemZ target.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "SystemZTargetMachine.h"
14 #include "SystemZISelLowering.h"
15 #include "llvm/Analysis/AliasAnalysis.h"
16 #include "llvm/CodeGen/SelectionDAGISel.h"
17 #include "llvm/Support/Debug.h"
18 #include "llvm/Support/KnownBits.h"
19 #include "llvm/Support/raw_ostream.h"
20
21 using namespace llvm;
22
23 #define DEBUG_TYPE "systemz-isel"
24 #define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"
25
26 namespace {
27 // Used to build addressing modes.
28 struct SystemZAddressingMode {
29 // The shape of the address.
30 enum AddrForm {
31 // base+displacement
32 FormBD,
33
34 // base+displacement+index for load and store operands
35 FormBDXNormal,
36
37 // base+displacement+index for load address operands
38 FormBDXLA,
39
40 // base+displacement+index+ADJDYNALLOC
41 FormBDXDynAlloc
42 };
43 AddrForm Form;
44
45 // The type of displacement. The enum names here correspond directly
46 // to the definitions in SystemZOperand.td. We could split them into
47 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
48 enum DispRange {
49 Disp12Only,
50 Disp12Pair,
51 Disp20Only,
52 Disp20Only128,
53 Disp20Pair
54 };
55 DispRange DR;
56
57 // The parts of the address. The address is equivalent to:
58 //
59 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
60 SDValue Base;
61 int64_t Disp;
62 SDValue Index;
63 bool IncludesDynAlloc;
64
SystemZAddressingMode__anon49db83c30111::SystemZAddressingMode65 SystemZAddressingMode(AddrForm form, DispRange dr)
66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}
67
68 // True if the address can have an index register.
hasIndexField__anon49db83c30111::SystemZAddressingMode69 bool hasIndexField() { return Form != FormBD; }
70
71 // True if the address can (and must) include ADJDYNALLOC.
isDynAlloc__anon49db83c30111::SystemZAddressingMode72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
dump__anon49db83c30111::SystemZAddressingMode74 void dump(const llvm::SelectionDAG *DAG) {
75 errs() << "SystemZAddressingMode " << this << '\n';
76
77 errs() << " Base ";
78 if (Base.getNode())
79 Base.getNode()->dump(DAG);
80 else
81 errs() << "null\n";
82
83 if (hasIndexField()) {
84 errs() << " Index ";
85 if (Index.getNode())
86 Index.getNode()->dump(DAG);
87 else
88 errs() << "null\n";
89 }
90
91 errs() << " Disp " << Disp;
92 if (IncludesDynAlloc)
93 errs() << " + ADJDYNALLOC";
94 errs() << '\n';
95 }
96 };
97
98 // Return a mask with Count low bits set.
allOnes(unsigned int Count)99 static uint64_t allOnes(unsigned int Count) {
100 assert(Count <= 64);
101 if (Count > 63)
102 return UINT64_MAX;
103 return (uint64_t(1) << Count) - 1;
104 }
105
106 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and
108 // Rotate (I5). The combined operand value is effectively:
109 //
110 // (or (rotl Input, Rotate), ~Mask)
111 //
112 // for RNSBG and:
113 //
114 // (and (rotl Input, Rotate), Mask)
115 //
116 // otherwise. The output value has BitSize bits, although Input may be
117 // narrower (in which case the upper bits are don't care), or wider (in which
118 // case the result will be truncated as part of the operation).
119 struct RxSBGOperands {
RxSBGOperands__anon49db83c30111::RxSBGOperands120 RxSBGOperands(unsigned Op, SDValue N)
121 : Opcode(Op), BitSize(N.getValueSizeInBits()),
122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123 Rotate(0) {}
124
125 unsigned Opcode;
126 unsigned BitSize;
127 uint64_t Mask;
128 SDValue Input;
129 unsigned Start;
130 unsigned End;
131 unsigned Rotate;
132 };
133
134 class SystemZDAGToDAGISel : public SelectionDAGISel {
135 const SystemZSubtarget *Subtarget;
136
137 // Used by SystemZOperands.td to create integer constants.
getImm(const SDNode * Node,uint64_t Imm) const138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140 }
141
getTargetMachine() const142 const SystemZTargetMachine &getTargetMachine() const {
143 return static_cast<const SystemZTargetMachine &>(TM);
144 }
145
getInstrInfo() const146 const SystemZInstrInfo *getInstrInfo() const {
147 return Subtarget->getInstrInfo();
148 }
149
150 // Try to fold more of the base or index of AM into AM, where IsBase
151 // selects between the base and index.
152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154 // Try to describe N in AM, returning true on success.
155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157 // Extract individual target operands from matched address AM.
158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159 SDValue &Base, SDValue &Disp) const;
160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161 SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163 // Try to match Addr as a FormBD address with displacement type DR.
164 // Return true on success, storing the base and displacement in
165 // Base and Disp respectively.
166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167 SDValue &Base, SDValue &Disp) const;
168
169 // Try to match Addr as a FormBDX address with displacement type DR.
170 // Return true on success and if the result had no index. Store the
171 // base and displacement in Base and Disp respectively.
172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173 SDValue &Base, SDValue &Disp) const;
174
175 // Try to match Addr as a FormBDX* address of form Form with
176 // displacement type DR. Return true on success, storing the base,
177 // displacement and index in Base, Disp and Index respectively.
178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179 SystemZAddressingMode::DispRange DR, SDValue Addr,
180 SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182 // PC-relative address matching routines used by SystemZOperands.td.
selectPCRelAddress(SDValue Addr,SDValue & Target) const183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184 if (SystemZISD::isPCREL(Addr.getOpcode())) {
185 Target = Addr.getOperand(0);
186 return true;
187 }
188 return false;
189 }
190
191 // BD matching routines used by SystemZOperands.td.
selectBDAddr12Only(SDValue Addr,SDValue & Base,SDValue & Disp) const192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194 }
selectBDAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197 }
selectBDAddr20Only(SDValue Addr,SDValue & Base,SDValue & Disp) const198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200 }
selectBDAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203 }
204
205 // MVI matching routines used by SystemZOperands.td.
selectMVIAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208 }
selectMVIAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp) const209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211 }
212
213 // BDX matching routines used by SystemZOperands.td.
selectBDXAddr12Only(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215 SDValue &Index) const {
216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217 SystemZAddressingMode::Disp12Only,
218 Addr, Base, Disp, Index);
219 }
selectBDXAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221 SDValue &Index) const {
222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223 SystemZAddressingMode::Disp12Pair,
224 Addr, Base, Disp, Index);
225 }
selectDynAlloc12Only(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227 SDValue &Index) const {
228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229 SystemZAddressingMode::Disp12Only,
230 Addr, Base, Disp, Index);
231 }
selectBDXAddr20Only(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233 SDValue &Index) const {
234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235 SystemZAddressingMode::Disp20Only,
236 Addr, Base, Disp, Index);
237 }
selectBDXAddr20Only128(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239 SDValue &Index) const {
240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241 SystemZAddressingMode::Disp20Only128,
242 Addr, Base, Disp, Index);
243 }
selectBDXAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245 SDValue &Index) const {
246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247 SystemZAddressingMode::Disp20Pair,
248 Addr, Base, Disp, Index);
249 }
selectLAAddr12Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251 SDValue &Index) const {
252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253 SystemZAddressingMode::Disp12Pair,
254 Addr, Base, Disp, Index);
255 }
selectLAAddr20Pair(SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257 SDValue &Index) const {
258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259 SystemZAddressingMode::Disp20Pair,
260 Addr, Base, Disp, Index);
261 }
262
263 // Try to match Addr as an address with a base, 12-bit displacement
264 // and index, where the index is element Elem of a vector.
265 // Return true on success, storing the base, displacement and vector
266 // in Base, Disp and Index respectively.
267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268 SDValue &Disp, SDValue &Index) const;
269
270 // Check whether (or Op (and X InsertMask)) is effectively an insertion
271 // of X into bits InsertMask of some Y != Op. Return true if so and
272 // set Op to that Y.
273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276 // Return true on success.
277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279 // Try to fold some of RxSBG.Input into other fields of RxSBG.
280 // Return true on success.
281 bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283 // Return an undefined value of type VT.
284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286 // Convert N to VT, if it isn't already.
287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289 // Try to implement AND or shift node N using RISBG with the zero flag set.
290 // Return the selected node on success, otherwise return null.
291 bool tryRISBGZero(SDNode *N);
292
293 // Try to use RISBG or Opcode to implement OR or XOR node N.
294 // Return the selected node on success, otherwise return null.
295 bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297 // If Op0 is null, then Node is a constant that can be loaded using:
298 //
299 // (Opcode UpperVal LowerVal)
300 //
301 // If Op0 is nonnull, then Node can be implemented using:
302 //
303 // (Opcode (Opcode Op0 UpperVal) LowerVal)
304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305 uint64_t UpperVal, uint64_t LowerVal);
306
307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308 SDNode *Node);
309
310 SDNode *loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL);
311
312 // Try to use gather instruction Opcode to implement vector insertion N.
313 bool tryGather(SDNode *N, unsigned Opcode);
314
315 // Try to use scatter instruction Opcode to implement store Store.
316 bool tryScatter(StoreSDNode *Store, unsigned Opcode);
317
318 // Change a chain of {load; op; store} of the same value into a simple op
319 // through memory of that value, if the uses of the modified value and its
320 // address are suitable.
321 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
322
323 // Return true if Load and Store are loads and stores of the same size
324 // and are guaranteed not to overlap. Such operations can be implemented
325 // using block (SS-format) instructions.
326 //
327 // Partial overlap would lead to incorrect code, since the block operations
328 // are logically bytewise, even though they have a fast path for the
329 // non-overlapping case. We also need to avoid full overlap (i.e. two
330 // addresses that might be equal at run time) because although that case
331 // would be handled correctly, it might be implemented by millicode.
332 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
333
334 // N is a (store (load Y), X) pattern. Return true if it can use an MVC
335 // from Y to X.
336 bool storeLoadCanUseMVC(SDNode *N) const;
337
338 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true
339 // if A[1 - I] == X and if N can use a block operation like NC from A[I]
340 // to X.
341 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
342
343 // Return true if N (a load or a store) fullfills the alignment
344 // requirements for a PC-relative access.
345 bool storeLoadIsAligned(SDNode *N) const;
346
347 // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
348 SDValue expandSelectBoolean(SDNode *Node);
349
350 public:
351 static char ID;
352
353 SystemZDAGToDAGISel() = delete;
354
SystemZDAGToDAGISel(SystemZTargetMachine & TM,CodeGenOptLevel OptLevel)355 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)
356 : SelectionDAGISel(ID, TM, OptLevel) {}
357
runOnMachineFunction(MachineFunction & MF)358 bool runOnMachineFunction(MachineFunction &MF) override {
359 const Function &F = MF.getFunction();
360 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {
361 if (F.hasFnAttribute("mnop-mcount"))
362 report_fatal_error("mnop-mcount only supported with fentry-call");
363 if (F.hasFnAttribute("mrecord-mcount"))
364 report_fatal_error("mrecord-mcount only supported with fentry-call");
365 }
366
367 Subtarget = &MF.getSubtarget<SystemZSubtarget>();
368 return SelectionDAGISel::runOnMachineFunction(MF);
369 }
370
371 // Override SelectionDAGISel.
372 void Select(SDNode *Node) override;
373 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
374 InlineAsm::ConstraintCode ConstraintID,
375 std::vector<SDValue> &OutOps) override;
376 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
377 void PreprocessISelDAG() override;
378
379 // Include the pieces autogenerated from the target description.
380 #include "SystemZGenDAGISel.inc"
381 };
382 } // end anonymous namespace
383
384 char SystemZDAGToDAGISel::ID = 0;
385
INITIALIZE_PASS(SystemZDAGToDAGISel,DEBUG_TYPE,PASS_NAME,false,false)386 INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)
387
388 FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
389 CodeGenOptLevel OptLevel) {
390 return new SystemZDAGToDAGISel(TM, OptLevel);
391 }
392
393 // Return true if Val should be selected as a displacement for an address
394 // with range DR. Here we're interested in the range of both the instruction
395 // described by DR and of any pairing instruction.
selectDisp(SystemZAddressingMode::DispRange DR,int64_t Val)396 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
397 switch (DR) {
398 case SystemZAddressingMode::Disp12Only:
399 return isUInt<12>(Val);
400
401 case SystemZAddressingMode::Disp12Pair:
402 case SystemZAddressingMode::Disp20Only:
403 case SystemZAddressingMode::Disp20Pair:
404 return isInt<20>(Val);
405
406 case SystemZAddressingMode::Disp20Only128:
407 return isInt<20>(Val) && isInt<20>(Val + 8);
408 }
409 llvm_unreachable("Unhandled displacement range");
410 }
411
412 // Change the base or index in AM to Value, where IsBase selects
413 // between the base and index.
changeComponent(SystemZAddressingMode & AM,bool IsBase,SDValue Value)414 static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
415 SDValue Value) {
416 if (IsBase)
417 AM.Base = Value;
418 else
419 AM.Index = Value;
420 }
421
422 // The base or index of AM is equivalent to Value + ADJDYNALLOC,
423 // where IsBase selects between the base and index. Try to fold the
424 // ADJDYNALLOC into AM.
expandAdjDynAlloc(SystemZAddressingMode & AM,bool IsBase,SDValue Value)425 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
426 SDValue Value) {
427 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {
428 changeComponent(AM, IsBase, Value);
429 AM.IncludesDynAlloc = true;
430 return true;
431 }
432 return false;
433 }
434
435 // The base of AM is equivalent to Base + Index. Try to use Index as
436 // the index register.
expandIndex(SystemZAddressingMode & AM,SDValue Base,SDValue Index)437 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
438 SDValue Index) {
439 if (AM.hasIndexField() && !AM.Index.getNode()) {
440 AM.Base = Base;
441 AM.Index = Index;
442 return true;
443 }
444 return false;
445 }
446
447 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
448 // between the base and index. Try to fold Op1 into AM's displacement.
expandDisp(SystemZAddressingMode & AM,bool IsBase,SDValue Op0,uint64_t Op1)449 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
450 SDValue Op0, uint64_t Op1) {
451 // First try adjusting the displacement.
452 int64_t TestDisp = AM.Disp + Op1;
453 if (selectDisp(AM.DR, TestDisp)) {
454 changeComponent(AM, IsBase, Op0);
455 AM.Disp = TestDisp;
456 return true;
457 }
458
459 // We could consider forcing the displacement into a register and
460 // using it as an index, but it would need to be carefully tuned.
461 return false;
462 }
463
expandAddress(SystemZAddressingMode & AM,bool IsBase) const464 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
465 bool IsBase) const {
466 SDValue N = IsBase ? AM.Base : AM.Index;
467 unsigned Opcode = N.getOpcode();
468 // Look through no-op truncations.
469 if (Opcode == ISD::TRUNCATE && N.getOperand(0).getValueSizeInBits() <= 64) {
470 N = N.getOperand(0);
471 Opcode = N.getOpcode();
472 }
473 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {
474 SDValue Op0 = N.getOperand(0);
475 SDValue Op1 = N.getOperand(1);
476
477 unsigned Op0Code = Op0->getOpcode();
478 unsigned Op1Code = Op1->getOpcode();
479
480 if (Op0Code == SystemZISD::ADJDYNALLOC)
481 return expandAdjDynAlloc(AM, IsBase, Op1);
482 if (Op1Code == SystemZISD::ADJDYNALLOC)
483 return expandAdjDynAlloc(AM, IsBase, Op0);
484
485 if (Op0Code == ISD::Constant)
486 return expandDisp(AM, IsBase, Op1,
487 cast<ConstantSDNode>(Op0)->getSExtValue());
488 if (Op1Code == ISD::Constant)
489 return expandDisp(AM, IsBase, Op0,
490 cast<ConstantSDNode>(Op1)->getSExtValue());
491
492 if (IsBase && expandIndex(AM, Op0, Op1))
493 return true;
494 }
495 if (Opcode == SystemZISD::PCREL_OFFSET) {
496 SDValue Full = N.getOperand(0);
497 SDValue Base = N.getOperand(1);
498 SDValue Anchor = Base.getOperand(0);
499 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
500 cast<GlobalAddressSDNode>(Anchor)->getOffset());
501 return expandDisp(AM, IsBase, Base, Offset);
502 }
503 return false;
504 }
505
506 // Return true if an instruction with displacement range DR should be
507 // used for displacement value Val. selectDisp(DR, Val) must already hold.
isValidDisp(SystemZAddressingMode::DispRange DR,int64_t Val)508 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
509 assert(selectDisp(DR, Val) && "Invalid displacement");
510 switch (DR) {
511 case SystemZAddressingMode::Disp12Only:
512 case SystemZAddressingMode::Disp20Only:
513 case SystemZAddressingMode::Disp20Only128:
514 return true;
515
516 case SystemZAddressingMode::Disp12Pair:
517 // Use the other instruction if the displacement is too large.
518 return isUInt<12>(Val);
519
520 case SystemZAddressingMode::Disp20Pair:
521 // Use the other instruction if the displacement is small enough.
522 return !isUInt<12>(Val);
523 }
524 llvm_unreachable("Unhandled displacement range");
525 }
526
527 // Return true if Base + Disp + Index should be performed by LA(Y).
shouldUseLA(SDNode * Base,int64_t Disp,SDNode * Index)528 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
529 // Don't use LA(Y) for constants.
530 if (!Base)
531 return false;
532
533 // Always use LA(Y) for frame addresses, since we know that the destination
534 // register is almost always (perhaps always) going to be different from
535 // the frame register.
536 if (Base->getOpcode() == ISD::FrameIndex)
537 return true;
538
539 if (Disp) {
540 // Always use LA(Y) if there is a base, displacement and index.
541 if (Index)
542 return true;
543
544 // Always use LA if the displacement is small enough. It should always
545 // be no worse than AGHI (and better if it avoids a move).
546 if (isUInt<12>(Disp))
547 return true;
548
549 // For similar reasons, always use LAY if the constant is too big for AGHI.
550 // LAY should be no worse than AGFI.
551 if (!isInt<16>(Disp))
552 return true;
553 } else {
554 // Don't use LA for plain registers.
555 if (!Index)
556 return false;
557
558 // Don't use LA for plain addition if the index operand is only used
559 // once. It should be a natural two-operand addition in that case.
560 if (Index->hasOneUse())
561 return false;
562
563 // Prefer addition if the second operation is sign-extended, in the
564 // hope of using AGF.
565 unsigned IndexOpcode = Index->getOpcode();
566 if (IndexOpcode == ISD::SIGN_EXTEND ||
567 IndexOpcode == ISD::SIGN_EXTEND_INREG)
568 return false;
569 }
570
571 // Don't use LA for two-operand addition if either operand is only
572 // used once. The addition instructions are better in that case.
573 if (Base->hasOneUse())
574 return false;
575
576 return true;
577 }
578
579 // Return true if Addr is suitable for AM, updating AM if so.
selectAddress(SDValue Addr,SystemZAddressingMode & AM) const580 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
581 SystemZAddressingMode &AM) const {
582 // Start out assuming that the address will need to be loaded separately,
583 // then try to extend it as much as we can.
584 AM.Base = Addr;
585
586 // First try treating the address as a constant.
587 if (Addr.getOpcode() == ISD::Constant &&
588 expandDisp(AM, true, SDValue(),
589 cast<ConstantSDNode>(Addr)->getSExtValue()))
590 ;
591 // Also see if it's a bare ADJDYNALLOC.
592 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
593 expandAdjDynAlloc(AM, true, SDValue()))
594 ;
595 else
596 // Otherwise try expanding each component.
597 while (expandAddress(AM, true) ||
598 (AM.Index.getNode() && expandAddress(AM, false)))
599 continue;
600
601 // Reject cases where it isn't profitable to use LA(Y).
602 if (AM.Form == SystemZAddressingMode::FormBDXLA &&
603 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
604 return false;
605
606 // Reject cases where the other instruction in a pair should be used.
607 if (!isValidDisp(AM.DR, AM.Disp))
608 return false;
609
610 // Make sure that ADJDYNALLOC is included where necessary.
611 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)
612 return false;
613
614 LLVM_DEBUG(AM.dump(CurDAG));
615 return true;
616 }
617
618 // Insert a node into the DAG at least before Pos. This will reposition
619 // the node as needed, and will assign it a node ID that is <= Pos's ID.
620 // Note that this does *not* preserve the uniqueness of node IDs!
621 // The selection DAG must no longer depend on their uniqueness when this
622 // function is used.
insertDAGNode(SelectionDAG * DAG,SDNode * Pos,SDValue N)623 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
624 if (N->getNodeId() == -1 ||
625 (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) >
626 SelectionDAGISel::getUninvalidatedNodeId(Pos))) {
627 DAG->RepositionNode(Pos->getIterator(), N.getNode());
628 // Mark Node as invalid for pruning as after this it may be a successor to a
629 // selected node but otherwise be in the same position of Pos.
630 // Conservatively mark it with the same -abs(Id) to assure node id
631 // invariant is preserved.
632 N->setNodeId(Pos->getNodeId());
633 SelectionDAGISel::InvalidateNodeId(N.getNode());
634 }
635 }
636
getAddressOperands(const SystemZAddressingMode & AM,EVT VT,SDValue & Base,SDValue & Disp) const637 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
638 EVT VT, SDValue &Base,
639 SDValue &Disp) const {
640 Base = AM.Base;
641 if (!Base.getNode())
642 // Register 0 means "no base". This is mostly useful for shifts.
643 Base = CurDAG->getRegister(0, VT);
644 else if (Base.getOpcode() == ISD::FrameIndex) {
645 // Lower a FrameIndex to a TargetFrameIndex.
646 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
647 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
648 } else if (Base.getValueType() != VT) {
649 // Truncate values from i64 to i32, for shifts.
650 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
651 "Unexpected truncation");
652 SDLoc DL(Base);
653 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
654 insertDAGNode(CurDAG, Base.getNode(), Trunc);
655 Base = Trunc;
656 }
657
658 // Lower the displacement to a TargetConstant.
659 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
660 }
661
getAddressOperands(const SystemZAddressingMode & AM,EVT VT,SDValue & Base,SDValue & Disp,SDValue & Index) const662 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
663 EVT VT, SDValue &Base,
664 SDValue &Disp,
665 SDValue &Index) const {
666 getAddressOperands(AM, VT, Base, Disp);
667
668 Index = AM.Index;
669 if (!Index.getNode())
670 // Register 0 means "no index".
671 Index = CurDAG->getRegister(0, VT);
672 }
673
selectBDAddr(SystemZAddressingMode::DispRange DR,SDValue Addr,SDValue & Base,SDValue & Disp) const674 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
675 SDValue Addr, SDValue &Base,
676 SDValue &Disp) const {
677 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
678 if (!selectAddress(Addr, AM))
679 return false;
680
681 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
682 return true;
683 }
684
selectMVIAddr(SystemZAddressingMode::DispRange DR,SDValue Addr,SDValue & Base,SDValue & Disp) const685 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
686 SDValue Addr, SDValue &Base,
687 SDValue &Disp) const {
688 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
689 if (!selectAddress(Addr, AM) || AM.Index.getNode())
690 return false;
691
692 getAddressOperands(AM, Addr.getValueType(), Base, Disp);
693 return true;
694 }
695
selectBDXAddr(SystemZAddressingMode::AddrForm Form,SystemZAddressingMode::DispRange DR,SDValue Addr,SDValue & Base,SDValue & Disp,SDValue & Index) const696 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
697 SystemZAddressingMode::DispRange DR,
698 SDValue Addr, SDValue &Base,
699 SDValue &Disp, SDValue &Index) const {
700 SystemZAddressingMode AM(Form, DR);
701 if (!selectAddress(Addr, AM))
702 return false;
703
704 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
705 return true;
706 }
707
selectBDVAddr12Only(SDValue Addr,SDValue Elem,SDValue & Base,SDValue & Disp,SDValue & Index) const708 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
709 SDValue &Base,
710 SDValue &Disp,
711 SDValue &Index) const {
712 SDValue Regs[2];
713 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
714 Regs[0].getNode() && Regs[1].getNode()) {
715 for (unsigned int I = 0; I < 2; ++I) {
716 Base = Regs[I];
717 Index = Regs[1 - I];
718 // We can't tell here whether the index vector has the right type
719 // for the access; the caller needs to do that instead.
720 if (Index.getOpcode() == ISD::ZERO_EXTEND)
721 Index = Index.getOperand(0);
722 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
723 Index.getOperand(1) == Elem) {
724 Index = Index.getOperand(0);
725 return true;
726 }
727 }
728 }
729 return false;
730 }
731
detectOrAndInsertion(SDValue & Op,uint64_t InsertMask) const732 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
733 uint64_t InsertMask) const {
734 // We're only interested in cases where the insertion is into some operand
735 // of Op, rather than into Op itself. The only useful case is an AND.
736 if (Op.getOpcode() != ISD::AND)
737 return false;
738
739 // We need a constant mask.
740 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
741 if (!MaskNode)
742 return false;
743
744 // It's not an insertion of Op.getOperand(0) if the two masks overlap.
745 uint64_t AndMask = MaskNode->getZExtValue();
746 if (InsertMask & AndMask)
747 return false;
748
749 // It's only an insertion if all bits are covered or are known to be zero.
750 // The inner check covers all cases but is more expensive.
751 uint64_t Used = allOnes(Op.getValueSizeInBits());
752 if (Used != (AndMask | InsertMask)) {
753 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
754 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
755 return false;
756 }
757
758 Op = Op.getOperand(0);
759 return true;
760 }
761
refineRxSBGMask(RxSBGOperands & RxSBG,uint64_t Mask) const762 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
763 uint64_t Mask) const {
764 const SystemZInstrInfo *TII = getInstrInfo();
765 if (RxSBG.Rotate != 0)
766 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
767 Mask &= RxSBG.Mask;
768 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
769 RxSBG.Mask = Mask;
770 return true;
771 }
772 return false;
773 }
774
775 // Return true if any bits of (RxSBG.Input & Mask) are significant.
maskMatters(RxSBGOperands & RxSBG,uint64_t Mask)776 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
777 // Rotate the mask in the same way as RxSBG.Input is rotated.
778 if (RxSBG.Rotate != 0)
779 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
780 return (Mask & RxSBG.Mask) != 0;
781 }
782
expandRxSBG(RxSBGOperands & RxSBG) const783 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
784 SDValue N = RxSBG.Input;
785 unsigned Opcode = N.getOpcode();
786 switch (Opcode) {
787 case ISD::TRUNCATE: {
788 if (RxSBG.Opcode == SystemZ::RNSBG)
789 return false;
790 if (N.getOperand(0).getValueSizeInBits() > 64)
791 return false;
792 uint64_t BitSize = N.getValueSizeInBits();
793 uint64_t Mask = allOnes(BitSize);
794 if (!refineRxSBGMask(RxSBG, Mask))
795 return false;
796 RxSBG.Input = N.getOperand(0);
797 return true;
798 }
799 case ISD::AND: {
800 if (RxSBG.Opcode == SystemZ::RNSBG)
801 return false;
802
803 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
804 if (!MaskNode)
805 return false;
806
807 SDValue Input = N.getOperand(0);
808 uint64_t Mask = MaskNode->getZExtValue();
809 if (!refineRxSBGMask(RxSBG, Mask)) {
810 // If some bits of Input are already known zeros, those bits will have
811 // been removed from the mask. See if adding them back in makes the
812 // mask suitable.
813 KnownBits Known = CurDAG->computeKnownBits(Input);
814 Mask |= Known.Zero.getZExtValue();
815 if (!refineRxSBGMask(RxSBG, Mask))
816 return false;
817 }
818 RxSBG.Input = Input;
819 return true;
820 }
821
822 case ISD::OR: {
823 if (RxSBG.Opcode != SystemZ::RNSBG)
824 return false;
825
826 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
827 if (!MaskNode)
828 return false;
829
830 SDValue Input = N.getOperand(0);
831 uint64_t Mask = ~MaskNode->getZExtValue();
832 if (!refineRxSBGMask(RxSBG, Mask)) {
833 // If some bits of Input are already known ones, those bits will have
834 // been removed from the mask. See if adding them back in makes the
835 // mask suitable.
836 KnownBits Known = CurDAG->computeKnownBits(Input);
837 Mask &= ~Known.One.getZExtValue();
838 if (!refineRxSBGMask(RxSBG, Mask))
839 return false;
840 }
841 RxSBG.Input = Input;
842 return true;
843 }
844
845 case ISD::ROTL: {
846 // Any 64-bit rotate left can be merged into the RxSBG.
847 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)
848 return false;
849 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
850 if (!CountNode)
851 return false;
852
853 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
854 RxSBG.Input = N.getOperand(0);
855 return true;
856 }
857
858 case ISD::ANY_EXTEND:
859 // Bits above the extended operand are don't-care.
860 RxSBG.Input = N.getOperand(0);
861 return true;
862
863 case ISD::ZERO_EXTEND:
864 if (RxSBG.Opcode != SystemZ::RNSBG) {
865 // Restrict the mask to the extended operand.
866 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
867 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
868 return false;
869
870 RxSBG.Input = N.getOperand(0);
871 return true;
872 }
873 [[fallthrough]];
874
875 case ISD::SIGN_EXTEND: {
876 // Check that the extension bits are don't-care (i.e. are masked out
877 // by the final mask).
878 unsigned BitSize = N.getValueSizeInBits();
879 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
880 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
881 // In the case where only the sign bit is active, increase Rotate with
882 // the extension width.
883 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)
884 RxSBG.Rotate += (BitSize - InnerBitSize);
885 else
886 return false;
887 }
888
889 RxSBG.Input = N.getOperand(0);
890 return true;
891 }
892
893 case ISD::SHL: {
894 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
895 if (!CountNode)
896 return false;
897
898 uint64_t Count = CountNode->getZExtValue();
899 unsigned BitSize = N.getValueSizeInBits();
900 if (Count < 1 || Count >= BitSize)
901 return false;
902
903 if (RxSBG.Opcode == SystemZ::RNSBG) {
904 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
905 // count bits from RxSBG.Input are ignored.
906 if (maskMatters(RxSBG, allOnes(Count)))
907 return false;
908 } else {
909 // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
910 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
911 return false;
912 }
913
914 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
915 RxSBG.Input = N.getOperand(0);
916 return true;
917 }
918
919 case ISD::SRL:
920 case ISD::SRA: {
921 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
922 if (!CountNode)
923 return false;
924
925 uint64_t Count = CountNode->getZExtValue();
926 unsigned BitSize = N.getValueSizeInBits();
927 if (Count < 1 || Count >= BitSize)
928 return false;
929
930 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {
931 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
932 // count bits from RxSBG.Input are ignored.
933 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
934 return false;
935 } else {
936 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
937 // which is similar to SLL above.
938 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
939 return false;
940 }
941
942 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
943 RxSBG.Input = N.getOperand(0);
944 return true;
945 }
946 default:
947 return false;
948 }
949 }
950
getUNDEF(const SDLoc & DL,EVT VT) const951 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
952 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
953 return SDValue(N, 0);
954 }
955
convertTo(const SDLoc & DL,EVT VT,SDValue N) const956 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
957 SDValue N) const {
958 if (N.getValueType() == MVT::i32 && VT == MVT::i64)
959 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
960 DL, VT, getUNDEF(DL, MVT::i64), N);
961 if (N.getValueType() == MVT::i64 && VT == MVT::i32)
962 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
963 assert(N.getValueType() == VT && "Unexpected value types");
964 return N;
965 }
966
tryRISBGZero(SDNode * N)967 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
968 SDLoc DL(N);
969 EVT VT = N->getValueType(0);
970 if (!VT.isInteger() || VT.getSizeInBits() > 64)
971 return false;
972 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
973 unsigned Count = 0;
974 while (expandRxSBG(RISBG))
975 // The widening or narrowing is expected to be free.
976 // Counting widening or narrowing as a saved operation will result in
977 // preferring an R*SBG over a simple shift/logical instruction.
978 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
979 RISBG.Input.getOpcode() != ISD::TRUNCATE)
980 Count += 1;
981 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input))
982 return false;
983
984 // Prefer to use normal shift instructions over RISBG, since they can handle
985 // all cases and are sometimes shorter.
986 if (Count == 1 && N->getOpcode() != ISD::AND)
987 return false;
988
989 // Prefer register extensions like LLC over RISBG. Also prefer to start
990 // out with normal ANDs if one instruction would be enough. We can convert
991 // these ANDs into an RISBG later if a three-address instruction is useful.
992 if (RISBG.Rotate == 0) {
993 bool PreferAnd = false;
994 // Prefer AND for any 32-bit and-immediate operation.
995 if (VT == MVT::i32)
996 PreferAnd = true;
997 // As well as for any 64-bit operation that can be implemented via LLC(R),
998 // LLH(R), LLGT(R), or one of the and-immediate instructions.
999 else if (RISBG.Mask == 0xff ||
1000 RISBG.Mask == 0xffff ||
1001 RISBG.Mask == 0x7fffffff ||
1002 SystemZ::isImmLF(~RISBG.Mask) ||
1003 SystemZ::isImmHF(~RISBG.Mask))
1004 PreferAnd = true;
1005 // And likewise for the LLZRGF instruction, which doesn't have a register
1006 // to register version.
1007 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
1008 if (Load->getMemoryVT() == MVT::i32 &&
1009 (Load->getExtensionType() == ISD::EXTLOAD ||
1010 Load->getExtensionType() == ISD::ZEXTLOAD) &&
1011 RISBG.Mask == 0xffffff00 &&
1012 Subtarget->hasLoadAndZeroRightmostByte())
1013 PreferAnd = true;
1014 }
1015 if (PreferAnd) {
1016 // Replace the current node with an AND. Note that the current node
1017 // might already be that same AND, in which case it is already CSE'd
1018 // with it, and we must not call ReplaceNode.
1019 SDValue In = convertTo(DL, VT, RISBG.Input);
1020 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1021 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1022 if (N != New.getNode()) {
1023 insertDAGNode(CurDAG, N, Mask);
1024 insertDAGNode(CurDAG, N, New);
1025 ReplaceNode(N, New.getNode());
1026 N = New.getNode();
1027 }
1028 // Now, select the machine opcode to implement this operation.
1029 if (!N->isMachineOpcode())
1030 SelectCode(N);
1031 return true;
1032 }
1033 }
1034
1035 unsigned Opcode = SystemZ::RISBG;
1036 // Prefer RISBGN if available, since it does not clobber CC.
1037 if (Subtarget->hasMiscellaneousExtensions())
1038 Opcode = SystemZ::RISBGN;
1039 EVT OpcodeVT = MVT::i64;
1040 if (VT == MVT::i32 && Subtarget->hasHighWord() &&
1041 // We can only use the 32-bit instructions if all source bits are
1042 // in the low 32 bits without wrapping, both after rotation (because
1043 // of the smaller range for Start and End) and before rotation
1044 // (because the input value is truncated).
1045 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&
1046 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&
1047 ((RISBG.End + RISBG.Rotate) & 63) >=
1048 ((RISBG.Start + RISBG.Rotate) & 63)) {
1049 Opcode = SystemZ::RISBMux;
1050 OpcodeVT = MVT::i32;
1051 RISBG.Start &= 31;
1052 RISBG.End &= 31;
1053 }
1054 SDValue Ops[5] = {
1055 getUNDEF(DL, OpcodeVT),
1056 convertTo(DL, OpcodeVT, RISBG.Input),
1057 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1058 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1059 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1060 };
1061 SDValue New = convertTo(
1062 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1063 ReplaceNode(N, New.getNode());
1064 return true;
1065 }
1066
tryRxSBG(SDNode * N,unsigned Opcode)1067 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1068 SDLoc DL(N);
1069 EVT VT = N->getValueType(0);
1070 if (!VT.isInteger() || VT.getSizeInBits() > 64)
1071 return false;
1072 // Try treating each operand of N as the second operand of the RxSBG
1073 // and see which goes deepest.
1074 RxSBGOperands RxSBG[] = {
1075 RxSBGOperands(Opcode, N->getOperand(0)),
1076 RxSBGOperands(Opcode, N->getOperand(1))
1077 };
1078 unsigned Count[] = { 0, 0 };
1079 for (unsigned I = 0; I < 2; ++I)
1080 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I]))
1081 // In cases of multiple users it seems better to keep the simple
1082 // instruction as they are one cycle faster, and it also helps in cases
1083 // where both inputs share a common node.
1084 // The widening or narrowing is expected to be free. Counting widening
1085 // or narrowing as a saved operation will result in preferring an R*SBG
1086 // over a simple shift/logical instruction.
1087 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1088 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1089 Count[I] += 1;
1090
1091 // Do nothing if neither operand is suitable.
1092 if (Count[0] == 0 && Count[1] == 0)
1093 return false;
1094
1095 // Pick the deepest second operand.
1096 unsigned I = Count[0] > Count[1] ? 0 : 1;
1097 SDValue Op0 = N->getOperand(I ^ 1);
1098
1099 // Prefer IC for character insertions from memory.
1100 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)
1101 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1102 if (Load->getMemoryVT() == MVT::i8)
1103 return false;
1104
1105 // See whether we can avoid an AND in the first operand by converting
1106 // ROSBG to RISBG.
1107 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {
1108 Opcode = SystemZ::RISBG;
1109 // Prefer RISBGN if available, since it does not clobber CC.
1110 if (Subtarget->hasMiscellaneousExtensions())
1111 Opcode = SystemZ::RISBGN;
1112 }
1113
1114 SDValue Ops[5] = {
1115 convertTo(DL, MVT::i64, Op0),
1116 convertTo(DL, MVT::i64, RxSBG[I].Input),
1117 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1118 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1119 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1120 };
1121 SDValue New = convertTo(
1122 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1123 ReplaceNode(N, New.getNode());
1124 return true;
1125 }
1126
splitLargeImmediate(unsigned Opcode,SDNode * Node,SDValue Op0,uint64_t UpperVal,uint64_t LowerVal)1127 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1128 SDValue Op0, uint64_t UpperVal,
1129 uint64_t LowerVal) {
1130 EVT VT = Node->getValueType(0);
1131 SDLoc DL(Node);
1132 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1133 if (Op0.getNode())
1134 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1135
1136 {
1137 // When we haven't passed in Op0, Upper will be a constant. In order to
1138 // prevent folding back to the large immediate in `Or = getNode(...)` we run
1139 // SelectCode first and end up with an opaque machine node. This means that
1140 // we need to use a handle to keep track of Upper in case it gets CSE'd by
1141 // SelectCode.
1142 //
1143 // Note that in the case where Op0 is passed in we could just call
1144 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1145 // the handle at all, but it's fine to do it here.
1146 //
1147 // TODO: This is a pretty hacky way to do this. Can we do something that
1148 // doesn't require a two paragraph explanation?
1149 HandleSDNode Handle(Upper);
1150 SelectCode(Upper.getNode());
1151 Upper = Handle.getValue();
1152 }
1153
1154 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1155 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1156
1157 ReplaceNode(Node, Or.getNode());
1158
1159 SelectCode(Or.getNode());
1160 }
1161
loadVectorConstant(const SystemZVectorConstantInfo & VCI,SDNode * Node)1162 void SystemZDAGToDAGISel::loadVectorConstant(
1163 const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1164 assert((VCI.Opcode == SystemZISD::BYTE_MASK ||
1165 VCI.Opcode == SystemZISD::REPLICATE ||
1166 VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1167 "Bad opcode!");
1168 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1169 EVT VT = Node->getValueType(0);
1170 SDLoc DL(Node);
1171 SmallVector<SDValue, 2> Ops;
1172 for (unsigned OpVal : VCI.OpVals)
1173 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
1174 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1175
1176 if (VCI.VecVT == VT.getSimpleVT())
1177 ReplaceNode(Node, Op.getNode());
1178 else if (VT.getSizeInBits() == 128) {
1179 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1180 ReplaceNode(Node, BitCast.getNode());
1181 SelectCode(BitCast.getNode());
1182 } else { // float or double
1183 unsigned SubRegIdx =
1184 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64);
1185 ReplaceNode(
1186 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1187 }
1188 SelectCode(Op.getNode());
1189 }
1190
loadPoolVectorConstant(APInt Val,EVT VT,SDLoc DL)1191 SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) {
1192 SDNode *ResNode;
1193 assert (VT.getSizeInBits() == 128);
1194
1195 SDValue CP = CurDAG->getTargetConstantPool(
1196 ConstantInt::get(Type::getInt128Ty(*CurDAG->getContext()), Val),
1197 TLI->getPointerTy(CurDAG->getDataLayout()));
1198
1199 EVT PtrVT = CP.getValueType();
1200 SDValue Ops[] = {
1201 SDValue(CurDAG->getMachineNode(SystemZ::LARL, DL, PtrVT, CP), 0),
1202 CurDAG->getTargetConstant(0, DL, PtrVT),
1203 CurDAG->getRegister(0, PtrVT),
1204 CurDAG->getEntryNode()
1205 };
1206 ResNode = CurDAG->getMachineNode(SystemZ::VL, DL, VT, MVT::Other, Ops);
1207
1208 // Annotate ResNode with memory operand information so that MachineInstr
1209 // queries work properly. This e.g. gives the register allocation the
1210 // required information for rematerialization.
1211 MachineFunction& MF = CurDAG->getMachineFunction();
1212 MachineMemOperand *MemOp =
1213 MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
1214 MachineMemOperand::MOLoad, 16, Align(8));
1215
1216 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
1217 return ResNode;
1218 }
1219
tryGather(SDNode * N,unsigned Opcode)1220 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1221 SDValue ElemV = N->getOperand(2);
1222 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1223 if (!ElemN)
1224 return false;
1225
1226 unsigned Elem = ElemN->getZExtValue();
1227 EVT VT = N->getValueType(0);
1228 if (Elem >= VT.getVectorNumElements())
1229 return false;
1230
1231 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1232 if (!Load || !Load->hasNUsesOfValue(1, 0))
1233 return false;
1234 if (Load->getMemoryVT().getSizeInBits() !=
1235 Load->getValueType(0).getSizeInBits())
1236 return false;
1237
1238 SDValue Base, Disp, Index;
1239 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1240 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1241 return false;
1242
1243 SDLoc DL(Load);
1244 SDValue Ops[] = {
1245 N->getOperand(0), Base, Disp, Index,
1246 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1247 };
1248 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1249 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1250 ReplaceNode(N, Res);
1251 return true;
1252 }
1253
tryScatter(StoreSDNode * Store,unsigned Opcode)1254 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1255 SDValue Value = Store->getValue();
1256 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1257 return false;
1258 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1259 return false;
1260
1261 SDValue ElemV = Value.getOperand(1);
1262 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1263 if (!ElemN)
1264 return false;
1265
1266 SDValue Vec = Value.getOperand(0);
1267 EVT VT = Vec.getValueType();
1268 unsigned Elem = ElemN->getZExtValue();
1269 if (Elem >= VT.getVectorNumElements())
1270 return false;
1271
1272 SDValue Base, Disp, Index;
1273 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1274 Index.getValueType() != VT.changeVectorElementTypeToInteger())
1275 return false;
1276
1277 SDLoc DL(Store);
1278 SDValue Ops[] = {
1279 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1280 Store->getChain()
1281 };
1282 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1283 return true;
1284 }
1285
1286 // Check whether or not the chain ending in StoreNode is suitable for doing
1287 // the {load; op; store} to modify transformation.
isFusableLoadOpStorePattern(StoreSDNode * StoreNode,SDValue StoredVal,SelectionDAG * CurDAG,LoadSDNode * & LoadNode,SDValue & InputChain)1288 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
1289 SDValue StoredVal, SelectionDAG *CurDAG,
1290 LoadSDNode *&LoadNode,
1291 SDValue &InputChain) {
1292 // Is the stored value result 0 of the operation?
1293 if (StoredVal.getResNo() != 0)
1294 return false;
1295
1296 // Are there other uses of the loaded value than the operation?
1297 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1298 return false;
1299
1300 // Is the store non-extending and non-indexed?
1301 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1302 return false;
1303
1304 SDValue Load = StoredVal->getOperand(0);
1305 // Is the stored value a non-extending and non-indexed load?
1306 if (!ISD::isNormalLoad(Load.getNode()))
1307 return false;
1308
1309 // Return LoadNode by reference.
1310 LoadNode = cast<LoadSDNode>(Load);
1311
1312 // Is store the only read of the loaded value?
1313 if (!Load.hasOneUse())
1314 return false;
1315
1316 // Is the address of the store the same as the load?
1317 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1318 LoadNode->getOffset() != StoreNode->getOffset())
1319 return false;
1320
1321 // Check if the chain is produced by the load or is a TokenFactor with
1322 // the load output chain as an operand. Return InputChain by reference.
1323 SDValue Chain = StoreNode->getChain();
1324
1325 bool ChainCheck = false;
1326 if (Chain == Load.getValue(1)) {
1327 ChainCheck = true;
1328 InputChain = LoadNode->getChain();
1329 } else if (Chain.getOpcode() == ISD::TokenFactor) {
1330 SmallVector<SDValue, 4> ChainOps;
1331 SmallVector<const SDNode *, 4> LoopWorklist;
1332 SmallPtrSet<const SDNode *, 16> Visited;
1333 const unsigned int Max = 1024;
1334 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
1335 SDValue Op = Chain.getOperand(i);
1336 if (Op == Load.getValue(1)) {
1337 ChainCheck = true;
1338 // Drop Load, but keep its chain. No cycle check necessary.
1339 ChainOps.push_back(Load.getOperand(0));
1340 continue;
1341 }
1342 LoopWorklist.push_back(Op.getNode());
1343 ChainOps.push_back(Op);
1344 }
1345
1346 if (ChainCheck) {
1347 // Add the other operand of StoredVal to worklist.
1348 for (SDValue Op : StoredVal->ops())
1349 if (Op.getNode() != LoadNode)
1350 LoopWorklist.push_back(Op.getNode());
1351
1352 // Check if Load is reachable from any of the nodes in the worklist.
1353 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1354 true))
1355 return false;
1356
1357 // Make a new TokenFactor with all the other input chains except
1358 // for the load.
1359 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1360 MVT::Other, ChainOps);
1361 }
1362 }
1363 if (!ChainCheck)
1364 return false;
1365
1366 return true;
1367 }
1368
1369 // Change a chain of {load; op; store} of the same value into a simple op
1370 // through memory of that value, if the uses of the modified value and its
1371 // address are suitable.
1372 //
1373 // The tablegen pattern memory operand pattern is currently not able to match
1374 // the case where the CC on the original operation are used.
1375 //
1376 // See the equivalent routine in X86ISelDAGToDAG for further comments.
tryFoldLoadStoreIntoMemOperand(SDNode * Node)1377 bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1378 StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1379 SDValue StoredVal = StoreNode->getOperand(1);
1380 unsigned Opc = StoredVal->getOpcode();
1381 SDLoc DL(StoreNode);
1382
1383 // Before we try to select anything, make sure this is memory operand size
1384 // and opcode we can handle. Note that this must match the code below that
1385 // actually lowers the opcodes.
1386 EVT MemVT = StoreNode->getMemoryVT();
1387 unsigned NewOpc = 0;
1388 bool NegateOperand = false;
1389 switch (Opc) {
1390 default:
1391 return false;
1392 case SystemZISD::SSUBO:
1393 NegateOperand = true;
1394 [[fallthrough]];
1395 case SystemZISD::SADDO:
1396 if (MemVT == MVT::i32)
1397 NewOpc = SystemZ::ASI;
1398 else if (MemVT == MVT::i64)
1399 NewOpc = SystemZ::AGSI;
1400 else
1401 return false;
1402 break;
1403 case SystemZISD::USUBO:
1404 NegateOperand = true;
1405 [[fallthrough]];
1406 case SystemZISD::UADDO:
1407 if (MemVT == MVT::i32)
1408 NewOpc = SystemZ::ALSI;
1409 else if (MemVT == MVT::i64)
1410 NewOpc = SystemZ::ALGSI;
1411 else
1412 return false;
1413 break;
1414 }
1415
1416 LoadSDNode *LoadNode = nullptr;
1417 SDValue InputChain;
1418 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1419 InputChain))
1420 return false;
1421
1422 SDValue Operand = StoredVal.getOperand(1);
1423 auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1424 if (!OperandC)
1425 return false;
1426 auto OperandV = OperandC->getAPIntValue();
1427 if (NegateOperand)
1428 OperandV = -OperandV;
1429 if (OperandV.getSignificantBits() > 8)
1430 return false;
1431 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1432
1433 SDValue Base, Disp;
1434 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1435 return false;
1436
1437 SDValue Ops[] = { Base, Disp, Operand, InputChain };
1438 MachineSDNode *Result =
1439 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1440 CurDAG->setNodeMemRefs(
1441 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1442
1443 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1444 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1445 CurDAG->RemoveDeadNode(Node);
1446 return true;
1447 }
1448
canUseBlockOperation(StoreSDNode * Store,LoadSDNode * Load) const1449 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1450 LoadSDNode *Load) const {
1451 // Check that the two memory operands have the same size.
1452 if (Load->getMemoryVT() != Store->getMemoryVT())
1453 return false;
1454
1455 // Volatility stops an access from being decomposed.
1456 if (Load->isVolatile() || Store->isVolatile())
1457 return false;
1458
1459 // There's no chance of overlap if the load is invariant.
1460 if (Load->isInvariant() && Load->isDereferenceable())
1461 return true;
1462
1463 // Otherwise we need to check whether there's an alias.
1464 const Value *V1 = Load->getMemOperand()->getValue();
1465 const Value *V2 = Store->getMemOperand()->getValue();
1466 if (!V1 || !V2)
1467 return false;
1468
1469 // Reject equality.
1470 uint64_t Size = Load->getMemoryVT().getStoreSize();
1471 int64_t End1 = Load->getSrcValueOffset() + Size;
1472 int64_t End2 = Store->getSrcValueOffset() + Size;
1473 if (V1 == V2 && End1 == End2)
1474 return false;
1475
1476 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),
1477 MemoryLocation(V2, End2, Store->getAAInfo()));
1478 }
1479
storeLoadCanUseMVC(SDNode * N) const1480 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1481 auto *Store = cast<StoreSDNode>(N);
1482 auto *Load = cast<LoadSDNode>(Store->getValue());
1483
1484 // Prefer not to use MVC if either address can use ... RELATIVE LONG
1485 // instructions.
1486 uint64_t Size = Load->getMemoryVT().getStoreSize();
1487 if (Size > 1 && Size <= 8) {
1488 // Prefer LHRL, LRL and LGRL.
1489 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1490 return false;
1491 // Prefer STHRL, STRL and STGRL.
1492 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1493 return false;
1494 }
1495
1496 return canUseBlockOperation(Store, Load);
1497 }
1498
storeLoadCanUseBlockBinary(SDNode * N,unsigned I) const1499 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1500 unsigned I) const {
1501 auto *StoreA = cast<StoreSDNode>(N);
1502 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1503 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1504 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&
1505 canUseBlockOperation(StoreA, LoadB);
1506 }
1507
storeLoadIsAligned(SDNode * N) const1508 bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
1509
1510 auto *MemAccess = cast<LSBaseSDNode>(N);
1511 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
1512 SDValue BasePtr = MemAccess->getBasePtr();
1513 MachineMemOperand *MMO = MemAccess->getMemOperand();
1514 assert(MMO && "Expected a memory operand.");
1515
1516 // The memory access must have a proper alignment and no index register.
1517 if (MemAccess->getAlign().value() < StoreSize ||
1518 !MemAccess->getOffset().isUndef())
1519 return false;
1520
1521 // The MMO must not have an unaligned offset.
1522 if (MMO->getOffset() % StoreSize != 0)
1523 return false;
1524
1525 // An access to GOT or the Constant Pool is aligned.
1526 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())
1527 if ((PSV->isGOT() || PSV->isConstantPool()))
1528 return true;
1529
1530 // Check the alignment of a Global Address.
1531 if (BasePtr.getNumOperands())
1532 if (GlobalAddressSDNode *GA =
1533 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) {
1534 // The immediate offset must be aligned.
1535 if (GA->getOffset() % StoreSize != 0)
1536 return false;
1537
1538 // The alignment of the symbol itself must be at least the store size.
1539 const GlobalValue *GV = GA->getGlobal();
1540 const DataLayout &DL = GV->getParent()->getDataLayout();
1541 if (GV->getPointerAlignment(DL).value() < StoreSize)
1542 return false;
1543 }
1544
1545 return true;
1546 }
1547
Select(SDNode * Node)1548 void SystemZDAGToDAGISel::Select(SDNode *Node) {
1549 // If we have a custom node, we already have selected!
1550 if (Node->isMachineOpcode()) {
1551 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1552 Node->setNodeId(-1);
1553 return;
1554 }
1555
1556 unsigned Opcode = Node->getOpcode();
1557 switch (Opcode) {
1558 case ISD::OR:
1559 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1560 if (tryRxSBG(Node, SystemZ::ROSBG))
1561 return;
1562 goto or_xor;
1563
1564 case ISD::XOR:
1565 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1566 if (tryRxSBG(Node, SystemZ::RXSBG))
1567 return;
1568 // Fall through.
1569 or_xor:
1570 // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1571 // split the operation into two. If both operands here happen to be
1572 // constant, leave this to common code to optimize.
1573 if (Node->getValueType(0) == MVT::i64 &&
1574 Node->getOperand(0).getOpcode() != ISD::Constant)
1575 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1576 uint64_t Val = Op1->getZExtValue();
1577 // Don't split the operation if we can match one of the combined
1578 // logical operations provided by miscellaneous-extensions-3.
1579 if (Subtarget->hasMiscellaneousExtensions3()) {
1580 unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1581 // Check whether this expression matches NAND/NOR/NXOR.
1582 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)
1583 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||
1584 ChildOpcode == ISD::XOR)
1585 break;
1586 // Check whether this expression matches OR-with-complement
1587 // (or matches an alternate pattern for NXOR).
1588 if (ChildOpcode == ISD::XOR) {
1589 auto Op0 = Node->getOperand(0);
1590 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1591 if (Op0Op1->getZExtValue() == (uint64_t)-1)
1592 break;
1593 }
1594 }
1595 // Don't split an XOR with -1 as LCGR/AGHI is more compact.
1596 if (Opcode == ISD::XOR && Op1->isAllOnes())
1597 break;
1598 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) {
1599 splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1600 Val - uint32_t(Val), uint32_t(Val));
1601 return;
1602 }
1603 }
1604 break;
1605
1606 case ISD::AND:
1607 if (Node->getOperand(1).getOpcode() != ISD::Constant)
1608 if (tryRxSBG(Node, SystemZ::RNSBG))
1609 return;
1610 [[fallthrough]];
1611 case ISD::ROTL:
1612 case ISD::SHL:
1613 case ISD::SRL:
1614 case ISD::ZERO_EXTEND:
1615 if (tryRISBGZero(Node))
1616 return;
1617 break;
1618
1619 case ISD::BSWAP:
1620 if (Node->getValueType(0) == MVT::i128) {
1621 SDLoc DL(Node);
1622 SDValue Src = Node->getOperand(0);
1623 Src = CurDAG->getNode(ISD::BITCAST, DL, MVT::v16i8, Src);
1624
1625 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };
1626 SDNode *Mask = loadPoolVectorConstant(APInt(128, Bytes), MVT::v16i8, DL);
1627 SDValue Ops[] = { Src, Src, SDValue(Mask, 0) };
1628 SDValue Res = SDValue(CurDAG->getMachineNode(SystemZ::VPERM, DL,
1629 MVT::v16i8, Ops), 0);
1630
1631 Res = CurDAG->getNode(ISD::BITCAST, DL, MVT::i128, Res);
1632 SDNode *ResNode = Res.getNode();
1633 ReplaceNode(Node, ResNode);
1634 SelectCode(Src.getNode());
1635 SelectCode(ResNode);
1636 return;
1637 }
1638 break;
1639
1640 case ISD::Constant:
1641 // If this is a 64-bit constant that is out of the range of LLILF,
1642 // LLIHF and LGFI, split it into two 32-bit pieces.
1643 if (Node->getValueType(0) == MVT::i64) {
1644 uint64_t Val = Node->getAsZExtVal();
1645 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) {
1646 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1647 uint32_t(Val));
1648 return;
1649 }
1650 }
1651 if (Node->getValueType(0) == MVT::i128) {
1652 const APInt &Val = Node->getAsAPIntVal();
1653 SystemZVectorConstantInfo VCI(Val);
1654 if (VCI.isVectorConstantLegal(*Subtarget)) {
1655 loadVectorConstant(VCI, Node);
1656 return;
1657 }
1658 // If we can't materialize the constant we need to use a literal pool.
1659 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128, SDLoc(Node));
1660 ReplaceNode(Node, ResNode);
1661 return;
1662 }
1663 break;
1664
1665 case SystemZISD::SELECT_CCMASK: {
1666 SDValue Op0 = Node->getOperand(0);
1667 SDValue Op1 = Node->getOperand(1);
1668 // Prefer to put any load first, so that it can be matched as a
1669 // conditional load. Likewise for constants in range for LOCHI.
1670 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||
1671 (Subtarget->hasLoadStoreOnCond2() &&
1672 Node->getValueType(0).isInteger() &&
1673 Node->getValueType(0).getSizeInBits() <= 64 &&
1674 Op1.getOpcode() == ISD::Constant &&
1675 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1676 !(Op0.getOpcode() == ISD::Constant &&
1677 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) {
1678 SDValue CCValid = Node->getOperand(2);
1679 SDValue CCMask = Node->getOperand(3);
1680 uint64_t ConstCCValid = CCValid.getNode()->getAsZExtVal();
1681 uint64_t ConstCCMask = CCMask.getNode()->getAsZExtVal();
1682 // Invert the condition.
1683 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
1684 SDLoc(Node), CCMask.getValueType());
1685 SDValue Op4 = Node->getOperand(4);
1686 SDNode *UpdatedNode =
1687 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1688 if (UpdatedNode != Node) {
1689 // In case this node already exists then replace Node with it.
1690 ReplaceNode(Node, UpdatedNode);
1691 Node = UpdatedNode;
1692 }
1693 }
1694 break;
1695 }
1696
1697 case ISD::INSERT_VECTOR_ELT: {
1698 EVT VT = Node->getValueType(0);
1699 unsigned ElemBitSize = VT.getScalarSizeInBits();
1700 if (ElemBitSize == 32) {
1701 if (tryGather(Node, SystemZ::VGEF))
1702 return;
1703 } else if (ElemBitSize == 64) {
1704 if (tryGather(Node, SystemZ::VGEG))
1705 return;
1706 }
1707 break;
1708 }
1709
1710 case ISD::BUILD_VECTOR: {
1711 auto *BVN = cast<BuildVectorSDNode>(Node);
1712 SystemZVectorConstantInfo VCI(BVN);
1713 if (VCI.isVectorConstantLegal(*Subtarget)) {
1714 loadVectorConstant(VCI, Node);
1715 return;
1716 }
1717 break;
1718 }
1719
1720 case ISD::ConstantFP: {
1721 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1722 if (Imm.isZero() || Imm.isNegZero())
1723 break;
1724 SystemZVectorConstantInfo VCI(Imm);
1725 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1726 assert(Success && "Expected legal FP immediate");
1727 loadVectorConstant(VCI, Node);
1728 return;
1729 }
1730
1731 case ISD::STORE: {
1732 if (tryFoldLoadStoreIntoMemOperand(Node))
1733 return;
1734 auto *Store = cast<StoreSDNode>(Node);
1735 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1736 if (ElemBitSize == 32) {
1737 if (tryScatter(Store, SystemZ::VSCEF))
1738 return;
1739 } else if (ElemBitSize == 64) {
1740 if (tryScatter(Store, SystemZ::VSCEG))
1741 return;
1742 }
1743 break;
1744 }
1745 }
1746
1747 SelectCode(Node);
1748 }
1749
SelectInlineAsmMemoryOperand(const SDValue & Op,InlineAsm::ConstraintCode ConstraintID,std::vector<SDValue> & OutOps)1750 bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(
1751 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1752 std::vector<SDValue> &OutOps) {
1753 SystemZAddressingMode::AddrForm Form;
1754 SystemZAddressingMode::DispRange DispRange;
1755 SDValue Base, Disp, Index;
1756
1757 switch(ConstraintID) {
1758 default:
1759 llvm_unreachable("Unexpected asm memory constraint");
1760 case InlineAsm::ConstraintCode::i:
1761 case InlineAsm::ConstraintCode::Q:
1762 case InlineAsm::ConstraintCode::ZQ:
1763 // Accept an address with a short displacement, but no index.
1764 Form = SystemZAddressingMode::FormBD;
1765 DispRange = SystemZAddressingMode::Disp12Only;
1766 break;
1767 case InlineAsm::ConstraintCode::R:
1768 case InlineAsm::ConstraintCode::ZR:
1769 // Accept an address with a short displacement and an index.
1770 Form = SystemZAddressingMode::FormBDXNormal;
1771 DispRange = SystemZAddressingMode::Disp12Only;
1772 break;
1773 case InlineAsm::ConstraintCode::S:
1774 case InlineAsm::ConstraintCode::ZS:
1775 // Accept an address with a long displacement, but no index.
1776 Form = SystemZAddressingMode::FormBD;
1777 DispRange = SystemZAddressingMode::Disp20Only;
1778 break;
1779 case InlineAsm::ConstraintCode::T:
1780 case InlineAsm::ConstraintCode::m:
1781 case InlineAsm::ConstraintCode::o:
1782 case InlineAsm::ConstraintCode::p:
1783 case InlineAsm::ConstraintCode::ZT:
1784 // Accept an address with a long displacement and an index.
1785 // m works the same as T, as this is the most general case.
1786 // We don't really have any special handling of "offsettable"
1787 // memory addresses, so just treat o the same as m.
1788 Form = SystemZAddressingMode::FormBDXNormal;
1789 DispRange = SystemZAddressingMode::Disp20Only;
1790 break;
1791 }
1792
1793 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1794 const TargetRegisterClass *TRC =
1795 Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1796 SDLoc DL(Base);
1797 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1798
1799 // Make sure that the base address doesn't go into %r0.
1800 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1801 if (Base.getOpcode() != ISD::TargetFrameIndex &&
1802 Base.getOpcode() != ISD::Register) {
1803 Base =
1804 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1805 DL, Base.getValueType(),
1806 Base, RC), 0);
1807 }
1808
1809 // Make sure that the index register isn't assigned to %r0 either.
1810 if (Index.getOpcode() != ISD::Register) {
1811 Index =
1812 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1813 DL, Index.getValueType(),
1814 Index, RC), 0);
1815 }
1816
1817 OutOps.push_back(Base);
1818 OutOps.push_back(Disp);
1819 OutOps.push_back(Index);
1820 return false;
1821 }
1822
1823 return true;
1824 }
1825
1826 // IsProfitableToFold - Returns true if is profitable to fold the specific
1827 // operand node N of U during instruction selection that starts at Root.
1828 bool
IsProfitableToFold(SDValue N,SDNode * U,SDNode * Root) const1829 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1830 SDNode *Root) const {
1831 // We want to avoid folding a LOAD into an ICMP node if as a result
1832 // we would be forced to spill the condition code into a GPR.
1833 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {
1834 if (!N.hasOneUse() || !U->hasOneUse())
1835 return false;
1836
1837 // The user of the CC value will usually be a CopyToReg into the
1838 // physical CC register, which in turn is glued and chained to the
1839 // actual instruction that uses the CC value. Bail out if we have
1840 // anything else than that.
1841 SDNode *CCUser = *U->use_begin();
1842 SDNode *CCRegUser = nullptr;
1843 if (CCUser->getOpcode() == ISD::CopyToReg ||
1844 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) {
1845 for (auto *U : CCUser->uses()) {
1846 if (CCRegUser == nullptr)
1847 CCRegUser = U;
1848 else if (CCRegUser != U)
1849 return false;
1850 }
1851 }
1852 if (CCRegUser == nullptr)
1853 return false;
1854
1855 // If the actual instruction is a branch, the only thing that remains to be
1856 // checked is whether the CCUser chain is a predecessor of the load.
1857 if (CCRegUser->isMachineOpcode() &&
1858 CCRegUser->getMachineOpcode() == SystemZ::BRC)
1859 return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1860
1861 // Otherwise, the instruction may have multiple operands, and we need to
1862 // verify that none of them are a predecessor of the load. This is exactly
1863 // the same check that would be done by common code if the CC setter were
1864 // glued to the CC user, so simply invoke that check here.
1865 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1866 return false;
1867 }
1868
1869 return true;
1870 }
1871
1872 namespace {
1873 // Represents a sequence for extracting a 0/1 value from an IPM result:
1874 // (((X ^ XORValue) + AddValue) >> Bit)
1875 struct IPMConversion {
IPMConversion__anon49db83c30211::IPMConversion1876 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1877 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1878
1879 int64_t XORValue;
1880 int64_t AddValue;
1881 unsigned Bit;
1882 };
1883 } // end anonymous namespace
1884
1885 // Return a sequence for getting a 1 from an IPM result when CC has a
1886 // value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1887 // The handling of CC values outside CCValid doesn't matter.
getIPMConversion(unsigned CCValid,unsigned CCMask)1888 static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1889 // Deal with cases where the result can be taken directly from a bit
1890 // of the IPM result.
1891 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1892 return IPMConversion(0, 0, SystemZ::IPM_CC);
1893 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1894 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1895
1896 // Deal with cases where we can add a value to force the sign bit
1897 // to contain the right value. Putting the bit in 31 means we can
1898 // use SRL rather than RISBG(L), and also makes it easier to get a
1899 // 0/-1 value, so it has priority over the other tests below.
1900 //
1901 // These sequences rely on the fact that the upper two bits of the
1902 // IPM result are zero.
1903 uint64_t TopBit = uint64_t(1) << 31;
1904 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1905 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1906 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1907 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1908 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1909 | SystemZ::CCMASK_1
1910 | SystemZ::CCMASK_2)))
1911 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1912 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1913 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1914 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1915 | SystemZ::CCMASK_2
1916 | SystemZ::CCMASK_3)))
1917 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1918
1919 // Next try inverting the value and testing a bit. 0/1 could be
1920 // handled this way too, but we dealt with that case above.
1921 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1922 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1923
1924 // Handle cases where adding a value forces a non-sign bit to contain
1925 // the right value.
1926 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1927 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1928 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1929 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1930
1931 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are
1932 // can be done by inverting the low CC bit and applying one of the
1933 // sign-based extractions above.
1934 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1935 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1936 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1937 return IPMConversion(1 << SystemZ::IPM_CC,
1938 TopBit - (3 << SystemZ::IPM_CC), 31);
1939 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1940 | SystemZ::CCMASK_1
1941 | SystemZ::CCMASK_3)))
1942 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1943 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1944 | SystemZ::CCMASK_2
1945 | SystemZ::CCMASK_3)))
1946 return IPMConversion(1 << SystemZ::IPM_CC,
1947 TopBit - (1 << SystemZ::IPM_CC), 31);
1948
1949 llvm_unreachable("Unexpected CC combination");
1950 }
1951
expandSelectBoolean(SDNode * Node)1952 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1953 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1954 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1955 if (!TrueOp || !FalseOp)
1956 return SDValue();
1957 if (FalseOp->getZExtValue() != 0)
1958 return SDValue();
1959 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)
1960 return SDValue();
1961
1962 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1963 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1964 if (!CCValidOp || !CCMaskOp)
1965 return SDValue();
1966 int CCValid = CCValidOp->getZExtValue();
1967 int CCMask = CCMaskOp->getZExtValue();
1968
1969 SDLoc DL(Node);
1970 SDValue CCReg = Node->getOperand(4);
1971 IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1972 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1973
1974 if (IPM.XORValue)
1975 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1976 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1977
1978 if (IPM.AddValue)
1979 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1980 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1981
1982 EVT VT = Node->getValueType(0);
1983 if (VT == MVT::i32 && IPM.Bit == 31) {
1984 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;
1985 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1986 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1987 } else {
1988 if (VT != MVT::i32)
1989 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1990
1991 if (TrueOp->getSExtValue() == 1) {
1992 // The SHR/AND sequence should get optimized to an RISBG.
1993 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1994 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1995 Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1996 CurDAG->getConstant(1, DL, VT));
1997 } else {
1998 // Sign-extend from IPM.Bit using a pair of shifts.
1999 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
2000 int SraAmt = VT.getSizeInBits() - 1;
2001 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
2002 CurDAG->getConstant(ShlAmt, DL, MVT::i32));
2003 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
2004 CurDAG->getConstant(SraAmt, DL, MVT::i32));
2005 }
2006 }
2007
2008 return Result;
2009 }
2010
PreprocessISelDAG()2011 void SystemZDAGToDAGISel::PreprocessISelDAG() {
2012 // If we have conditional immediate loads, we always prefer
2013 // using those over an IPM sequence.
2014 if (Subtarget->hasLoadStoreOnCond2())
2015 return;
2016
2017 bool MadeChange = false;
2018
2019 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
2020 E = CurDAG->allnodes_end();
2021 I != E;) {
2022 SDNode *N = &*I++;
2023 if (N->use_empty())
2024 continue;
2025
2026 SDValue Res;
2027 switch (N->getOpcode()) {
2028 default: break;
2029 case SystemZISD::SELECT_CCMASK:
2030 Res = expandSelectBoolean(N);
2031 break;
2032 }
2033
2034 if (Res) {
2035 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");
2036 LLVM_DEBUG(N->dump(CurDAG));
2037 LLVM_DEBUG(dbgs() << "\nNew: ");
2038 LLVM_DEBUG(Res.getNode()->dump(CurDAG));
2039 LLVM_DEBUG(dbgs() << "\n");
2040
2041 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
2042 MadeChange = true;
2043 }
2044 }
2045
2046 if (MadeChange)
2047 CurDAG->RemoveDeadNodes();
2048 }
2049