1 //===- StackMaps.cpp ------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/CodeGen/StackMaps.h"
10 #include "llvm/ADT/DenseMapInfo.h"
11 #include "llvm/ADT/STLExtras.h"
12 #include "llvm/ADT/Twine.h"
13 #include "llvm/CodeGen/AsmPrinter.h"
14 #include "llvm/CodeGen/MachineFrameInfo.h"
15 #include "llvm/CodeGen/MachineFunction.h"
16 #include "llvm/CodeGen/MachineInstr.h"
17 #include "llvm/CodeGen/MachineOperand.h"
18 #include "llvm/CodeGen/TargetOpcodes.h"
19 #include "llvm/CodeGen/TargetRegisterInfo.h"
20 #include "llvm/CodeGen/TargetSubtargetInfo.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/MC/MCContext.h"
23 #include "llvm/MC/MCExpr.h"
24 #include "llvm/MC/MCObjectFileInfo.h"
25 #include "llvm/MC/MCRegisterInfo.h"
26 #include "llvm/MC/MCStreamer.h"
27 #include "llvm/Support/CommandLine.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/ErrorHandling.h"
30 #include "llvm/Support/MathExtras.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include <algorithm>
33 #include <cassert>
34 #include <cstdint>
35 #include <iterator>
36 #include <utility>
37
38 using namespace llvm;
39
40 #define DEBUG_TYPE "stackmaps"
41
42 static cl::opt<int> StackMapVersion(
43 "stackmap-version", cl::init(3), cl::Hidden,
44 cl::desc("Specify the stackmap encoding version (default = 3)"));
45
46 const char *StackMaps::WSMP = "Stack Maps: ";
47
getConstMetaVal(const MachineInstr & MI,unsigned Idx)48 static uint64_t getConstMetaVal(const MachineInstr &MI, unsigned Idx) {
49 assert(MI.getOperand(Idx).isImm() &&
50 MI.getOperand(Idx).getImm() == StackMaps::ConstantOp);
51 const auto &MO = MI.getOperand(Idx + 1);
52 assert(MO.isImm());
53 return MO.getImm();
54 }
55
StackMapOpers(const MachineInstr * MI)56 StackMapOpers::StackMapOpers(const MachineInstr *MI)
57 : MI(MI) {
58 assert(getVarIdx() <= MI->getNumOperands() &&
59 "invalid stackmap definition");
60 }
61
PatchPointOpers(const MachineInstr * MI)62 PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
63 : MI(MI), HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() &&
64 !MI->getOperand(0).isImplicit()) {
65 #ifndef NDEBUG
66 unsigned CheckStartIdx = 0, e = MI->getNumOperands();
67 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
68 MI->getOperand(CheckStartIdx).isDef() &&
69 !MI->getOperand(CheckStartIdx).isImplicit())
70 ++CheckStartIdx;
71
72 assert(getMetaIdx() == CheckStartIdx &&
73 "Unexpected additional definition in Patchpoint intrinsic.");
74 #endif
75 }
76
getNextScratchIdx(unsigned StartIdx) const77 unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
78 if (!StartIdx)
79 StartIdx = getVarIdx();
80
81 // Find the next scratch register (implicit def and early clobber)
82 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
83 while (ScratchIdx < e &&
84 !(MI->getOperand(ScratchIdx).isReg() &&
85 MI->getOperand(ScratchIdx).isDef() &&
86 MI->getOperand(ScratchIdx).isImplicit() &&
87 MI->getOperand(ScratchIdx).isEarlyClobber()))
88 ++ScratchIdx;
89
90 assert(ScratchIdx != e && "No scratch register available");
91 return ScratchIdx;
92 }
93
getNumGcMapEntriesIdx()94 unsigned StatepointOpers::getNumGcMapEntriesIdx() {
95 // Take index of num of allocas and skip all allocas records.
96 unsigned CurIdx = getNumAllocaIdx();
97 unsigned NumAllocas = getConstMetaVal(*MI, CurIdx - 1);
98 CurIdx++;
99 while (NumAllocas--)
100 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
101 return CurIdx + 1; // skip <StackMaps::ConstantOp>
102 }
103
getNumAllocaIdx()104 unsigned StatepointOpers::getNumAllocaIdx() {
105 // Take index of num of gc ptrs and skip all gc ptr records.
106 unsigned CurIdx = getNumGCPtrIdx();
107 unsigned NumGCPtrs = getConstMetaVal(*MI, CurIdx - 1);
108 CurIdx++;
109 while (NumGCPtrs--)
110 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
111 return CurIdx + 1; // skip <StackMaps::ConstantOp>
112 }
113
getNumGCPtrIdx()114 unsigned StatepointOpers::getNumGCPtrIdx() {
115 // Take index of num of deopt args and skip all deopt records.
116 unsigned CurIdx = getNumDeoptArgsIdx();
117 unsigned NumDeoptArgs = getConstMetaVal(*MI, CurIdx - 1);
118 CurIdx++;
119 while (NumDeoptArgs--) {
120 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
121 }
122 return CurIdx + 1; // skip <StackMaps::ConstantOp>
123 }
124
getFirstGCPtrIdx()125 int StatepointOpers::getFirstGCPtrIdx() {
126 unsigned NumGCPtrsIdx = getNumGCPtrIdx();
127 unsigned NumGCPtrs = getConstMetaVal(*MI, NumGCPtrsIdx - 1);
128 if (NumGCPtrs == 0)
129 return -1;
130 ++NumGCPtrsIdx; // skip <num gc ptrs>
131 assert(NumGCPtrsIdx < MI->getNumOperands());
132 return (int)NumGCPtrsIdx;
133 }
134
getGCPointerMap(SmallVectorImpl<std::pair<unsigned,unsigned>> & GCMap)135 unsigned StatepointOpers::getGCPointerMap(
136 SmallVectorImpl<std::pair<unsigned, unsigned>> &GCMap) {
137 unsigned CurIdx = getNumGcMapEntriesIdx();
138 unsigned GCMapSize = getConstMetaVal(*MI, CurIdx - 1);
139 CurIdx++;
140 for (unsigned N = 0; N < GCMapSize; ++N) {
141 unsigned B = MI->getOperand(CurIdx++).getImm();
142 unsigned D = MI->getOperand(CurIdx++).getImm();
143 GCMap.push_back(std::make_pair(B, D));
144 }
145
146 return GCMapSize;
147 }
148
isFoldableReg(Register Reg) const149 bool StatepointOpers::isFoldableReg(Register Reg) const {
150 unsigned FoldableAreaStart = getVarIdx();
151 for (const MachineOperand &MO : MI->uses()) {
152 if (MI->getOperandNo(&MO) >= FoldableAreaStart)
153 break;
154 if (MO.isReg() && MO.getReg() == Reg)
155 return false;
156 }
157 return true;
158 }
159
isFoldableReg(const MachineInstr * MI,Register Reg)160 bool StatepointOpers::isFoldableReg(const MachineInstr *MI, Register Reg) {
161 if (MI->getOpcode() != TargetOpcode::STATEPOINT)
162 return false;
163 return StatepointOpers(MI).isFoldableReg(Reg);
164 }
165
StackMaps(AsmPrinter & AP)166 StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
167 if (StackMapVersion != 3)
168 llvm_unreachable("Unsupported stackmap version!");
169 }
170
getNextMetaArgIdx(const MachineInstr * MI,unsigned CurIdx)171 unsigned StackMaps::getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx) {
172 assert(CurIdx < MI->getNumOperands() && "Bad meta arg index");
173 const auto &MO = MI->getOperand(CurIdx);
174 if (MO.isImm()) {
175 switch (MO.getImm()) {
176 default:
177 llvm_unreachable("Unrecognized operand type.");
178 case StackMaps::DirectMemRefOp:
179 CurIdx += 2;
180 break;
181 case StackMaps::IndirectMemRefOp:
182 CurIdx += 3;
183 break;
184 case StackMaps::ConstantOp:
185 ++CurIdx;
186 break;
187 }
188 }
189 ++CurIdx;
190 assert(CurIdx < MI->getNumOperands() && "points past operand list");
191 return CurIdx;
192 }
193
194 /// Go up the super-register chain until we hit a valid dwarf register number.
getDwarfRegNum(unsigned Reg,const TargetRegisterInfo * TRI)195 static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) {
196 int RegNum = TRI->getDwarfRegNum(Reg, false);
197 for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNum < 0; ++SR)
198 RegNum = TRI->getDwarfRegNum(*SR, false);
199
200 assert(RegNum >= 0 && "Invalid Dwarf register number.");
201 return (unsigned)RegNum;
202 }
203
204 MachineInstr::const_mop_iterator
parseOperand(MachineInstr::const_mop_iterator MOI,MachineInstr::const_mop_iterator MOE,LocationVec & Locs,LiveOutVec & LiveOuts) const205 StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
206 MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
207 LiveOutVec &LiveOuts) const {
208 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
209 if (MOI->isImm()) {
210 switch (MOI->getImm()) {
211 default:
212 llvm_unreachable("Unrecognized operand type.");
213 case StackMaps::DirectMemRefOp: {
214 auto &DL = AP.MF->getDataLayout();
215
216 unsigned Size = DL.getPointerSizeInBits();
217 assert((Size % 8) == 0 && "Need pointer size in bytes.");
218 Size /= 8;
219 Register Reg = (++MOI)->getReg();
220 int64_t Imm = (++MOI)->getImm();
221 Locs.emplace_back(StackMaps::Location::Direct, Size,
222 getDwarfRegNum(Reg, TRI), Imm);
223 break;
224 }
225 case StackMaps::IndirectMemRefOp: {
226 int64_t Size = (++MOI)->getImm();
227 assert(Size > 0 && "Need a valid size for indirect memory locations.");
228 Register Reg = (++MOI)->getReg();
229 int64_t Imm = (++MOI)->getImm();
230 Locs.emplace_back(StackMaps::Location::Indirect, Size,
231 getDwarfRegNum(Reg, TRI), Imm);
232 break;
233 }
234 case StackMaps::ConstantOp: {
235 ++MOI;
236 assert(MOI->isImm() && "Expected constant operand.");
237 int64_t Imm = MOI->getImm();
238 Locs.emplace_back(Location::Constant, sizeof(int64_t), 0, Imm);
239 break;
240 }
241 }
242 return ++MOI;
243 }
244
245 // The physical register number will ultimately be encoded as a DWARF regno.
246 // The stack map also records the size of a spill slot that can hold the
247 // register content. (The runtime can track the actual size of the data type
248 // if it needs to.)
249 if (MOI->isReg()) {
250 // Skip implicit registers (this includes our scratch registers)
251 if (MOI->isImplicit())
252 return ++MOI;
253
254 if (MOI->isUndef()) {
255 // Record `undef` register as constant. Use same value as ISel uses.
256 Locs.emplace_back(Location::Constant, sizeof(int64_t), 0, 0xFEFEFEFE);
257 return ++MOI;
258 }
259
260 assert(MOI->getReg().isPhysical() &&
261 "Virtreg operands should have been rewritten before now.");
262 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(MOI->getReg());
263 assert(!MOI->getSubReg() && "Physical subreg still around.");
264
265 unsigned Offset = 0;
266 unsigned DwarfRegNum = getDwarfRegNum(MOI->getReg(), TRI);
267 unsigned LLVMRegNum = *TRI->getLLVMRegNum(DwarfRegNum, false);
268 unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNum, MOI->getReg());
269 if (SubRegIdx)
270 Offset = TRI->getSubRegIdxOffset(SubRegIdx);
271
272 Locs.emplace_back(Location::Register, TRI->getSpillSize(*RC),
273 DwarfRegNum, Offset);
274 return ++MOI;
275 }
276
277 if (MOI->isRegLiveOut())
278 LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut());
279
280 return ++MOI;
281 }
282
print(raw_ostream & OS)283 void StackMaps::print(raw_ostream &OS) {
284 const TargetRegisterInfo *TRI =
285 AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr;
286 OS << WSMP << "callsites:\n";
287 for (const auto &CSI : CSInfos) {
288 const LocationVec &CSLocs = CSI.Locations;
289 const LiveOutVec &LiveOuts = CSI.LiveOuts;
290
291 OS << WSMP << "callsite " << CSI.ID << "\n";
292 OS << WSMP << " has " << CSLocs.size() << " locations\n";
293
294 unsigned Idx = 0;
295 for (const auto &Loc : CSLocs) {
296 OS << WSMP << "\t\tLoc " << Idx << ": ";
297 switch (Loc.Type) {
298 case Location::Unprocessed:
299 OS << "<Unprocessed operand>";
300 break;
301 case Location::Register:
302 OS << "Register ";
303 if (TRI)
304 OS << printReg(Loc.Reg, TRI);
305 else
306 OS << Loc.Reg;
307 break;
308 case Location::Direct:
309 OS << "Direct ";
310 if (TRI)
311 OS << printReg(Loc.Reg, TRI);
312 else
313 OS << Loc.Reg;
314 if (Loc.Offset)
315 OS << " + " << Loc.Offset;
316 break;
317 case Location::Indirect:
318 OS << "Indirect ";
319 if (TRI)
320 OS << printReg(Loc.Reg, TRI);
321 else
322 OS << Loc.Reg;
323 OS << "+" << Loc.Offset;
324 break;
325 case Location::Constant:
326 OS << "Constant " << Loc.Offset;
327 break;
328 case Location::ConstantIndex:
329 OS << "Constant Index " << Loc.Offset;
330 break;
331 }
332 OS << "\t[encoding: .byte " << Loc.Type << ", .byte 0"
333 << ", .short " << Loc.Size << ", .short " << Loc.Reg << ", .short 0"
334 << ", .int " << Loc.Offset << "]\n";
335 Idx++;
336 }
337
338 OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n";
339
340 Idx = 0;
341 for (const auto &LO : LiveOuts) {
342 OS << WSMP << "\t\tLO " << Idx << ": ";
343 if (TRI)
344 OS << printReg(LO.Reg, TRI);
345 else
346 OS << LO.Reg;
347 OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte "
348 << LO.Size << "]\n";
349 Idx++;
350 }
351 }
352 }
353
354 /// Create a live-out register record for the given register Reg.
355 StackMaps::LiveOutReg
createLiveOutReg(unsigned Reg,const TargetRegisterInfo * TRI) const356 StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
357 unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI);
358 unsigned Size = TRI->getSpillSize(*TRI->getMinimalPhysRegClass(Reg));
359 return LiveOutReg(Reg, DwarfRegNum, Size);
360 }
361
362 /// Parse the register live-out mask and return a vector of live-out registers
363 /// that need to be recorded in the stackmap.
364 StackMaps::LiveOutVec
parseRegisterLiveOutMask(const uint32_t * Mask) const365 StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
366 assert(Mask && "No register mask specified");
367 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
368 LiveOutVec LiveOuts;
369
370 // Create a LiveOutReg for each bit that is set in the register mask.
371 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
372 if ((Mask[Reg / 32] >> (Reg % 32)) & 1)
373 LiveOuts.push_back(createLiveOutReg(Reg, TRI));
374
375 // We don't need to keep track of a register if its super-register is already
376 // in the list. Merge entries that refer to the same dwarf register and use
377 // the maximum size that needs to be spilled.
378
379 llvm::sort(LiveOuts, [](const LiveOutReg &LHS, const LiveOutReg &RHS) {
380 // Only sort by the dwarf register number.
381 return LHS.DwarfRegNum < RHS.DwarfRegNum;
382 });
383
384 for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) {
385 for (auto *II = std::next(I); II != E; ++II) {
386 if (I->DwarfRegNum != II->DwarfRegNum) {
387 // Skip all the now invalid entries.
388 I = --II;
389 break;
390 }
391 I->Size = std::max(I->Size, II->Size);
392 if (TRI->isSuperRegister(I->Reg, II->Reg))
393 I->Reg = II->Reg;
394 II->Reg = 0; // mark for deletion.
395 }
396 }
397
398 llvm::erase_if(LiveOuts, [](const LiveOutReg &LO) { return LO.Reg == 0; });
399
400 return LiveOuts;
401 }
402
403 // See statepoint MI format description in StatepointOpers' class comment
404 // in include/llvm/CodeGen/StackMaps.h
parseStatepointOpers(const MachineInstr & MI,MachineInstr::const_mop_iterator MOI,MachineInstr::const_mop_iterator MOE,LocationVec & Locations,LiveOutVec & LiveOuts)405 void StackMaps::parseStatepointOpers(const MachineInstr &MI,
406 MachineInstr::const_mop_iterator MOI,
407 MachineInstr::const_mop_iterator MOE,
408 LocationVec &Locations,
409 LiveOutVec &LiveOuts) {
410 LLVM_DEBUG(dbgs() << "record statepoint : " << MI << "\n");
411 StatepointOpers SO(&MI);
412 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); // CC
413 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); // Flags
414 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); // Num Deopts
415
416 // Record Deopt Args.
417 unsigned NumDeoptArgs = Locations.back().Offset;
418 assert(Locations.back().Type == Location::Constant);
419 assert(NumDeoptArgs == SO.getNumDeoptArgs());
420
421 while (NumDeoptArgs--)
422 MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
423
424 // Record gc base/derived pairs
425 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
426 ++MOI;
427 assert(MOI->isImm());
428 unsigned NumGCPointers = MOI->getImm();
429 ++MOI;
430 if (NumGCPointers) {
431 // Map logical index of GC ptr to MI operand index.
432 SmallVector<unsigned, 8> GCPtrIndices;
433 unsigned GCPtrIdx = (unsigned)SO.getFirstGCPtrIdx();
434 assert((int)GCPtrIdx != -1);
435 assert(MOI - MI.operands_begin() == GCPtrIdx + 0LL);
436 while (NumGCPointers--) {
437 GCPtrIndices.push_back(GCPtrIdx);
438 GCPtrIdx = StackMaps::getNextMetaArgIdx(&MI, GCPtrIdx);
439 }
440
441 SmallVector<std::pair<unsigned, unsigned>, 8> GCPairs;
442 unsigned NumGCPairs = SO.getGCPointerMap(GCPairs);
443 (void)NumGCPairs;
444 LLVM_DEBUG(dbgs() << "NumGCPairs = " << NumGCPairs << "\n");
445
446 auto MOB = MI.operands_begin();
447 for (auto &P : GCPairs) {
448 assert(P.first < GCPtrIndices.size() && "base pointer index not found");
449 assert(P.second < GCPtrIndices.size() &&
450 "derived pointer index not found");
451 unsigned BaseIdx = GCPtrIndices[P.first];
452 unsigned DerivedIdx = GCPtrIndices[P.second];
453 LLVM_DEBUG(dbgs() << "Base : " << BaseIdx << " Derived : " << DerivedIdx
454 << "\n");
455 (void)parseOperand(MOB + BaseIdx, MOE, Locations, LiveOuts);
456 (void)parseOperand(MOB + DerivedIdx, MOE, Locations, LiveOuts);
457 }
458
459 MOI = MOB + GCPtrIdx;
460 }
461
462 // Record gc allocas
463 assert(MOI < MOE);
464 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
465 ++MOI;
466 unsigned NumAllocas = MOI->getImm();
467 ++MOI;
468 while (NumAllocas--) {
469 MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
470 assert(MOI < MOE);
471 }
472 }
473
recordStackMapOpers(const MCSymbol & MILabel,const MachineInstr & MI,uint64_t ID,MachineInstr::const_mop_iterator MOI,MachineInstr::const_mop_iterator MOE,bool recordResult)474 void StackMaps::recordStackMapOpers(const MCSymbol &MILabel,
475 const MachineInstr &MI, uint64_t ID,
476 MachineInstr::const_mop_iterator MOI,
477 MachineInstr::const_mop_iterator MOE,
478 bool recordResult) {
479 MCContext &OutContext = AP.OutStreamer->getContext();
480
481 LocationVec Locations;
482 LiveOutVec LiveOuts;
483
484 if (recordResult) {
485 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
486 parseOperand(MI.operands_begin(), std::next(MI.operands_begin()), Locations,
487 LiveOuts);
488 }
489
490 // Parse operands.
491 if (MI.getOpcode() == TargetOpcode::STATEPOINT)
492 parseStatepointOpers(MI, MOI, MOE, Locations, LiveOuts);
493 else
494 while (MOI != MOE)
495 MOI = parseOperand(MOI, MOE, Locations, LiveOuts);
496
497 // Move large constants into the constant pool.
498 for (auto &Loc : Locations) {
499 // Constants are encoded as sign-extended integers.
500 // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool.
501 if (Loc.Type == Location::Constant && !isInt<32>(Loc.Offset)) {
502 Loc.Type = Location::ConstantIndex;
503 // ConstPool is intentionally a MapVector of 'uint64_t's (as
504 // opposed to 'int64_t's). We should never be in a situation
505 // where we have to insert either the tombstone or the empty
506 // keys into a map, and for a DenseMap<uint64_t, T> these are
507 // (uint64_t)0 and (uint64_t)-1. They can be and are
508 // represented using 32 bit integers.
509 assert((uint64_t)Loc.Offset != DenseMapInfo<uint64_t>::getEmptyKey() &&
510 (uint64_t)Loc.Offset !=
511 DenseMapInfo<uint64_t>::getTombstoneKey() &&
512 "empty and tombstone keys should fit in 32 bits!");
513 auto Result = ConstPool.insert(std::make_pair(Loc.Offset, Loc.Offset));
514 Loc.Offset = Result.first - ConstPool.begin();
515 }
516 }
517
518 // Create an expression to calculate the offset of the callsite from function
519 // entry.
520 const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub(
521 MCSymbolRefExpr::create(&MILabel, OutContext),
522 MCSymbolRefExpr::create(AP.CurrentFnSymForSize, OutContext), OutContext);
523
524 CSInfos.emplace_back(CSOffsetExpr, ID, std::move(Locations),
525 std::move(LiveOuts));
526
527 // Record the stack size of the current function and update callsite count.
528 const MachineFrameInfo &MFI = AP.MF->getFrameInfo();
529 const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
530 bool HasDynamicFrameSize =
531 MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(*(AP.MF));
532 uint64_t FrameSize = HasDynamicFrameSize ? UINT64_MAX : MFI.getStackSize();
533
534 auto CurrentIt = FnInfos.find(AP.CurrentFnSym);
535 if (CurrentIt != FnInfos.end())
536 CurrentIt->second.RecordCount++;
537 else
538 FnInfos.insert(std::make_pair(AP.CurrentFnSym, FunctionInfo(FrameSize)));
539 }
540
recordStackMap(const MCSymbol & L,const MachineInstr & MI)541 void StackMaps::recordStackMap(const MCSymbol &L, const MachineInstr &MI) {
542 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
543
544 StackMapOpers opers(&MI);
545 const int64_t ID = MI.getOperand(PatchPointOpers::IDPos).getImm();
546 recordStackMapOpers(L, MI, ID, std::next(MI.operands_begin(),
547 opers.getVarIdx()),
548 MI.operands_end());
549 }
550
recordPatchPoint(const MCSymbol & L,const MachineInstr & MI)551 void StackMaps::recordPatchPoint(const MCSymbol &L, const MachineInstr &MI) {
552 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
553
554 PatchPointOpers opers(&MI);
555 const int64_t ID = opers.getID();
556 auto MOI = std::next(MI.operands_begin(), opers.getStackMapStartIdx());
557 recordStackMapOpers(L, MI, ID, MOI, MI.operands_end(),
558 opers.isAnyReg() && opers.hasDef());
559
560 #ifndef NDEBUG
561 // verify anyregcc
562 auto &Locations = CSInfos.back().Locations;
563 if (opers.isAnyReg()) {
564 unsigned NArgs = opers.getNumCallArgs();
565 for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i)
566 assert(Locations[i].Type == Location::Register &&
567 "anyreg arg must be in reg.");
568 }
569 #endif
570 }
571
recordStatepoint(const MCSymbol & L,const MachineInstr & MI)572 void StackMaps::recordStatepoint(const MCSymbol &L, const MachineInstr &MI) {
573 assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint");
574
575 StatepointOpers opers(&MI);
576 const unsigned StartIdx = opers.getVarIdx();
577 recordStackMapOpers(L, MI, opers.getID(), MI.operands_begin() + StartIdx,
578 MI.operands_end(), false);
579 }
580
581 /// Emit the stackmap header.
582 ///
583 /// Header {
584 /// uint8 : Stack Map Version (currently 3)
585 /// uint8 : Reserved (expected to be 0)
586 /// uint16 : Reserved (expected to be 0)
587 /// }
588 /// uint32 : NumFunctions
589 /// uint32 : NumConstants
590 /// uint32 : NumRecords
emitStackmapHeader(MCStreamer & OS)591 void StackMaps::emitStackmapHeader(MCStreamer &OS) {
592 // Header.
593 OS.emitIntValue(StackMapVersion, 1); // Version.
594 OS.emitIntValue(0, 1); // Reserved.
595 OS.emitInt16(0); // Reserved.
596
597 // Num functions.
598 LLVM_DEBUG(dbgs() << WSMP << "#functions = " << FnInfos.size() << '\n');
599 OS.emitInt32(FnInfos.size());
600 // Num constants.
601 LLVM_DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
602 OS.emitInt32(ConstPool.size());
603 // Num callsites.
604 LLVM_DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
605 OS.emitInt32(CSInfos.size());
606 }
607
608 /// Emit the function frame record for each function.
609 ///
610 /// StkSizeRecord[NumFunctions] {
611 /// uint64 : Function Address
612 /// uint64 : Stack Size
613 /// uint64 : Record Count
614 /// }
emitFunctionFrameRecords(MCStreamer & OS)615 void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
616 // Function Frame records.
617 LLVM_DEBUG(dbgs() << WSMP << "functions:\n");
618 for (auto const &FR : FnInfos) {
619 LLVM_DEBUG(dbgs() << WSMP << "function addr: " << FR.first
620 << " frame size: " << FR.second.StackSize
621 << " callsite count: " << FR.second.RecordCount << '\n');
622 OS.emitSymbolValue(FR.first, 8);
623 OS.emitIntValue(FR.second.StackSize, 8);
624 OS.emitIntValue(FR.second.RecordCount, 8);
625 }
626 }
627
628 /// Emit the constant pool.
629 ///
630 /// int64 : Constants[NumConstants]
emitConstantPoolEntries(MCStreamer & OS)631 void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
632 // Constant pool entries.
633 LLVM_DEBUG(dbgs() << WSMP << "constants:\n");
634 for (const auto &ConstEntry : ConstPool) {
635 LLVM_DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
636 OS.emitIntValue(ConstEntry.second, 8);
637 }
638 }
639
640 /// Emit the callsite info for each callsite.
641 ///
642 /// StkMapRecord[NumRecords] {
643 /// uint64 : PatchPoint ID
644 /// uint32 : Instruction Offset
645 /// uint16 : Reserved (record flags)
646 /// uint16 : NumLocations
647 /// Location[NumLocations] {
648 /// uint8 : Register | Direct | Indirect | Constant | ConstantIndex
649 /// uint8 : Size in Bytes
650 /// uint16 : Dwarf RegNum
651 /// int32 : Offset
652 /// }
653 /// uint16 : Padding
654 /// uint16 : NumLiveOuts
655 /// LiveOuts[NumLiveOuts] {
656 /// uint16 : Dwarf RegNum
657 /// uint8 : Reserved
658 /// uint8 : Size in Bytes
659 /// }
660 /// uint32 : Padding (only if required to align to 8 byte)
661 /// }
662 ///
663 /// Location Encoding, Type, Value:
664 /// 0x1, Register, Reg (value in register)
665 /// 0x2, Direct, Reg + Offset (frame index)
666 /// 0x3, Indirect, [Reg + Offset] (spilled value)
667 /// 0x4, Constant, Offset (small constant)
668 /// 0x5, ConstIndex, Constants[Offset] (large constant)
emitCallsiteEntries(MCStreamer & OS)669 void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
670 LLVM_DEBUG(print(dbgs()));
671 // Callsite entries.
672 for (const auto &CSI : CSInfos) {
673 const LocationVec &CSLocs = CSI.Locations;
674 const LiveOutVec &LiveOuts = CSI.LiveOuts;
675
676 // Verify stack map entry. It's better to communicate a problem to the
677 // runtime than crash in case of in-process compilation. Currently, we do
678 // simple overflow checks, but we may eventually communicate other
679 // compilation errors this way.
680 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
681 OS.emitIntValue(UINT64_MAX, 8); // Invalid ID.
682 OS.emitValue(CSI.CSOffsetExpr, 4);
683 OS.emitInt16(0); // Reserved.
684 OS.emitInt16(0); // 0 locations.
685 OS.emitInt16(0); // padding.
686 OS.emitInt16(0); // 0 live-out registers.
687 OS.emitInt32(0); // padding.
688 continue;
689 }
690
691 OS.emitIntValue(CSI.ID, 8);
692 OS.emitValue(CSI.CSOffsetExpr, 4);
693
694 // Reserved for flags.
695 OS.emitInt16(0);
696 OS.emitInt16(CSLocs.size());
697
698 for (const auto &Loc : CSLocs) {
699 OS.emitIntValue(Loc.Type, 1);
700 OS.emitIntValue(0, 1); // Reserved
701 OS.emitInt16(Loc.Size);
702 OS.emitInt16(Loc.Reg);
703 OS.emitInt16(0); // Reserved
704 OS.emitInt32(Loc.Offset);
705 }
706
707 // Emit alignment to 8 byte.
708 OS.emitValueToAlignment(Align(8));
709
710 // Num live-out registers and padding to align to 4 byte.
711 OS.emitInt16(0);
712 OS.emitInt16(LiveOuts.size());
713
714 for (const auto &LO : LiveOuts) {
715 OS.emitInt16(LO.DwarfRegNum);
716 OS.emitIntValue(0, 1);
717 OS.emitIntValue(LO.Size, 1);
718 }
719 // Emit alignment to 8 byte.
720 OS.emitValueToAlignment(Align(8));
721 }
722 }
723
724 /// Serialize the stackmap data.
serializeToStackMapSection()725 void StackMaps::serializeToStackMapSection() {
726 (void)WSMP;
727 // Bail out if there's no stack map data.
728 assert((!CSInfos.empty() || ConstPool.empty()) &&
729 "Expected empty constant pool too!");
730 assert((!CSInfos.empty() || FnInfos.empty()) &&
731 "Expected empty function record too!");
732 if (CSInfos.empty())
733 return;
734
735 MCContext &OutContext = AP.OutStreamer->getContext();
736 MCStreamer &OS = *AP.OutStreamer;
737
738 // Create the section.
739 MCSection *StackMapSection =
740 OutContext.getObjectFileInfo()->getStackMapSection();
741 OS.switchSection(StackMapSection);
742
743 // Emit a dummy symbol to force section inclusion.
744 OS.emitLabel(OutContext.getOrCreateSymbol(Twine("__LLVM_StackMaps")));
745
746 // Serialize data.
747 LLVM_DEBUG(dbgs() << "********** Stack Map Output **********\n");
748 emitStackmapHeader(OS);
749 emitFunctionFrameRecords(OS);
750 emitConstantPoolEntries(OS);
751 emitCallsiteEntries(OS);
752 OS.addBlankLine();
753
754 // Clean up.
755 CSInfos.clear();
756 ConstPool.clear();
757 }
758