1 | //===- StackMaps.h - StackMaps ----------------------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #ifndef LLVM_CODEGEN_STACKMAPS_H |
10 | #define LLVM_CODEGEN_STACKMAPS_H |
11 | |
12 | #include "llvm/ADT/MapVector.h" |
13 | #include "llvm/ADT/SmallVector.h" |
14 | #include "llvm/CodeGen/MachineInstr.h" |
15 | #include "llvm/IR/CallingConv.h" |
16 | #include "llvm/Support/Debug.h" |
17 | #include <algorithm> |
18 | #include <cassert> |
19 | #include <cstdint> |
20 | #include <vector> |
21 | |
22 | namespace llvm { |
23 | |
24 | class AsmPrinter; |
25 | class MCSymbol; |
26 | class MCExpr; |
27 | class MCStreamer; |
28 | class raw_ostream; |
29 | class TargetRegisterInfo; |
30 | |
31 | /// MI-level stackmap operands. |
32 | /// |
33 | /// MI stackmap operations take the form: |
34 | /// <id>, <numBytes>, live args... |
35 | class StackMapOpers { |
36 | public: |
37 | /// Enumerate the meta operands. |
38 | enum { IDPos, NBytesPos }; |
39 | |
40 | private: |
41 | const MachineInstr* MI; |
42 | |
43 | public: |
44 | explicit StackMapOpers(const MachineInstr *MI); |
45 | |
46 | /// Return the ID for the given stackmap |
47 | uint64_t getID() const { return MI->getOperand(i: IDPos).getImm(); } |
48 | |
49 | /// Return the number of patchable bytes the given stackmap should emit. |
50 | uint32_t getNumPatchBytes() const { |
51 | return MI->getOperand(i: NBytesPos).getImm(); |
52 | } |
53 | |
54 | /// Get the operand index of the variable list of non-argument operands. |
55 | /// These hold the "live state". |
56 | unsigned getVarIdx() const { |
57 | // Skip ID, nShadowBytes. |
58 | return 2; |
59 | } |
60 | }; |
61 | |
62 | /// MI-level patchpoint operands. |
63 | /// |
64 | /// MI patchpoint operations take the form: |
65 | /// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ... |
66 | /// |
67 | /// IR patchpoint intrinsics do not have the <cc> operand because calling |
68 | /// convention is part of the subclass data. |
69 | /// |
70 | /// SD patchpoint nodes do not have a def operand because it is part of the |
71 | /// SDValue. |
72 | /// |
73 | /// Patchpoints following the anyregcc convention are handled specially. For |
74 | /// these, the stack map also records the location of the return value and |
75 | /// arguments. |
76 | class PatchPointOpers { |
77 | public: |
78 | /// Enumerate the meta operands. |
79 | enum { IDPos, NBytesPos, TargetPos, NArgPos, CCPos, MetaEnd }; |
80 | |
81 | private: |
82 | const MachineInstr *MI; |
83 | bool HasDef; |
84 | |
85 | unsigned getMetaIdx(unsigned Pos = 0) const { |
86 | assert(Pos < MetaEnd && "Meta operand index out of range." ); |
87 | return (HasDef ? 1 : 0) + Pos; |
88 | } |
89 | |
90 | const MachineOperand &getMetaOper(unsigned Pos) const { |
91 | return MI->getOperand(i: getMetaIdx(Pos)); |
92 | } |
93 | |
94 | public: |
95 | explicit PatchPointOpers(const MachineInstr *MI); |
96 | |
97 | bool isAnyReg() const { return (getCallingConv() == CallingConv::AnyReg); } |
98 | bool hasDef() const { return HasDef; } |
99 | |
100 | /// Return the ID for the given patchpoint. |
101 | uint64_t getID() const { return getMetaOper(Pos: IDPos).getImm(); } |
102 | |
103 | /// Return the number of patchable bytes the given patchpoint should emit. |
104 | uint32_t getNumPatchBytes() const { |
105 | return getMetaOper(Pos: NBytesPos).getImm(); |
106 | } |
107 | |
108 | /// Returns the target of the underlying call. |
109 | const MachineOperand &getCallTarget() const { |
110 | return getMetaOper(Pos: TargetPos); |
111 | } |
112 | |
113 | /// Returns the calling convention |
114 | CallingConv::ID getCallingConv() const { |
115 | return getMetaOper(Pos: CCPos).getImm(); |
116 | } |
117 | |
118 | unsigned getArgIdx() const { return getMetaIdx() + MetaEnd; } |
119 | |
120 | /// Return the number of call arguments |
121 | uint32_t getNumCallArgs() const { |
122 | return MI->getOperand(i: getMetaIdx(Pos: NArgPos)).getImm(); |
123 | } |
124 | |
125 | /// Get the operand index of the variable list of non-argument operands. |
126 | /// These hold the "live state". |
127 | unsigned getVarIdx() const { |
128 | return getMetaIdx() + MetaEnd + getNumCallArgs(); |
129 | } |
130 | |
131 | /// Get the index at which stack map locations will be recorded. |
132 | /// Arguments are not recorded unless the anyregcc convention is used. |
133 | unsigned getStackMapStartIdx() const { |
134 | if (isAnyReg()) |
135 | return getArgIdx(); |
136 | return getVarIdx(); |
137 | } |
138 | |
139 | /// Get the next scratch register operand index. |
140 | unsigned getNextScratchIdx(unsigned StartIdx = 0) const; |
141 | }; |
142 | |
143 | /// MI-level Statepoint operands |
144 | /// |
145 | /// Statepoint operands take the form: |
146 | /// <id>, <num patch bytes >, <num call arguments>, <call target>, |
147 | /// [call arguments...], |
148 | /// <StackMaps::ConstantOp>, <calling convention>, |
149 | /// <StackMaps::ConstantOp>, <statepoint flags>, |
150 | /// <StackMaps::ConstantOp>, <num deopt args>, [deopt args...], |
151 | /// <StackMaps::ConstantOp>, <num gc pointer args>, [gc pointer args...], |
152 | /// <StackMaps::ConstantOp>, <num gc allocas>, [gc allocas args...], |
153 | /// <StackMaps::ConstantOp>, <num entries in gc map>, [base/derived pairs] |
154 | /// base/derived pairs in gc map are logical indices into <gc pointer args> |
155 | /// section. |
156 | /// All gc pointers assigned to VRegs produce new value (in form of MI Def |
157 | /// operand) and are tied to it. |
158 | class StatepointOpers { |
159 | // TODO:: we should change the STATEPOINT representation so that CC and |
160 | // Flags should be part of meta operands, with args and deopt operands, and |
161 | // gc operands all prefixed by their length and a type code. This would be |
162 | // much more consistent. |
163 | |
164 | // These values are absolute offsets into the operands of the statepoint |
165 | // instruction. |
166 | enum { IDPos, NBytesPos, NCallArgsPos, CallTargetPos, MetaEnd }; |
167 | |
168 | // These values are relative offsets from the start of the statepoint meta |
169 | // arguments (i.e. the end of the call arguments). |
170 | enum { CCOffset = 1, FlagsOffset = 3, NumDeoptOperandsOffset = 5 }; |
171 | |
172 | public: |
173 | explicit StatepointOpers(const MachineInstr *MI) : MI(MI) { |
174 | NumDefs = MI->getNumDefs(); |
175 | } |
176 | |
177 | /// Get index of statepoint ID operand. |
178 | unsigned getIDPos() const { return NumDefs + IDPos; } |
179 | |
180 | /// Get index of Num Patch Bytes operand. |
181 | unsigned getNBytesPos() const { return NumDefs + NBytesPos; } |
182 | |
183 | /// Get index of Num Call Arguments operand. |
184 | unsigned getNCallArgsPos() const { return NumDefs + NCallArgsPos; } |
185 | |
186 | /// Get starting index of non call related arguments |
187 | /// (calling convention, statepoint flags, vm state and gc state). |
188 | unsigned getVarIdx() const { |
189 | return MI->getOperand(i: NumDefs + NCallArgsPos).getImm() + MetaEnd + NumDefs; |
190 | } |
191 | |
192 | /// Get index of Calling Convention operand. |
193 | unsigned getCCIdx() const { return getVarIdx() + CCOffset; } |
194 | |
195 | /// Get index of Flags operand. |
196 | unsigned getFlagsIdx() const { return getVarIdx() + FlagsOffset; } |
197 | |
198 | /// Get index of Number Deopt Arguments operand. |
199 | unsigned getNumDeoptArgsIdx() const { |
200 | return getVarIdx() + NumDeoptOperandsOffset; |
201 | } |
202 | |
203 | /// Return the ID for the given statepoint. |
204 | uint64_t getID() const { return MI->getOperand(i: NumDefs + IDPos).getImm(); } |
205 | |
206 | /// Return the number of patchable bytes the given statepoint should emit. |
207 | uint32_t getNumPatchBytes() const { |
208 | return MI->getOperand(i: NumDefs + NBytesPos).getImm(); |
209 | } |
210 | |
211 | /// Return the target of the underlying call. |
212 | const MachineOperand &getCallTarget() const { |
213 | return MI->getOperand(i: NumDefs + CallTargetPos); |
214 | } |
215 | |
216 | /// Return the calling convention. |
217 | CallingConv::ID getCallingConv() const { |
218 | return MI->getOperand(i: getCCIdx()).getImm(); |
219 | } |
220 | |
221 | /// Return the statepoint flags. |
222 | uint64_t getFlags() const { return MI->getOperand(i: getFlagsIdx()).getImm(); } |
223 | |
224 | uint64_t getNumDeoptArgs() const { |
225 | return MI->getOperand(i: getNumDeoptArgsIdx()).getImm(); |
226 | } |
227 | |
228 | /// Get index of number of gc map entries. |
229 | unsigned getNumGcMapEntriesIdx(); |
230 | |
231 | /// Get index of number of gc allocas. |
232 | unsigned getNumAllocaIdx(); |
233 | |
234 | /// Get index of number of GC pointers. |
235 | unsigned getNumGCPtrIdx(); |
236 | |
237 | /// Get index of first GC pointer operand of -1 if there are none. |
238 | int getFirstGCPtrIdx(); |
239 | |
240 | /// Get vector of base/derived pairs from statepoint. |
241 | /// Elements are indices into GC Pointer operand list (logical). |
242 | /// Returns number of elements in GCMap. |
243 | unsigned |
244 | getGCPointerMap(SmallVectorImpl<std::pair<unsigned, unsigned>> &GCMap); |
245 | |
246 | /// Return true if Reg is used only in operands which can be folded to |
247 | /// stack usage. |
248 | bool isFoldableReg(Register Reg) const; |
249 | |
250 | /// Return true if Reg is used only in operands of MI which can be folded to |
251 | /// stack usage and MI is a statepoint instruction. |
252 | static bool isFoldableReg(const MachineInstr *MI, Register Reg); |
253 | |
254 | private: |
255 | const MachineInstr *MI; |
256 | unsigned NumDefs; |
257 | }; |
258 | |
259 | class StackMaps { |
260 | public: |
261 | struct Location { |
262 | enum LocationType : uint16_t { |
263 | Unprocessed, |
264 | Register, |
265 | Direct, |
266 | Indirect, |
267 | Constant, |
268 | ConstantIndex |
269 | }; |
270 | LocationType Type = Unprocessed; |
271 | uint16_t Size = 0; |
272 | uint16_t Reg = 0; |
273 | int32_t Offset = 0; |
274 | |
275 | Location() = default; |
276 | Location(LocationType Type, uint16_t Size, uint16_t Reg, int32_t Offset) |
277 | : Type(Type), Size(Size), Reg(Reg), Offset(Offset) {} |
278 | }; |
279 | |
280 | struct LiveOutReg { |
281 | uint16_t Reg = 0; |
282 | uint16_t DwarfRegNum = 0; |
283 | uint16_t Size = 0; |
284 | |
285 | LiveOutReg() = default; |
286 | LiveOutReg(uint16_t Reg, uint16_t DwarfRegNum, uint16_t Size) |
287 | : Reg(Reg), DwarfRegNum(DwarfRegNum), Size(Size) {} |
288 | }; |
289 | |
290 | // OpTypes are used to encode information about the following logical |
291 | // operand (which may consist of several MachineOperands) for the |
292 | // OpParser. |
293 | using OpType = enum { DirectMemRefOp, IndirectMemRefOp, ConstantOp }; |
294 | |
295 | StackMaps(AsmPrinter &AP); |
296 | |
297 | /// Get index of next meta operand. |
298 | /// Similar to parseOperand, but does not actually parses operand meaning. |
299 | static unsigned getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx); |
300 | |
301 | void reset() { |
302 | CSInfos.clear(); |
303 | ConstPool.clear(); |
304 | FnInfos.clear(); |
305 | } |
306 | |
307 | using LocationVec = SmallVector<Location, 8>; |
308 | using LiveOutVec = SmallVector<LiveOutReg, 8>; |
309 | using ConstantPool = MapVector<uint64_t, uint64_t>; |
310 | |
311 | struct FunctionInfo { |
312 | uint64_t StackSize = 0; |
313 | uint64_t RecordCount = 1; |
314 | |
315 | FunctionInfo() = default; |
316 | explicit FunctionInfo(uint64_t StackSize) : StackSize(StackSize) {} |
317 | }; |
318 | |
319 | struct CallsiteInfo { |
320 | const MCExpr *CSOffsetExpr = nullptr; |
321 | uint64_t ID = 0; |
322 | LocationVec Locations; |
323 | LiveOutVec LiveOuts; |
324 | |
325 | CallsiteInfo() = default; |
326 | CallsiteInfo(const MCExpr *CSOffsetExpr, uint64_t ID, |
327 | LocationVec &&Locations, LiveOutVec &&LiveOuts) |
328 | : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(std::move(Locations)), |
329 | LiveOuts(std::move(LiveOuts)) {} |
330 | }; |
331 | |
332 | using FnInfoMap = MapVector<const MCSymbol *, FunctionInfo>; |
333 | using CallsiteInfoList = std::vector<CallsiteInfo>; |
334 | |
335 | /// Generate a stackmap record for a stackmap instruction. |
336 | /// |
337 | /// MI must be a raw STACKMAP, not a PATCHPOINT. |
338 | void recordStackMap(const MCSymbol &L, |
339 | const MachineInstr &MI); |
340 | |
341 | /// Generate a stackmap record for a patchpoint instruction. |
342 | void recordPatchPoint(const MCSymbol &L, |
343 | const MachineInstr &MI); |
344 | |
345 | /// Generate a stackmap record for a statepoint instruction. |
346 | void recordStatepoint(const MCSymbol &L, |
347 | const MachineInstr &MI); |
348 | |
349 | /// If there is any stack map data, create a stack map section and serialize |
350 | /// the map info into it. This clears the stack map data structures |
351 | /// afterwards. |
352 | void serializeToStackMapSection(); |
353 | |
354 | /// Get call site info. |
355 | CallsiteInfoList &getCSInfos() { return CSInfos; } |
356 | |
357 | /// Get function info. |
358 | FnInfoMap &getFnInfos() { return FnInfos; } |
359 | |
360 | private: |
361 | static const char *WSMP; |
362 | |
363 | AsmPrinter &AP; |
364 | CallsiteInfoList CSInfos; |
365 | ConstantPool ConstPool; |
366 | FnInfoMap FnInfos; |
367 | |
368 | MachineInstr::const_mop_iterator |
369 | parseOperand(MachineInstr::const_mop_iterator MOI, |
370 | MachineInstr::const_mop_iterator MOE, LocationVec &Locs, |
371 | LiveOutVec &LiveOuts); |
372 | |
373 | /// Specialized parser of statepoint operands. |
374 | /// They do not directly correspond to StackMap record entries. |
375 | void parseStatepointOpers(const MachineInstr &MI, |
376 | MachineInstr::const_mop_iterator MOI, |
377 | MachineInstr::const_mop_iterator MOE, |
378 | LocationVec &Locations, LiveOutVec &LiveOuts); |
379 | |
380 | /// Create a live-out register record for the given register @p Reg. |
381 | LiveOutReg createLiveOutReg(unsigned Reg, |
382 | const TargetRegisterInfo *TRI) const; |
383 | |
384 | /// Parse the register live-out mask and return a vector of live-out |
385 | /// registers that need to be recorded in the stackmap. |
386 | LiveOutVec parseRegisterLiveOutMask(const uint32_t *Mask) const; |
387 | |
388 | /// Record the locations of the operands of the provided instruction in a |
389 | /// record keyed by the provided label. For instructions w/AnyReg calling |
390 | /// convention the return register is also recorded if requested. For |
391 | /// STACKMAP, and PATCHPOINT the label is expected to immediately *preceed* |
392 | /// lowering of the MI to MCInsts. For STATEPOINT, it expected to |
393 | /// immediately *follow*. It's not clear this difference was intentional, |
394 | /// but it exists today. |
395 | void recordStackMapOpers(const MCSymbol &L, |
396 | const MachineInstr &MI, uint64_t ID, |
397 | MachineInstr::const_mop_iterator MOI, |
398 | MachineInstr::const_mop_iterator MOE, |
399 | bool recordResult = false); |
400 | |
401 | /// Emit the stackmap header. |
402 | void (MCStreamer &OS); |
403 | |
404 | /// Emit the function frame record for each function. |
405 | void emitFunctionFrameRecords(MCStreamer &OS); |
406 | |
407 | /// Emit the constant pool. |
408 | void emitConstantPoolEntries(MCStreamer &OS); |
409 | |
410 | /// Emit the callsite info for each stackmap/patchpoint intrinsic call. |
411 | void emitCallsiteEntries(MCStreamer &OS); |
412 | |
413 | void print(raw_ostream &OS); |
414 | void debug() { print(OS&: dbgs()); } |
415 | }; |
416 | |
417 | } // end namespace llvm |
418 | |
419 | #endif // LLVM_CODEGEN_STACKMAPS_H |
420 | |