1	//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the RISC-V implementation of TargetFrameLowering class.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "RISCVFrameLowering.h"
14	#include "RISCVMachineFunctionInfo.h"
15	#include "RISCVSubtarget.h"
16	#include "llvm/BinaryFormat/Dwarf.h"
17	#include "llvm/CodeGen/MachineFrameInfo.h"
18	#include "llvm/CodeGen/MachineFunction.h"
19	#include "llvm/CodeGen/MachineInstrBuilder.h"
20	#include "llvm/CodeGen/MachineRegisterInfo.h"
21	#include "llvm/CodeGen/RegisterScavenging.h"
22	#include "llvm/IR/DiagnosticInfo.h"
23	#include "llvm/MC/MCDwarf.h"
24	#include "llvm/Support/LEB128.h"
25
26	#include <algorithm>
27
28	using namespace llvm;
29
30	static Align getABIStackAlignment(RISCVABI::ABI ABI) {
31	if (ABI == RISCVABI::ABI_ILP32E)
32	return Align(4);
33	if (ABI == RISCVABI::ABI_LP64E)
34	return Align(8);
35	return Align(16);
36	}
37
38	RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
39	: TargetFrameLowering(
40	StackGrowsDown, getABIStackAlignment(ABI: STI.getTargetABI()),
41	/*LocalAreaOffset=*/0,
42	/*TransientStackAlignment=*/getABIStackAlignment(ABI: STI.getTargetABI())),
43	STI(STI) {}
44
45	static const MCPhysReg AllPopRegs[] = {
46	RISCV::X1, RISCV::X8, RISCV::X9, RISCV::X18, RISCV::X19,
47	RISCV::X20, RISCV::X21, RISCV::X22, RISCV::X23, RISCV::X24,
48	RISCV::X25, RISCV::X26, RISCV::X27};
49
50	// For now we use x3, a.k.a gp, as pointer to shadow call stack.
51	// User should not use x3 in their asm.
52	static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
53	MachineBasicBlock::iterator MI,
54	const DebugLoc &DL) {
55	if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))
56	return;
57
58	const auto &STI = MF.getSubtarget<RISCVSubtarget>();
59	const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
60	Register RAReg = TRI->getRARegister();
61
62	// Do not save RA to the SCS if it's not saved to the regular stack,
63	// i.e. RA is not at risk of being overwritten.
64	std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
65	if (llvm::none_of(
66	Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
67	return;
68
69	const RISCVInstrInfo *TII = STI.getInstrInfo();
70	if (!STI.hasForcedSWShadowStack() && STI.hasStdExtZicfiss()) {
71	BuildMI(MBB, MI, DL, TII->get(RISCV::SSPUSH)).addReg(RAReg);
72	return;
73	}
74
75	Register SCSPReg = RISCVABI::getSCSPReg();
76
77	bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
78	int64_t SlotSize = STI.getXLen() / 8;
79	// Store return address to shadow call stack
80	// addi gp, gp, [4|8]
81	// s[w|d] ra, -[4|8](gp)
82	BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
83	.addReg(SCSPReg, RegState::Define)
84	.addReg(SCSPReg)
85	.addImm(SlotSize)
86	.setMIFlag(MachineInstr::FrameSetup);
87	BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
88	.addReg(RAReg)
89	.addReg(SCSPReg)
90	.addImm(-SlotSize)
91	.setMIFlag(MachineInstr::FrameSetup);
92
93	// Emit a CFI instruction that causes SlotSize to be subtracted from the value
94	// of the shadow stack pointer when unwinding past this frame.
95	char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
96	assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
97
98	char Offset = static_cast<char>(-SlotSize) & 0x7f;
99	const char CFIInst[] = {
100	dwarf::DW_CFA_val_expression,
101	DwarfSCSReg, // register
102	2, // length
103	static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
104	Offset, // addend (sleb128)
105	};
106
107	unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createEscape(
108	L: nullptr, Vals: StringRef(CFIInst, sizeof(CFIInst))));
109	BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
110	.addCFIIndex(CFIIndex)
111	.setMIFlag(MachineInstr::FrameSetup);
112	}
113
114	static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
115	MachineBasicBlock::iterator MI,
116	const DebugLoc &DL) {
117	if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))
118	return;
119
120	const auto &STI = MF.getSubtarget<RISCVSubtarget>();
121	Register RAReg = STI.getRegisterInfo()->getRARegister();
122
123	// See emitSCSPrologue() above.
124	std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
125	if (llvm::none_of(
126	Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
127	return;
128
129	const RISCVInstrInfo *TII = STI.getInstrInfo();
130	if (!STI.hasForcedSWShadowStack() && STI.hasStdExtZicfiss()) {
131	BuildMI(MBB, MI, DL, TII->get(RISCV::SSPOPCHK)).addReg(RAReg);
132	return;
133	}
134
135	Register SCSPReg = RISCVABI::getSCSPReg();
136
137	bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
138	int64_t SlotSize = STI.getXLen() / 8;
139	// Load return address from shadow call stack
140	// l[w|d] ra, -[4|8](gp)
141	// addi gp, gp, -[4|8]
142	BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW))
143	.addReg(RAReg, RegState::Define)
144	.addReg(SCSPReg)
145	.addImm(-SlotSize)
146	.setMIFlag(MachineInstr::FrameDestroy);
147	BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
148	.addReg(SCSPReg, RegState::Define)
149	.addReg(SCSPReg)
150	.addImm(-SlotSize)
151	.setMIFlag(MachineInstr::FrameDestroy);
152	// Restore the SCS pointer
153	unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRestore(
154	L: nullptr, Register: STI.getRegisterInfo()->getDwarfRegNum(SCSPReg, /*IsEH*/ true)));
155	BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
156	.addCFIIndex(CFIIndex)
157	.setMIFlags(MachineInstr::FrameDestroy);
158	}
159
160	// Get the ID of the libcall used for spilling and restoring callee saved
161	// registers. The ID is representative of the number of registers saved or
162	// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
163	// single register.
164	static int getLibCallID(const MachineFunction &MF,
165	const std::vector<CalleeSavedInfo> &CSI) {
166	const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
167
168	if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
169	return -1;
170
171	Register MaxReg = RISCV::NoRegister;
172	for (auto &CS : CSI)
173	// RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to
174	// registers which can be saved by libcall.
175	if (CS.getFrameIdx() < 0)
176	MaxReg = std::max(a: MaxReg.id(), b: CS.getReg().id());
177
178	if (MaxReg == RISCV::NoRegister)
179	return -1;
180
181	switch (MaxReg) {
182	default:
183	llvm_unreachable("Something has gone wrong!");
184	case /*s11*/ RISCV::X27: return 12;
185	case /*s10*/ RISCV::X26: return 11;
186	case /*s9*/ RISCV::X25: return 10;
187	case /*s8*/ RISCV::X24: return 9;
188	case /*s7*/ RISCV::X23: return 8;
189	case /*s6*/ RISCV::X22: return 7;
190	case /*s5*/ RISCV::X21: return 6;
191	case /*s4*/ RISCV::X20: return 5;
192	case /*s3*/ RISCV::X19: return 4;
193	case /*s2*/ RISCV::X18: return 3;
194	case /*s1*/ RISCV::X9: return 2;
195	case /*s0*/ RISCV::X8: return 1;
196	case /*ra*/ RISCV::X1: return 0;
197	}
198	}
199
200	// Get the name of the libcall used for spilling callee saved registers.
201	// If this function will not use save/restore libcalls, then return a nullptr.
202	static const char *
203	getSpillLibCallName(const MachineFunction &MF,
204	const std::vector<CalleeSavedInfo> &CSI) {
205	static const char *const SpillLibCalls[] = {
206	"__riscv_save_0",
207	"__riscv_save_1",
208	"__riscv_save_2",
209	"__riscv_save_3",
210	"__riscv_save_4",
211	"__riscv_save_5",
212	"__riscv_save_6",
213	"__riscv_save_7",
214	"__riscv_save_8",
215	"__riscv_save_9",
216	"__riscv_save_10",
217	"__riscv_save_11",
218	"__riscv_save_12"
219	};
220
221	int LibCallID = getLibCallID(MF, CSI);
222	if (LibCallID == -1)
223	return nullptr;
224	return SpillLibCalls[LibCallID];
225	}
226
227	// Get the name of the libcall used for restoring callee saved registers.
228	// If this function will not use save/restore libcalls, then return a nullptr.
229	static const char *
230	getRestoreLibCallName(const MachineFunction &MF,
231	const std::vector<CalleeSavedInfo> &CSI) {
232	static const char *const RestoreLibCalls[] = {
233	"__riscv_restore_0",
234	"__riscv_restore_1",
235	"__riscv_restore_2",
236	"__riscv_restore_3",
237	"__riscv_restore_4",
238	"__riscv_restore_5",
239	"__riscv_restore_6",
240	"__riscv_restore_7",
241	"__riscv_restore_8",
242	"__riscv_restore_9",
243	"__riscv_restore_10",
244	"__riscv_restore_11",
245	"__riscv_restore_12"
246	};
247
248	int LibCallID = getLibCallID(MF, CSI);
249	if (LibCallID == -1)
250	return nullptr;
251	return RestoreLibCalls[LibCallID];
252	}
253
254	// Return encoded value and register count for PUSH/POP instruction,
255	// representing registers to store/load.
256	static std::pair<unsigned, unsigned>
257	getPushPopEncodingAndNum(const Register MaxReg) {
258	switch (MaxReg) {
259	default:
260	llvm_unreachable("Unexpected Reg for Push/Pop Inst");
261	case RISCV::X27: /*s11*/
262	case RISCV::X26: /*s10*/
263	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S11, y: 13);
264	case RISCV::X25: /*s9*/
265	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S9, y: 11);
266	case RISCV::X24: /*s8*/
267	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S8, y: 10);
268	case RISCV::X23: /*s7*/
269	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S7, y: 9);
270	case RISCV::X22: /*s6*/
271	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S6, y: 8);
272	case RISCV::X21: /*s5*/
273	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S5, y: 7);
274	case RISCV::X20: /*s4*/
275	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S4, y: 6);
276	case RISCV::X19: /*s3*/
277	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S3, y: 5);
278	case RISCV::X18: /*s2*/
279	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S2, y: 4);
280	case RISCV::X9: /*s1*/
281	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S1, y: 3);
282	case RISCV::X8: /*s0*/
283	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0, y: 2);
284	case RISCV::X1: /*ra*/
285	return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA, y: 1);
286	}
287	}
288
289	// Get the max reg of Push/Pop for restoring callee saved registers.
290	static Register getMaxPushPopReg(const MachineFunction &MF,
291	const std::vector<CalleeSavedInfo> &CSI) {
292	Register MaxPushPopReg = RISCV::NoRegister;
293	for (auto &CS : CSI) {
294	if (llvm::is_contained(Range: AllPopRegs, Element: CS.getReg().id()))
295	MaxPushPopReg = std::max(a: MaxPushPopReg.id(), b: CS.getReg().id());
296	}
297	// if rlist is {rs, s0-s10}, then s11 will also be included
298	if (MaxPushPopReg == RISCV::X26)
299	MaxPushPopReg = RISCV::X27;
300	return MaxPushPopReg;
301	}
302
303	// Return true if the specified function should have a dedicated frame
304	// pointer register. This is true if frame pointer elimination is
305	// disabled, if it needs dynamic stack realignment, if the function has
306	// variable sized allocas, or if the frame address is taken.
307	bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const {
308	const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
309
310	const MachineFrameInfo &MFI = MF.getFrameInfo();
311	return MF.getTarget().Options.DisableFramePointerElim(MF) ||
312	RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
313	MFI.isFrameAddressTaken();
314	}
315
316	bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
317	const MachineFrameInfo &MFI = MF.getFrameInfo();
318	const TargetRegisterInfo *TRI = STI.getRegisterInfo();
319
320	// If we do not reserve stack space for outgoing arguments in prologue,
321	// we will adjust the stack pointer before call instruction. After the
322	// adjustment, we can not use SP to access the stack objects for the
323	// arguments. Instead, use BP to access these stack objects.
324	return (MFI.hasVarSizedObjects() ||
325	(!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
326	MFI.getMaxCallFrameSize() != 0))) &&
327	TRI->hasStackRealignment(MF);
328	}
329
330	// Determines the size of the frame and maximum call frame size.
331	void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
332	MachineFrameInfo &MFI = MF.getFrameInfo();
333	auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
334
335	// Get the number of bytes to allocate from the FrameInfo.
336	uint64_t FrameSize = MFI.getStackSize();
337
338	// Get the alignment.
339	Align StackAlign = getStackAlign();
340
341	// Make sure the frame is aligned.
342	FrameSize = alignTo(Size: FrameSize, A: StackAlign);
343
344	// Update frame info.
345	MFI.setStackSize(FrameSize);
346
347	// When using SP or BP to access stack objects, we may require extra padding
348	// to ensure the bottom of the RVV stack is correctly aligned within the main
349	// stack. We calculate this as the amount required to align the scalar local
350	// variable section up to the RVV alignment.
351	const TargetRegisterInfo *TRI = STI.getRegisterInfo();
352	if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
353	int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
354	RVFI->getVarArgsSaveSize();
355	if (auto RVVPadding =
356	offsetToAlignment(Value: ScalarLocalVarSize, Alignment: RVFI->getRVVStackAlign()))
357	RVFI->setRVVPadding(RVVPadding);
358	}
359	}
360
361	// Returns the stack size including RVV padding (when required), rounded back
362	// up to the required stack alignment.
363	uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
364	const MachineFunction &MF) const {
365	const MachineFrameInfo &MFI = MF.getFrameInfo();
366	auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
367	return alignTo(Size: MFI.getStackSize() + RVFI->getRVVPadding(), A: getStackAlign());
368	}
369
370	// Returns the register used to hold the frame pointer.
371	static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; }
372
373	// Returns the register used to hold the stack pointer.
374	static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; }
375
376	static SmallVector<CalleeSavedInfo, 8>
377	getUnmanagedCSI(const MachineFunction &MF,
378	const std::vector<CalleeSavedInfo> &CSI) {
379	const MachineFrameInfo &MFI = MF.getFrameInfo();
380	SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
381
382	for (auto &CS : CSI) {
383	int FI = CS.getFrameIdx();
384	if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default)
385	NonLibcallCSI.push_back(Elt: CS);
386	}
387
388	return NonLibcallCSI;
389	}
390
391	static SmallVector<CalleeSavedInfo, 8>
392	getRVVCalleeSavedInfo(const MachineFunction &MF,
393	const std::vector<CalleeSavedInfo> &CSI) {
394	const MachineFrameInfo &MFI = MF.getFrameInfo();
395	SmallVector<CalleeSavedInfo, 8> RVVCSI;
396
397	for (auto &CS : CSI) {
398	int FI = CS.getFrameIdx();
399	if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector)
400	RVVCSI.push_back(Elt: CS);
401	}
402
403	return RVVCSI;
404	}
405
406	void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF,
407	MachineBasicBlock &MBB,
408	MachineBasicBlock::iterator MBBI,
409	const DebugLoc &DL, int64_t Amount,
410	MachineInstr::MIFlag Flag) const {
411	assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
412
413	const Register SPReg = getSPReg(STI);
414
415	// Optimize compile time offset case
416	StackOffset Offset = StackOffset::getScalable(Scalable: Amount);
417	if (auto VLEN = STI.getRealVLen()) {
418	// 1. Multiply the number of v-slots by the (constant) length of register
419	const int64_t VLENB = *VLEN / 8;
420	assert(Amount % 8 == 0 &&
421	"Reserve the stack by the multiple of one vector size.");
422	const int64_t NumOfVReg = Amount / 8;
423	const int64_t FixedOffset = NumOfVReg * VLENB;
424	if (!isInt<32>(x: FixedOffset)) {
425	report_fatal_error(
426	reason: "Frame size outside of the signed 32-bit range not supported");
427	}
428	Offset = StackOffset::getFixed(Fixed: FixedOffset);
429	}
430
431	const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
432	// We must keep the stack pointer aligned through any intermediate
433	// updates.
434	RI.adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset,
435	Flag, RequiredAlign: getStackAlign());
436	}
437
438	static void appendScalableVectorExpression(const TargetRegisterInfo &TRI,
439	SmallVectorImpl<char> &Expr,
440	int FixedOffset, int ScalableOffset,
441	llvm::raw_string_ostream &Comment) {
442	unsigned DwarfVLenB = TRI.getDwarfRegNum(RISCV::VLENB, true);
443	uint8_t Buffer[16];
444	if (FixedOffset) {
445	Expr.push_back(Elt: dwarf::DW_OP_consts);
446	Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: FixedOffset, p: Buffer));
447	Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
448	Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(x: FixedOffset);
449	}
450
451	Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_consts);
452	Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: ScalableOffset, p: Buffer));
453
454	Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_bregx);
455	Expr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfVLenB, p: Buffer));
456	Expr.push_back(Elt: 0);
457
458	Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_mul);
459	Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
460
461	Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(x: ScalableOffset)
462	<< " * vlenb";
463	}
464
465	static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
466	Register Reg,
467	uint64_t FixedOffset,
468	uint64_t ScalableOffset) {
469	assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
470	SmallString<64> Expr;
471	std::string CommentBuffer;
472	llvm::raw_string_ostream Comment(CommentBuffer);
473	// Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
474	unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true);
475	Expr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
476	Expr.push_back(Elt: 0);
477	if (Reg == RISCV::X2)
478	Comment << "sp";
479	else
480	Comment << printReg(Reg, TRI: &TRI);
481
482	appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
483	Comment);
484
485	SmallString<64> DefCfaExpr;
486	uint8_t Buffer[16];
487	DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
488	DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer));
489	DefCfaExpr.append(RHS: Expr.str());
490
491	return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
492	Comment: Comment.str());
493	}
494
495	static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI,
496	Register Reg, uint64_t FixedOffset,
497	uint64_t ScalableOffset) {
498	assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
499	SmallString<64> Expr;
500	std::string CommentBuffer;
501	llvm::raw_string_ostream Comment(CommentBuffer);
502	Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
503
504	// Build up the expression (FixedOffset + ScalableOffset * VLENB).
505	appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
506	Comment);
507
508	SmallString<64> DefCfaExpr;
509	uint8_t Buffer[16];
510	unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true);
511	DefCfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
512	DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfReg, p: Buffer));
513	DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer));
514	DefCfaExpr.append(RHS: Expr.str());
515
516	return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
517	Comment: Comment.str());
518	}
519
520	void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
521	MachineBasicBlock &MBB) const {
522	MachineFrameInfo &MFI = MF.getFrameInfo();
523	auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
524	const RISCVRegisterInfo *RI = STI.getRegisterInfo();
525	const RISCVInstrInfo *TII = STI.getInstrInfo();
526	MachineBasicBlock::iterator MBBI = MBB.begin();
527
528	Register FPReg = getFPReg(STI);
529	Register SPReg = getSPReg(STI);
530	Register BPReg = RISCVABI::getBPReg();
531
532	// Debug location must be unknown since the first debug location is used
533	// to determine the end of the prologue.
534	DebugLoc DL;
535
536	// All calls are tail calls in GHC calling conv, and functions have no
537	// prologue/epilogue.
538	if (MF.getFunction().getCallingConv() == CallingConv::GHC)
539	return;
540
541	// Emit prologue for shadow call stack.
542	emitSCSPrologue(MF, MBB, MI: MBBI, DL);
543
544	auto FirstFrameSetup = MBBI;
545
546	// Since spillCalleeSavedRegisters may have inserted a libcall, skip past
547	// any instructions marked as FrameSetup
548	while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
549	++MBBI;
550
551	// Determine the correct frame layout
552	determineFrameLayout(MF);
553
554	// If libcalls are used to spill and restore callee-saved registers, the frame
555	// has two sections; the opaque section managed by the libcalls, and the
556	// section managed by MachineFrameInfo which can also hold callee saved
557	// registers in fixed stack slots, both of which have negative frame indices.
558	// This gets even more complicated when incoming arguments are passed via the
559	// stack, as these too have negative frame indices. An example is detailed
560	// below:
561	//
562	// | incoming arg | <- FI[-3]
563	// | libcallspill |
564	// | calleespill | <- FI[-2]
565	// | calleespill | <- FI[-1]
566	// | this_frame | <- FI[0]
567	//
568	// For negative frame indices, the offset from the frame pointer will differ
569	// depending on which of these groups the frame index applies to.
570	// The following calculates the correct offset knowing the number of callee
571	// saved registers spilt by the two methods.
572	if (int LibCallRegs = getLibCallID(MF, CSI: MFI.getCalleeSavedInfo()) + 1) {
573	// Calculate the size of the frame managed by the libcall. The stack
574	// alignment of these libcalls should be the same as how we set it in
575	// getABIStackAlignment.
576	unsigned LibCallFrameSize =
577	alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
578	RVFI->setLibCallStackSize(LibCallFrameSize);
579	}
580
581	// FIXME (note copied from Lanai): This appears to be overallocating. Needs
582	// investigation. Get the number of bytes to allocate from the FrameInfo.
583	uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
584	uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
585	uint64_t RVVStackSize = RVFI->getRVVStackSize();
586
587	// Early exit if there is no need to allocate on the stack
588	if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
589	return;
590
591	// If the stack pointer has been marked as reserved, then produce an error if
592	// the frame requires stack allocation
593	if (STI.isRegisterReservedByUser(i: SPReg))
594	MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
595	MF.getFunction(), "Stack pointer required, but has been reserved."});
596
597	uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
598	// Split the SP adjustment to reduce the offsets of callee saved spill.
599	if (FirstSPAdjustAmount) {
600	StackSize = FirstSPAdjustAmount;
601	RealStackSize = FirstSPAdjustAmount;
602	}
603
604	if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
605	FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) {
606	// Use available stack adjustment in push instruction to allocate additional
607	// stack space. Align the stack size down to a multiple of 16. This is
608	// needed for RVE.
609	// FIXME: Can we increase the stack size to a multiple of 16 instead?
610	uint64_t Spimm = std::min(a: alignDown(Value: StackSize, Align: 16), b: (uint64_t)48);
611	FirstFrameSetup->getOperand(i: 1).setImm(Spimm);
612	StackSize -= Spimm;
613	}
614
615	if (StackSize != 0) {
616	// Allocate space on the stack if necessary.
617	RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
618	Offset: StackOffset::getFixed(Fixed: -StackSize), Flag: MachineInstr::FrameSetup,
619	RequiredAlign: getStackAlign());
620	}
621
622	// Emit ".cfi_def_cfa_offset RealStackSize"
623	unsigned CFIIndex = MF.addFrameInst(
624	Inst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: RealStackSize));
625	BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
626	.addCFIIndex(CFIIndex)
627	.setMIFlag(MachineInstr::FrameSetup);
628
629	const auto &CSI = MFI.getCalleeSavedInfo();
630
631	// The frame pointer is callee-saved, and code has been generated for us to
632	// save it to the stack. We need to skip over the storing of callee-saved
633	// registers as the frame pointer must be modified after it has been saved
634	// to the stack, not before.
635	// FIXME: assumes exactly one instruction is used to save each callee-saved
636	// register.
637	std::advance(i&: MBBI, n: getUnmanagedCSI(MF, CSI).size());
638
639	// Iterate over list of callee-saved registers and emit .cfi_offset
640	// directives.
641	for (const auto &Entry : CSI) {
642	int FrameIdx = Entry.getFrameIdx();
643	if (FrameIdx >= 0 &&
644	MFI.getStackID(ObjectIdx: FrameIdx) == TargetStackID::ScalableVector)
645	continue;
646
647	int64_t Offset = MFI.getObjectOffset(ObjectIdx: FrameIdx);
648	Register Reg = Entry.getReg();
649	unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createOffset(
650	L: nullptr, Register: RI->getDwarfRegNum(Reg, true), Offset));
651	BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
652	.addCFIIndex(CFIIndex)
653	.setMIFlag(MachineInstr::FrameSetup);
654	}
655
656	// Generate new FP.
657	if (hasFP(MF)) {
658	if (STI.isRegisterReservedByUser(i: FPReg))
659	MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
660	MF.getFunction(), "Frame pointer required, but has been reserved."});
661	// The frame pointer does need to be reserved from register allocation.
662	assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
663
664	RI->adjustReg(MBB, II: MBBI, DL, DestReg: FPReg, SrcReg: SPReg,
665	Offset: StackOffset::getFixed(Fixed: RealStackSize - RVFI->getVarArgsSaveSize()),
666	Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
667
668	// Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
669	unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa(
670	L: nullptr, Register: RI->getDwarfRegNum(FPReg, true), Offset: RVFI->getVarArgsSaveSize()));
671	BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
672	.addCFIIndex(CFIIndex)
673	.setMIFlag(MachineInstr::FrameSetup);
674	}
675
676	// Emit the second SP adjustment after saving callee saved registers.
677	if (FirstSPAdjustAmount) {
678	uint64_t SecondSPAdjustAmount =
679	getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
680	assert(SecondSPAdjustAmount > 0 &&
681	"SecondSPAdjustAmount should be greater than zero");
682	RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
683	Offset: StackOffset::getFixed(Fixed: -SecondSPAdjustAmount),
684	Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
685
686	// If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0",
687	// don't emit an sp-based .cfi_def_cfa_offset
688	if (!hasFP(MF)) {
689	// Emit ".cfi_def_cfa_offset StackSize"
690	unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfaOffset(
691	L: nullptr, Offset: getStackSizeWithRVVPadding(MF)));
692	BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
693	.addCFIIndex(CFIIndex)
694	.setMIFlag(MachineInstr::FrameSetup);
695	}
696	}
697
698	if (RVVStackSize) {
699	adjustStackForRVV(MF, MBB, MBBI, DL, Amount: -RVVStackSize,
700	Flag: MachineInstr::FrameSetup);
701	if (!hasFP(MF)) {
702	// Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
703	unsigned CFIIndex = MF.addFrameInst(Inst: createDefCFAExpression(
704	*RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
705	BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
706	.addCFIIndex(CFIIndex)
707	.setMIFlag(MachineInstr::FrameSetup);
708	}
709
710	std::advance(i&: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
711	emitCalleeSavedRVVPrologCFI(MBB, MI: MBBI, HasFP: hasFP(MF));
712	}
713
714	if (hasFP(MF)) {
715	// Realign Stack
716	const RISCVRegisterInfo *RI = STI.getRegisterInfo();
717	if (RI->hasStackRealignment(MF)) {
718	Align MaxAlignment = MFI.getMaxAlign();
719
720	const RISCVInstrInfo *TII = STI.getInstrInfo();
721	if (isInt<12>(x: -(int)MaxAlignment.value())) {
722	BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg)
723	.addReg(SPReg)
724	.addImm(-(int)MaxAlignment.value())
725	.setMIFlag(MachineInstr::FrameSetup);
726	} else {
727	unsigned ShiftAmount = Log2(A: MaxAlignment);
728	Register VR =
729	MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);
730	BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR)
731	.addReg(SPReg)
732	.addImm(ShiftAmount)
733	.setMIFlag(MachineInstr::FrameSetup);
734	BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg)
735	.addReg(VR)
736	.addImm(ShiftAmount)
737	.setMIFlag(MachineInstr::FrameSetup);
738	}
739	// FP will be used to restore the frame in the epilogue, so we need
740	// another base register BP to record SP after re-alignment. SP will
741	// track the current stack after allocating variable sized objects.
742	if (hasBP(MF)) {
743	// move BP, SP
744	BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg)
745	.addReg(SPReg)
746	.addImm(0)
747	.setMIFlag(MachineInstr::FrameSetup);
748	}
749	}
750	}
751	}
752
753	void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
754	MachineBasicBlock &MBB) const {
755	const RISCVRegisterInfo *RI = STI.getRegisterInfo();
756	MachineFrameInfo &MFI = MF.getFrameInfo();
757	auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
758	Register FPReg = getFPReg(STI);
759	Register SPReg = getSPReg(STI);
760
761	// All calls are tail calls in GHC calling conv, and functions have no
762	// prologue/epilogue.
763	if (MF.getFunction().getCallingConv() == CallingConv::GHC)
764	return;
765
766	// Get the insert location for the epilogue. If there were no terminators in
767	// the block, get the last instruction.
768	MachineBasicBlock::iterator MBBI = MBB.end();
769	DebugLoc DL;
770	if (!MBB.empty()) {
771	MBBI = MBB.getLastNonDebugInstr();
772	if (MBBI != MBB.end())
773	DL = MBBI->getDebugLoc();
774
775	MBBI = MBB.getFirstTerminator();
776
777	// If callee-saved registers are saved via libcall, place stack adjustment
778	// before this call.
779	while (MBBI != MBB.begin() &&
780	std::prev(x: MBBI)->getFlag(Flag: MachineInstr::FrameDestroy))
781	--MBBI;
782	}
783
784	const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo());
785
786	// Skip to before the restores of scalar callee-saved registers
787	// FIXME: assumes exactly one instruction is used to restore each
788	// callee-saved register.
789	auto LastFrameDestroy = MBBI;
790	if (!CSI.empty())
791	LastFrameDestroy = std::prev(x: MBBI, n: CSI.size());
792
793	uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
794	uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
795	uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
796	uint64_t RVVStackSize = RVFI->getRVVStackSize();
797
798	// Restore the stack pointer using the value of the frame pointer. Only
799	// necessary if the stack pointer was modified, meaning the stack size is
800	// unknown.
801	//
802	// In order to make sure the stack point is right through the EH region,
803	// we also need to restore stack pointer from the frame pointer if we
804	// don't preserve stack space within prologue/epilogue for outgoing variables,
805	// normally it's just checking the variable sized object is present or not
806	// is enough, but we also don't preserve that at prologue/epilogue when
807	// have vector objects in stack.
808	if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
809	!hasReservedCallFrame(MF)) {
810	assert(hasFP(MF) && "frame pointer should not have been eliminated");
811	RI->adjustReg(MBB, II: LastFrameDestroy, DL, DestReg: SPReg, SrcReg: FPReg,
812	Offset: StackOffset::getFixed(Fixed: -FPOffset),
813	Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
814	} else {
815	if (RVVStackSize)
816	adjustStackForRVV(MF, MBB, MBBI: LastFrameDestroy, DL, Amount: RVVStackSize,
817	Flag: MachineInstr::FrameDestroy);
818	}
819
820	uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
821	if (FirstSPAdjustAmount) {
822	uint64_t SecondSPAdjustAmount =
823	getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
824	assert(SecondSPAdjustAmount > 0 &&
825	"SecondSPAdjustAmount should be greater than zero");
826
827	RI->adjustReg(MBB, II: LastFrameDestroy, DL, DestReg: SPReg, SrcReg: SPReg,
828	Offset: StackOffset::getFixed(Fixed: SecondSPAdjustAmount),
829	Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
830	}
831
832	if (FirstSPAdjustAmount)
833	StackSize = FirstSPAdjustAmount;
834
835	if (RVFI->isPushable(MF) && MBBI != MBB.end() &&
836	MBBI->getOpcode() == RISCV::CM_POP) {
837	// Use available stack adjustment in pop instruction to deallocate stack
838	// space. Align the stack size down to a multiple of 16. This is needed for
839	// RVE.
840	// FIXME: Can we increase the stack size to a multiple of 16 instead?
841	uint64_t Spimm = std::min(a: alignDown(Value: StackSize, Align: 16), b: (uint64_t)48);
842	MBBI->getOperand(i: 1).setImm(Spimm);
843	StackSize -= Spimm;
844	}
845
846	// Deallocate stack
847	if (StackSize != 0) {
848	RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: StackSize),
849	Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
850	}
851
852	// Emit epilogue for shadow call stack.
853	emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
854	}
855
856	StackOffset
857	RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
858	Register &FrameReg) const {
859	const MachineFrameInfo &MFI = MF.getFrameInfo();
860	const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
861	const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
862
863	// Callee-saved registers should be referenced relative to the stack
864	// pointer (positive offset), otherwise use the frame pointer (negative
865	// offset).
866	const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo());
867	int MinCSFI = 0;
868	int MaxCSFI = -1;
869	StackOffset Offset;
870	auto StackID = MFI.getStackID(ObjectIdx: FI);
871
872	assert((StackID == TargetStackID::Default ||
873	StackID == TargetStackID::ScalableVector) &&
874	"Unexpected stack ID for the frame object.");
875	if (StackID == TargetStackID::Default) {
876	Offset =
877	StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) - getOffsetOfLocalArea() +
878	MFI.getOffsetAdjustment());
879	} else if (StackID == TargetStackID::ScalableVector) {
880	Offset = StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: FI));
881	}
882
883	uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
884
885	if (CSI.size()) {
886	MinCSFI = CSI[0].getFrameIdx();
887	MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
888	}
889
890	if (FI >= MinCSFI && FI <= MaxCSFI) {
891	FrameReg = RISCV::X2;
892
893	if (FirstSPAdjustAmount)
894	Offset += StackOffset::getFixed(Fixed: FirstSPAdjustAmount);
895	else
896	Offset += StackOffset::getFixed(Fixed: getStackSizeWithRVVPadding(MF));
897	return Offset;
898	}
899
900	if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FI)) {
901	// If the stack was realigned, the frame pointer is set in order to allow
902	// SP to be restored, so we need another base register to record the stack
903	// after realignment.
904	// |--------------------------| -- <-- FP
905	// | callee-allocated save | | <----|
906	// | area for register varargs| | |
907	// |--------------------------| | |
908	// | callee-saved registers | | |
909	// |--------------------------| -- |
910	// | realignment (the size of | | |
911	// | this area is not counted | | |
912	// | in MFI.getStackSize()) | | |
913	// |--------------------------| -- |-- MFI.getStackSize()
914	// | RVV alignment padding | | |
915	// | (not counted in | | |
916	// | MFI.getStackSize() but | | |
917	// | counted in | | |
918	// | RVFI.getRVVStackSize()) | | |
919	// |--------------------------| -- |
920	// | RVV objects | | |
921	// | (not counted in | | |
922	// | MFI.getStackSize()) | | |
923	// |--------------------------| -- |
924	// | padding before RVV | | |
925	// | (not counted in | | |
926	// | MFI.getStackSize() or in | | |
927	// | RVFI.getRVVStackSize()) | | |
928	// |--------------------------| -- |
929	// | scalar local variables | | <----'
930	// |--------------------------| -- <-- BP (if var sized objects present)
931	// | VarSize objects | |
932	// |--------------------------| -- <-- SP
933	if (hasBP(MF)) {
934	FrameReg = RISCVABI::getBPReg();
935	} else {
936	// VarSize objects must be empty in this case!
937	assert(!MFI.hasVarSizedObjects());
938	FrameReg = RISCV::X2;
939	}
940	} else {
941	FrameReg = RI->getFrameRegister(MF);
942	}
943
944	if (FrameReg == getFPReg(STI)) {
945	Offset += StackOffset::getFixed(Fixed: RVFI->getVarArgsSaveSize());
946	// When using FP to access scalable vector objects, we need to minus
947	// the frame size.
948	//
949	// |--------------------------| -- <-- FP
950	// | callee-allocated save | |
951	// | area for register varargs| |
952	// |--------------------------| |
953	// | callee-saved registers | |
954	// |--------------------------| | MFI.getStackSize()
955	// | scalar local variables | |
956	// |--------------------------| -- (Offset of RVV objects is from here.)
957	// | RVV objects |
958	// |--------------------------|
959	// | VarSize objects |
960	// |--------------------------| <-- SP
961	if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
962	assert(!RI->hasStackRealignment(MF) &&
963	"Can't index across variable sized realign");
964	// We don't expect any extra RVV alignment padding, as the stack size
965	// and RVV object sections should be correct aligned in their own
966	// right.
967	assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
968	"Inconsistent stack layout");
969	Offset -= StackOffset::getFixed(Fixed: MFI.getStackSize());
970	}
971	return Offset;
972	}
973
974	// This case handles indexing off both SP and BP.
975	// If indexing off SP, there must not be any var sized objects
976	assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
977
978	// When using SP to access frame objects, we need to add RVV stack size.
979	//
980	// |--------------------------| -- <-- FP
981	// | callee-allocated save | | <----|
982	// | area for register varargs| | |
983	// |--------------------------| | |
984	// | callee-saved registers | | |
985	// |--------------------------| -- |
986	// | RVV alignment padding | | |
987	// | (not counted in | | |
988	// | MFI.getStackSize() but | | |
989	// | counted in | | |
990	// | RVFI.getRVVStackSize()) | | |
991	// |--------------------------| -- |
992	// | RVV objects | | |-- MFI.getStackSize()
993	// | (not counted in | | |
994	// | MFI.getStackSize()) | | |
995	// |--------------------------| -- |
996	// | padding before RVV | | |
997	// | (not counted in | | |
998	// | MFI.getStackSize()) | | |
999	// |--------------------------| -- |
1000	// | scalar local variables | | <----'
1001	// |--------------------------| -- <-- BP (if var sized objects present)
1002	// | VarSize objects | |
1003	// |--------------------------| -- <-- SP
1004	//
1005	// The total amount of padding surrounding RVV objects is described by
1006	// RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1007	// objects to the required alignment.
1008	if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) {
1009	if (MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1010	assert(!RI->hasStackRealignment(MF) &&
1011	"Can't index across variable sized realign");
1012	Offset += StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF),
1013	Scalable: RVFI->getRVVStackSize());
1014	} else {
1015	Offset += StackOffset::getFixed(Fixed: MFI.getStackSize());
1016	}
1017	} else if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1018	// Ensure the base of the RVV stack is correctly aligned: add on the
1019	// alignment padding.
1020	int ScalarLocalVarSize = MFI.getStackSize() -
1021	RVFI->getCalleeSavedStackSize() -
1022	RVFI->getRVPushStackSize() -
1023	RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1024	Offset += StackOffset::get(Fixed: ScalarLocalVarSize, Scalable: RVFI->getRVVStackSize());
1025	}
1026	return Offset;
1027	}
1028
1029	void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
1030	BitVector &SavedRegs,
1031	RegScavenger *RS) const {
1032	TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1033	// Unconditionally spill RA and FP only if the function uses a frame
1034	// pointer.
1035	if (hasFP(MF)) {
1036	SavedRegs.set(RISCV::X1);
1037	SavedRegs.set(RISCV::X8);
1038	}
1039	// Mark BP as used if function has dedicated base pointer.
1040	if (hasBP(MF))
1041	SavedRegs.set(RISCVABI::getBPReg());
1042	}
1043
1044	std::pair<int64_t, Align>
1045	RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1046	MachineFrameInfo &MFI = MF.getFrameInfo();
1047	// Create a buffer of RVV objects to allocate.
1048	SmallVector<int, 8> ObjectsToAllocate;
1049	auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1050	for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1051	unsigned StackID = MFI.getStackID(ObjectIdx: I);
1052	if (StackID != TargetStackID::ScalableVector)
1053	continue;
1054	if (MFI.isDeadObjectIndex(ObjectIdx: I))
1055	continue;
1056
1057	ObjectsToAllocate.push_back(Elt: I);
1058	}
1059	};
1060	// First push RVV Callee Saved object, then push RVV stack object
1061	std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1062	const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1063	if (!RVVCSI.empty())
1064	pushRVVObjects(RVVCSI[0].getFrameIdx(),
1065	RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1066	pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1067
1068	// The minimum alignment is 16 bytes.
1069	Align RVVStackAlign(16);
1070	const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1071
1072	if (!ST.hasVInstructions()) {
1073	assert(ObjectsToAllocate.empty() &&
1074	"Can't allocate scalable-vector objects without V instructions");
1075	return std::make_pair(x: 0, y&: RVVStackAlign);
1076	}
1077
1078	// Allocate all RVV locals and spills
1079	int64_t Offset = 0;
1080	for (int FI : ObjectsToAllocate) {
1081	// ObjectSize in bytes.
1082	int64_t ObjectSize = MFI.getObjectSize(ObjectIdx: FI);
1083	auto ObjectAlign = std::max(a: Align(8), b: MFI.getObjectAlign(ObjectIdx: FI));
1084	// If the data type is the fractional vector type, reserve one vector
1085	// register for it.
1086	if (ObjectSize < 8)
1087	ObjectSize = 8;
1088	Offset = alignTo(Size: Offset + ObjectSize, A: ObjectAlign);
1089	MFI.setObjectOffset(ObjectIdx: FI, SPOffset: -Offset);
1090	// Update the maximum alignment of the RVV stack section
1091	RVVStackAlign = std::max(a: RVVStackAlign, b: ObjectAlign);
1092	}
1093
1094	// Ensure the alignment of the RVV stack. Since we want the most-aligned
1095	// object right at the bottom (i.e., any padding at the top of the frame),
1096	// readjust all RVV objects down by the alignment padding.
1097	uint64_t StackSize = Offset;
1098	if (auto AlignmentPadding = offsetToAlignment(Value: StackSize, Alignment: RVVStackAlign)) {
1099	StackSize += AlignmentPadding;
1100	for (int FI : ObjectsToAllocate)
1101	MFI.setObjectOffset(ObjectIdx: FI, SPOffset: MFI.getObjectOffset(ObjectIdx: FI) - AlignmentPadding);
1102	}
1103
1104	return std::make_pair(x&: StackSize, y&: RVVStackAlign);
1105	}
1106
1107	static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
1108	// For RVV spill, scalable stack offsets computing requires up to two scratch
1109	// registers
1110	static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
1111
1112	// For RVV spill, non-scalable stack offsets computing requires up to one
1113	// scratch register.
1114	static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
1115
1116	// ADDI instruction's destination register can be used for computing
1117	// offsets. So Scalable stack offsets require up to one scratch register.
1118	static constexpr unsigned ScavSlotsADDIScalableObject = 1;
1119
1120	static constexpr unsigned MaxScavSlotsNumKnown =
1121	std::max(l: {ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
1122	ScavSlotsNumRVVSpillNonScalableObject});
1123
1124	unsigned MaxScavSlotsNum = 0;
1125	if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
1126	return false;
1127	for (const MachineBasicBlock &MBB : MF)
1128	for (const MachineInstr &MI : MBB) {
1129	bool IsRVVSpill = RISCV::isRVVSpill(MI);
1130	for (auto &MO : MI.operands()) {
1131	if (!MO.isFI())
1132	continue;
1133	bool IsScalableVectorID = MF.getFrameInfo().getStackID(ObjectIdx: MO.getIndex()) ==
1134	TargetStackID::ScalableVector;
1135	if (IsRVVSpill) {
1136	MaxScavSlotsNum = std::max(
1137	a: MaxScavSlotsNum, b: IsScalableVectorID
1138	? ScavSlotsNumRVVSpillScalableObject
1139	: ScavSlotsNumRVVSpillNonScalableObject);
1140	} else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
1141	MaxScavSlotsNum =
1142	std::max(a: MaxScavSlotsNum, b: ScavSlotsADDIScalableObject);
1143	}
1144	}
1145	if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
1146	return MaxScavSlotsNumKnown;
1147	}
1148	return MaxScavSlotsNum;
1149	}
1150
1151	static bool hasRVVFrameObject(const MachineFunction &MF) {
1152	// Originally, the function will scan all the stack objects to check whether
1153	// if there is any scalable vector object on the stack or not. However, it
1154	// causes errors in the register allocator. In issue 53016, it returns false
1155	// before RA because there is no RVV stack objects. After RA, it returns true
1156	// because there are spilling slots for RVV values during RA. It will not
1157	// reserve BP during register allocation and generate BP access in the PEI
1158	// pass due to the inconsistent behavior of the function.
1159	//
1160	// The function is changed to use hasVInstructions() as the return value. It
1161	// is not precise, but it can make the register allocation correct.
1162	//
1163	// FIXME: Find a better way to make the decision or revisit the solution in
1164	// D103622.
1165	//
1166	// Refer to https://github.com/llvm/llvm-project/issues/53016.
1167	return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
1168	}
1169
1170	static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF,
1171	const RISCVInstrInfo &TII) {
1172	unsigned FnSize = 0;
1173	for (auto &MBB : MF) {
1174	for (auto &MI : MBB) {
1175	// Far branches over 20-bit offset will be relaxed in branch relaxation
1176	// pass. In the worst case, conditional branches will be relaxed into
1177	// the following instruction sequence. Unconditional branches are
1178	// relaxed in the same way, with the exception that there is no first
1179	// branch instruction.
1180	//
1181	// foo
1182	// bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
1183	// sd s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1184	// jump .restore, s11 # 8 bytes
1185	// .rev_cond
1186	// bar
1187	// j .dest_bb # 4 bytes, or 2 bytes in RVC
1188	// .restore:
1189	// ld s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1190	// .dest:
1191	// baz
1192	if (MI.isConditionalBranch())
1193	FnSize += TII.getInstSizeInBytes(MI);
1194	if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
1195	if (MF.getSubtarget<RISCVSubtarget>().hasStdExtCOrZca())
1196	FnSize += 2 + 8 + 2 + 2;
1197	else
1198	FnSize += 4 + 8 + 4 + 4;
1199	continue;
1200	}
1201
1202	FnSize += TII.getInstSizeInBytes(MI);
1203	}
1204	}
1205	return FnSize;
1206	}
1207
1208	void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
1209	MachineFunction &MF, RegScavenger *RS) const {
1210	const RISCVRegisterInfo *RegInfo =
1211	MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
1212	const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
1213	MachineFrameInfo &MFI = MF.getFrameInfo();
1214	const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1215	auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1216
1217	int64_t RVVStackSize;
1218	Align RVVStackAlign;
1219	std::tie(args&: RVVStackSize, args&: RVVStackAlign) = assignRVVStackObjectOffsets(MF);
1220
1221	RVFI->setRVVStackSize(RVVStackSize);
1222	RVFI->setRVVStackAlign(RVVStackAlign);
1223
1224	if (hasRVVFrameObject(MF)) {
1225	// Ensure the entire stack is aligned to at least the RVV requirement: some
1226	// scalable-vector object alignments are not considered by the
1227	// target-independent code.
1228	MFI.ensureMaxAlignment(Alignment: RVVStackAlign);
1229	}
1230
1231	unsigned ScavSlotsNum = 0;
1232
1233	// estimateStackSize has been observed to under-estimate the final stack
1234	// size, so give ourselves wiggle-room by checking for stack size
1235	// representable an 11-bit signed field rather than 12-bits.
1236	if (!isInt<11>(x: MFI.estimateStackSize(MF)))
1237	ScavSlotsNum = 1;
1238
1239	// Far branches over 20-bit offset require a spill slot for scratch register.
1240	bool IsLargeFunction = !isInt<20>(x: estimateFunctionSizeInBytes(MF, TII: *TII));
1241	if (IsLargeFunction)
1242	ScavSlotsNum = std::max(a: ScavSlotsNum, b: 1u);
1243
1244	// RVV loads & stores have no capacity to hold the immediate address offsets
1245	// so we must always reserve an emergency spill slot if the MachineFunction
1246	// contains any RVV spills.
1247	ScavSlotsNum = std::max(a: ScavSlotsNum, b: getScavSlotsNumForRVV(MF));
1248
1249	for (unsigned I = 0; I < ScavSlotsNum; I++) {
1250	int FI = MFI.CreateStackObject(Size: RegInfo->getSpillSize(*RC),
1251	Alignment: RegInfo->getSpillAlign(*RC), isSpillSlot: false);
1252	RS->addScavengingFrameIndex(FI);
1253
1254	if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
1255	RVFI->setBranchRelaxationScratchFrameIndex(FI);
1256	}
1257
1258	unsigned Size = RVFI->getReservedSpillsSize();
1259	for (const auto &Info : MFI.getCalleeSavedInfo()) {
1260	int FrameIdx = Info.getFrameIdx();
1261	if (FrameIdx < 0 || MFI.getStackID(ObjectIdx: FrameIdx) != TargetStackID::Default)
1262	continue;
1263
1264	Size += MFI.getObjectSize(ObjectIdx: FrameIdx);
1265	}
1266	RVFI->setCalleeSavedStackSize(Size);
1267	}
1268
1269	// Not preserve stack space within prologue for outgoing variables when the
1270	// function contains variable size objects or there are vector objects accessed
1271	// by the frame pointer.
1272	// Let eliminateCallFramePseudoInstr preserve stack space for it.
1273	bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
1274	return !MF.getFrameInfo().hasVarSizedObjects() &&
1275	!(hasFP(MF) && hasRVVFrameObject(MF));
1276	}
1277
1278	// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
1279	MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
1280	MachineFunction &MF, MachineBasicBlock &MBB,
1281	MachineBasicBlock::iterator MI) const {
1282	Register SPReg = RISCV::X2;
1283	DebugLoc DL = MI->getDebugLoc();
1284
1285	if (!hasReservedCallFrame(MF)) {
1286	// If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
1287	// ADJCALLSTACKUP must be converted to instructions manipulating the stack
1288	// pointer. This is necessary when there is a variable length stack
1289	// allocation (e.g. alloca), which means it's not possible to allocate
1290	// space for outgoing arguments from within the function prologue.
1291	int64_t Amount = MI->getOperand(i: 0).getImm();
1292
1293	if (Amount != 0) {
1294	// Ensure the stack remains aligned after adjustment.
1295	Amount = alignSPAdjust(SPAdj: Amount);
1296
1297	if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
1298	Amount = -Amount;
1299
1300	const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
1301	RI.adjustReg(MBB, II: MI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: Amount),
1302	Flag: MachineInstr::NoFlags, RequiredAlign: getStackAlign());
1303	}
1304	}
1305
1306	return MBB.erase(I: MI);
1307	}
1308
1309	// We would like to split the SP adjustment to reduce prologue/epilogue
1310	// as following instructions. In this way, the offset of the callee saved
1311	// register could fit in a single store. Supposed that the first sp adjust
1312	// amount is 2032.
1313	// add sp,sp,-2032
1314	// sw ra,2028(sp)
1315	// sw s0,2024(sp)
1316	// sw s1,2020(sp)
1317	// sw s3,2012(sp)
1318	// sw s4,2008(sp)
1319	// add sp,sp,-64
1320	uint64_t
1321	RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
1322	const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1323	const MachineFrameInfo &MFI = MF.getFrameInfo();
1324	const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
1325	uint64_t StackSize = getStackSizeWithRVVPadding(MF);
1326
1327	// Disable SplitSPAdjust if save-restore libcall is used. The callee-saved
1328	// registers will be pushed by the save-restore libcalls, so we don't have to
1329	// split the SP adjustment in this case.
1330	if (RVFI->getReservedSpillsSize())
1331	return 0;
1332
1333	// Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
1334	// 12-bit and there exists a callee-saved register needing to be pushed.
1335	if (!isInt<12>(x: StackSize) && (CSI.size() > 0)) {
1336	// FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
1337	// 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
1338	// instructions. Offsets smaller than 2048 can fit in a single load/store
1339	// instruction, and we have to stick with the stack alignment. 2048 has
1340	// 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
1341	// RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
1342	const uint64_t StackAlign = getStackAlign().value();
1343
1344	// Amount of (2048 - StackAlign) will prevent callee saved and restored
1345	// instructions be compressed, so try to adjust the amount to the largest
1346	// offset that stack compression instructions accept when target supports
1347	// compression instructions.
1348	if (STI.hasStdExtCOrZca()) {
1349	// The compression extensions may support the following instructions:
1350	// riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
1351	// c.swsp rs2, offset[7:2] => 2^(6 + 2)
1352	// c.flwsp rd, offset[7:2] => 2^(6 + 2)
1353	// c.fswsp rs2, offset[7:2] => 2^(6 + 2)
1354	// riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
1355	// c.sdsp rs2, offset[8:3] => 2^(6 + 3)
1356	// c.fldsp rd, offset[8:3] => 2^(6 + 3)
1357	// c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
1358	const uint64_t RVCompressLen = STI.getXLen() * 8;
1359	// Compared with amount (2048 - StackAlign), StackSize needs to
1360	// satisfy the following conditions to avoid using more instructions
1361	// to adjust the sp after adjusting the amount, such as
1362	// StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
1363	// case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
1364	// case2: Amount is RVCompressLen: use addi + addi to adjust sp.
1365	auto CanCompress = [&](uint64_t CompressLen) -> bool {
1366	if (StackSize <= 2047 + CompressLen ||
1367	(StackSize > 2048 * 2 - StackAlign &&
1368	StackSize <= 2047 * 2 + CompressLen) ||
1369	StackSize > 2048 * 3 - StackAlign)
1370	return true;
1371
1372	return false;
1373	};
1374	// In the epilogue, addi sp, sp, 496 is used to recover the sp and it
1375	// can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
1376	// addi sp, sp, 512 can not be compressed. So try to use 496 first.
1377	const uint64_t ADDI16SPCompressLen = 496;
1378	if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
1379	return ADDI16SPCompressLen;
1380	if (CanCompress(RVCompressLen))
1381	return RVCompressLen;
1382	}
1383	return 2048 - StackAlign;
1384	}
1385	return 0;
1386	}
1387
1388	// Offsets which need to be scale by XLen representing locations of CSRs which
1389	// are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
1390	static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = {
1391	{/*ra*/ RISCV::X1, -1}, {/*s0*/ RISCV::X8, -2},
1392	{/*s1*/ RISCV::X9, -3}, {/*s2*/ RISCV::X18, -4},
1393	{/*s3*/ RISCV::X19, -5}, {/*s4*/ RISCV::X20, -6},
1394	{/*s5*/ RISCV::X21, -7}, {/*s6*/ RISCV::X22, -8},
1395	{/*s7*/ RISCV::X23, -9}, {/*s8*/ RISCV::X24, -10},
1396	{/*s9*/ RISCV::X25, -11}, {/*s10*/ RISCV::X26, -12},
1397	{/*s11*/ RISCV::X27, -13}};
1398
1399	bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
1400	MachineFunction &MF, const TargetRegisterInfo *TRI,
1401	std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
1402	unsigned &MaxCSFrameIndex) const {
1403	// Early exit if no callee saved registers are modified!
1404	if (CSI.empty())
1405	return true;
1406
1407	auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1408
1409	if (RVFI->isPushable(MF)) {
1410	// Determine how many GPRs we need to push and save it to RVFI.
1411	Register MaxReg = getMaxPushPopReg(MF, CSI);
1412	if (MaxReg != RISCV::NoRegister) {
1413	auto [RegEnc, PushedRegNum] = getPushPopEncodingAndNum(MaxReg);
1414	RVFI->setRVPushRegs(PushedRegNum);
1415	RVFI->setRVPushStackSize(alignTo(Value: (STI.getXLen() / 8) * PushedRegNum, Align: 16));
1416
1417	// Use encoded number to represent registers to spill.
1418	RVFI->setRVPushRlist(RegEnc);
1419	}
1420	}
1421
1422	MachineFrameInfo &MFI = MF.getFrameInfo();
1423	const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
1424
1425	for (auto &CS : CSI) {
1426	unsigned Reg = CS.getReg();
1427	const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
1428	unsigned Size = RegInfo->getSpillSize(RC: *RC);
1429
1430	// This might need a fixed stack slot.
1431	if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
1432	const auto *FII = llvm::find_if(
1433	FixedCSRFIMap, [&](auto P) { return P.first == CS.getReg(); });
1434	if (FII != std::end(FixedCSRFIMap)) {
1435	int64_t Offset;
1436	if (RVFI->isPushable(MF))
1437	Offset = -((FII->second + RVFI->getRVPushRegs() + 1) * (int64_t)Size);
1438	else
1439	Offset = FII->second * (int64_t)Size;
1440
1441	int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
1442	assert(FrameIdx < 0);
1443	CS.setFrameIdx(FrameIdx);
1444	continue;
1445	}
1446	}
1447
1448	// Not a fixed slot.
1449	Align Alignment = RegInfo->getSpillAlign(RC: *RC);
1450	// We may not be able to satisfy the desired alignment specification of
1451	// the TargetRegisterClass if the stack alignment is smaller. Use the
1452	// min.
1453	Alignment = std::min(a: Alignment, b: getStackAlign());
1454	int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
1455	if ((unsigned)FrameIdx < MinCSFrameIndex)
1456	MinCSFrameIndex = FrameIdx;
1457	if ((unsigned)FrameIdx > MaxCSFrameIndex)
1458	MaxCSFrameIndex = FrameIdx;
1459	CS.setFrameIdx(FrameIdx);
1460	}
1461
1462	// Allocate a fixed object that covers the full push or libcall size.
1463	if (RVFI->isPushable(MF)) {
1464	if (int64_t PushSize = RVFI->getRVPushStackSize())
1465	MFI.CreateFixedSpillStackObject(Size: PushSize, SPOffset: -PushSize);
1466	} else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
1467	int64_t LibCallFrameSize =
1468	alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
1469	MFI.CreateFixedSpillStackObject(Size: LibCallFrameSize, SPOffset: -LibCallFrameSize);
1470	}
1471
1472	return true;
1473	}
1474
1475	bool RISCVFrameLowering::spillCalleeSavedRegisters(
1476	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1477	ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1478	if (CSI.empty())
1479	return true;
1480
1481	MachineFunction *MF = MBB.getParent();
1482	const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1483	DebugLoc DL;
1484	if (MI != MBB.end() && !MI->isDebugInstr())
1485	DL = MI->getDebugLoc();
1486
1487	// Emit CM.PUSH with base SPimm & evaluate Push stack
1488	RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1489	if (RVFI->isPushable(MF: *MF)) {
1490	unsigned PushedRegNum = RVFI->getRVPushRegs();
1491	if (PushedRegNum > 0) {
1492	// Use encoded number to represent registers to spill.
1493	int RegEnc = RVFI->getRVPushRlist();
1494	MachineInstrBuilder PushBuilder =
1495	BuildMI(MBB, MI, DL, TII.get(RISCV::CM_PUSH))
1496	.setMIFlag(MachineInstr::FrameSetup);
1497	PushBuilder.addImm(Val: (int64_t)RegEnc);
1498	PushBuilder.addImm(Val: 0);
1499
1500	for (unsigned i = 0; i < PushedRegNum; i++)
1501	PushBuilder.addUse(RegNo: AllPopRegs[i], Flags: RegState::Implicit);
1502	}
1503	} else if (const char *SpillLibCall = getSpillLibCallName(MF: *MF, CSI)) {
1504	// Add spill libcall via non-callee-saved register t0.
1505	BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5)
1506	.addExternalSymbol(SpillLibCall, RISCVII::MO_CALL)
1507	.setMIFlag(MachineInstr::FrameSetup);
1508
1509	// Add registers spilled in libcall as liveins.
1510	for (auto &CS : CSI)
1511	MBB.addLiveIn(PhysReg: CS.getReg());
1512	}
1513
1514	// Manually spill values not spilled by libcall & Push/Pop.
1515	const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI);
1516	const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
1517
1518	auto storeRegToStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
1519	for (auto &CS : CSInfo) {
1520	// Insert the spill to the stack frame.
1521	Register Reg = CS.getReg();
1522	const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1523	TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: !MBB.isLiveIn(Reg),
1524	FrameIndex: CS.getFrameIdx(), RC, TRI, VReg: Register());
1525	}
1526	};
1527	storeRegToStackSlot(UnmanagedCSI);
1528	storeRegToStackSlot(RVVCSI);
1529
1530	return true;
1531	}
1532
1533	void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
1534	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, bool HasFP) const {
1535	MachineFunction *MF = MBB.getParent();
1536	const MachineFrameInfo &MFI = MF->getFrameInfo();
1537	RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1538	const TargetInstrInfo &TII = *STI.getInstrInfo();
1539	DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
1540
1541	const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
1542	if (RVVCSI.empty())
1543	return;
1544
1545	uint64_t FixedSize = getStackSizeWithRVVPadding(MF: *MF);
1546	if (!HasFP) {
1547	uint64_t ScalarLocalVarSize =
1548	MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
1549	RVFI->getRVPushStackSize() - RVFI->getVarArgsSaveSize() +
1550	RVFI->getRVVPadding();
1551	FixedSize -= ScalarLocalVarSize;
1552	}
1553
1554	for (auto &CS : RVVCSI) {
1555	// Insert the spill to the stack frame.
1556	int FI = CS.getFrameIdx();
1557	if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1558	unsigned CFIIndex = MF->addFrameInst(
1559	Inst: createDefCFAOffset(*STI.getRegisterInfo(), CS.getReg(), -FixedSize,
1560	MFI.getObjectOffset(ObjectIdx: FI) / 8));
1561	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
1562	.addCFIIndex(CFIIndex)
1563	.setMIFlag(MachineInstr::FrameSetup);
1564	}
1565	}
1566	}
1567
1568	bool RISCVFrameLowering::restoreCalleeSavedRegisters(
1569	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1570	MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1571	if (CSI.empty())
1572	return true;
1573
1574	MachineFunction *MF = MBB.getParent();
1575	const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1576	DebugLoc DL;
1577	if (MI != MBB.end() && !MI->isDebugInstr())
1578	DL = MI->getDebugLoc();
1579
1580	// Manually restore values not restored by libcall & Push/Pop.
1581	// Reverse the restore order in epilog. In addition, the return
1582	// address will be restored first in the epilogue. It increases
1583	// the opportunity to avoid the load-to-use data hazard between
1584	// loading RA and return by RA. loadRegFromStackSlot can insert
1585	// multiple instructions.
1586	const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI);
1587	const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
1588
1589	auto loadRegFromStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
1590	for (auto &CS : CSInfo) {
1591	Register Reg = CS.getReg();
1592	const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1593	TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: CS.getFrameIdx(), RC, TRI,
1594	VReg: Register());
1595	assert(MI != MBB.begin() &&
1596	"loadRegFromStackSlot didn't insert any code!");
1597	}
1598	};
1599	loadRegFromStackSlot(RVVCSI);
1600	loadRegFromStackSlot(UnmanagedCSI);
1601
1602	RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1603	if (RVFI->isPushable(MF: *MF)) {
1604	int RegEnc = RVFI->getRVPushRlist();
1605	if (RegEnc != llvm::RISCVZC::RLISTENCODE::INVALID_RLIST) {
1606	MachineInstrBuilder PopBuilder =
1607	BuildMI(MBB, MI, DL, TII.get(RISCV::CM_POP))
1608	.setMIFlag(MachineInstr::FrameDestroy);
1609	// Use encoded number to represent registers to restore.
1610	PopBuilder.addImm(Val: RegEnc);
1611	PopBuilder.addImm(Val: 0);
1612
1613	for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
1614	PopBuilder.addDef(RegNo: AllPopRegs[i], Flags: RegState::ImplicitDefine);
1615	}
1616	} else {
1617	const char *RestoreLibCall = getRestoreLibCallName(MF: *MF, CSI);
1618	if (RestoreLibCall) {
1619	// Add restore libcall via tail call.
1620	MachineBasicBlock::iterator NewMI =
1621	BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL))
1622	.addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL)
1623	.setMIFlag(MachineInstr::FrameDestroy);
1624
1625	// Remove trailing returns, since the terminator is now a tail call to the
1626	// restore function.
1627	if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
1628	NewMI->copyImplicitOps(MF&: *MF, MI: *MI);
1629	MI->eraseFromParent();
1630	}
1631	}
1632	}
1633	return true;
1634	}
1635
1636	bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
1637	// Keep the conventional code flow when not optimizing.
1638	if (MF.getFunction().hasOptNone())
1639	return false;
1640
1641	return true;
1642	}
1643
1644	bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
1645	MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1646	const MachineFunction *MF = MBB.getParent();
1647	const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1648
1649	if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
1650	return true;
1651
1652	// Inserting a call to a __riscv_save libcall requires the use of the register
1653	// t0 (X5) to hold the return address. Therefore if this register is already
1654	// used we can't insert the call.
1655
1656	RegScavenger RS;
1657	RS.enterBasicBlock(MBB&: *TmpMBB);
1658	return !RS.isRegUsed(RISCV::X5);
1659	}
1660
1661	bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
1662	const MachineFunction *MF = MBB.getParent();
1663	MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1664	const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1665
1666	if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
1667	return true;
1668
1669	// Using the __riscv_restore libcalls to restore CSRs requires a tail call.
1670	// This means if we still need to continue executing code within this function
1671	// the restore cannot take place in this basic block.
1672
1673	if (MBB.succ_size() > 1)
1674	return false;
1675
1676	MachineBasicBlock *SuccMBB =
1677	MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
1678
1679	// Doing a tail call should be safe if there are no successors, because either
1680	// we have a returning block or the end of the block is unreachable, so the
1681	// restore will be eliminated regardless.
1682	if (!SuccMBB)
1683	return true;
1684
1685	// The successor can only contain a return, since we would effectively be
1686	// replacing the successor with our own tail return at the end of our block.
1687	return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
1688	}
1689
1690	bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
1691	switch (ID) {
1692	case TargetStackID::Default:
1693	case TargetStackID::ScalableVector:
1694	return true;
1695	case TargetStackID::NoAlloc:
1696	case TargetStackID::SGPRSpill:
1697	case TargetStackID::WasmLocal:
1698	return false;
1699	}
1700	llvm_unreachable("Invalid TargetStackID::Value");
1701	}
1702
1703	TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
1704	return TargetStackID::ScalableVector;
1705	}
1706