1//===- bolt/Passes/RetpolineInsertion.cpp ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements RetpolineInsertion class, which replaces indirect
10// branches (calls and jumps) with calls to retpolines to protect against branch
11// target injection attacks.
12// A unique retpoline is created for each register holding the address of the
13// callee, if the callee address is in memory %r11 is used if available to
14// hold the address of the callee before calling the retpoline, otherwise an
15// address pattern specific retpoline is called where the callee address is
16// loaded inside the retpoline.
17// The user can determine when to assume %r11 available using r11-availability
18// option, by default %r11 is assumed not available.
19// Adding lfence instruction to the body of the speculate code is enabled by
20// default and can be controlled by the user using retpoline-lfence option.
21//
22//===----------------------------------------------------------------------===//
23
24#include "bolt/Passes/RetpolineInsertion.h"
25#include "llvm/MC/MCInstPrinter.h"
26#include "llvm/Support/raw_ostream.h"
27
28#define DEBUG_TYPE "bolt-retpoline"
29
30using namespace llvm;
31using namespace bolt;
32namespace opts {
33
34extern cl::OptionCategory BoltCategory;
35
36static llvm::cl::opt<bool>
37 InsertRetpolines("insert-retpolines",
38 cl::desc("run retpoline insertion pass"),
39 cl::cat(BoltCategory));
40
41static llvm::cl::opt<bool> RetpolineLfence(
42 "retpoline-lfence",
43 cl::desc("determine if lfence instruction should exist in the retpoline"),
44 cl::init(Val: true), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltCategory));
45
46static cl::opt<RetpolineInsertion::AvailabilityOptions> R11Availability(
47 "r11-availability",
48 cl::desc("determine the availability of r11 before indirect branches"),
49 cl::init(Val: RetpolineInsertion::AvailabilityOptions::NEVER),
50 cl::values(clEnumValN(RetpolineInsertion::AvailabilityOptions::NEVER,
51 "never", "r11 not available"),
52 clEnumValN(RetpolineInsertion::AvailabilityOptions::ALWAYS,
53 "always", "r11 available before calls and jumps"),
54 clEnumValN(RetpolineInsertion::AvailabilityOptions::ABI, "abi",
55 "r11 available before calls but not before jumps")),
56 cl::ZeroOrMore, cl::cat(BoltCategory));
57
58} // namespace opts
59
60namespace llvm {
61namespace bolt {
62
63// Retpoline function structure:
64// BB0: call BB2
65// BB1: pause
66// lfence
67// jmp BB1
68// BB2: mov %reg, (%rsp)
69// ret
70// or
71// BB2: push %r11
72// mov Address, %r11
73// mov %r11, 8(%rsp)
74// pop %r11
75// ret
76BinaryFunction *createNewRetpoline(BinaryContext &BC,
77 const std::string &RetpolineTag,
78 const IndirectBranchInfo &BrInfo,
79 bool R11Available) {
80 auto &MIB = *BC.MIB;
81 MCContext &Ctx = *BC.Ctx;
82 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Creating a new retpoline function["
83 << RetpolineTag << "]\n");
84
85 BinaryFunction *NewRetpoline =
86 BC.createInjectedBinaryFunction(Name: RetpolineTag, IsSimple: true);
87 std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks(3);
88 for (int I = 0; I < 3; I++) {
89 MCSymbol *Symbol =
90 Ctx.createNamedTempSymbol(Name: Twine(RetpolineTag + "_BB" + to_string(Value: I)));
91 NewBlocks[I] = NewRetpoline->createBasicBlock(Label: Symbol);
92 NewBlocks[I].get()->setCFIState(0);
93 }
94
95 BinaryBasicBlock &BB0 = *NewBlocks[0].get();
96 BinaryBasicBlock &BB1 = *NewBlocks[1].get();
97 BinaryBasicBlock &BB2 = *NewBlocks[2].get();
98
99 BB0.addSuccessor(Succ: &BB2, Count: 0, MispredictedCount: 0);
100 BB1.addSuccessor(Succ: &BB1, Count: 0, MispredictedCount: 0);
101
102 // Build BB0
103 MCInst DirectCall;
104 MIB.createDirectCall(Inst&: DirectCall, Target: BB2.getLabel(), Ctx: &Ctx, /*IsTailCall*/ false);
105 BB0.addInstruction(Inst: DirectCall);
106
107 // Build BB1
108 MCInst Pause;
109 MIB.createPause(Inst&: Pause);
110 BB1.addInstruction(Inst: Pause);
111
112 if (opts::RetpolineLfence) {
113 MCInst Lfence;
114 MIB.createLfence(Inst&: Lfence);
115 BB1.addInstruction(Inst: Lfence);
116 }
117
118 InstructionListType Seq;
119 MIB.createShortJmp(Seq, Target: BB1.getLabel(), Ctx: &Ctx);
120 BB1.addInstructions(Begin: Seq.begin(), End: Seq.end());
121
122 // Build BB2
123 if (BrInfo.isMem()) {
124 if (R11Available) {
125 MCInst StoreToStack;
126 MIB.createSaveToStack(Inst&: StoreToStack, StackReg: MIB.getStackPointer(), Offset: 0,
127 SrcReg: MIB.getX86R11(), Size: 8);
128 BB2.addInstruction(Inst: StoreToStack);
129 } else {
130 MCInst PushR11;
131 MIB.createPushRegister(Inst&: PushR11, Reg: MIB.getX86R11(), Size: 8);
132 BB2.addInstruction(Inst: PushR11);
133
134 MCInst LoadCalleeAddrs;
135 const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
136 MIB.createLoad(Inst&: LoadCalleeAddrs, BaseReg: MemRef.BaseRegNum, Scale: MemRef.ScaleImm,
137 IndexReg: MemRef.IndexRegNum, Offset: MemRef.DispImm, OffsetExpr: MemRef.DispExpr,
138 AddrSegmentReg: MemRef.SegRegNum, DstReg: MIB.getX86R11(), Size: 8);
139
140 BB2.addInstruction(Inst: LoadCalleeAddrs);
141
142 MCInst StoreToStack;
143 MIB.createSaveToStack(Inst&: StoreToStack, StackReg: MIB.getStackPointer(), Offset: 8,
144 SrcReg: MIB.getX86R11(), Size: 8);
145 BB2.addInstruction(Inst: StoreToStack);
146
147 MCInst PopR11;
148 MIB.createPopRegister(Inst&: PopR11, Reg: MIB.getX86R11(), Size: 8);
149 BB2.addInstruction(Inst: PopR11);
150 }
151 } else if (BrInfo.isReg()) {
152 MCInst StoreToStack;
153 MIB.createSaveToStack(Inst&: StoreToStack, StackReg: MIB.getStackPointer(), Offset: 0,
154 SrcReg: BrInfo.BranchReg, Size: 8);
155 BB2.addInstruction(Inst: StoreToStack);
156 } else {
157 llvm_unreachable("not expected");
158 }
159
160 // return
161 MCInst Return;
162 MIB.createReturn(Inst&: Return);
163 BB2.addInstruction(Inst: Return);
164 NewRetpoline->insertBasicBlocks(Start: nullptr, NewBBs: std::move(NewBlocks),
165 /* UpdateLayout */ true,
166 /* UpdateCFIState */ false);
167
168 NewRetpoline->updateState(State: BinaryFunction::State::CFG_Finalized);
169 return NewRetpoline;
170}
171
172std::string createRetpolineFunctionTag(BinaryContext &BC,
173 const IndirectBranchInfo &BrInfo,
174 bool R11Available) {
175 std::string Tag;
176 llvm::raw_string_ostream TagOS(Tag);
177 TagOS << "__retpoline_";
178
179 if (BrInfo.isReg()) {
180 BC.InstPrinter->printRegName(OS&: TagOS, Reg: BrInfo.BranchReg);
181 TagOS << "_";
182 return Tag;
183 }
184
185 // Memory Branch
186 if (R11Available)
187 return "__retpoline_r11";
188
189 const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
190
191 TagOS << "mem_";
192
193 if (MemRef.BaseRegNum != BC.MIB->getNoRegister())
194 BC.InstPrinter->printRegName(OS&: TagOS, Reg: MemRef.BaseRegNum);
195
196 TagOS << "+";
197 if (MemRef.DispExpr)
198 MemRef.DispExpr->print(OS&: TagOS, MAI: BC.AsmInfo.get());
199 else
200 TagOS << MemRef.DispImm;
201
202 if (MemRef.IndexRegNum != BC.MIB->getNoRegister()) {
203 TagOS << "+" << MemRef.ScaleImm << "*";
204 BC.InstPrinter->printRegName(OS&: TagOS, Reg: MemRef.IndexRegNum);
205 }
206
207 if (MemRef.SegRegNum != BC.MIB->getNoRegister()) {
208 TagOS << "_seg_";
209 BC.InstPrinter->printRegName(OS&: TagOS, Reg: MemRef.SegRegNum);
210 }
211
212 return Tag;
213}
214
215BinaryFunction *RetpolineInsertion::getOrCreateRetpoline(
216 BinaryContext &BC, const IndirectBranchInfo &BrInfo, bool R11Available) {
217 const std::string RetpolineTag =
218 createRetpolineFunctionTag(BC, BrInfo, R11Available);
219
220 if (CreatedRetpolines.count(x: RetpolineTag))
221 return CreatedRetpolines[RetpolineTag];
222
223 return CreatedRetpolines[RetpolineTag] =
224 createNewRetpoline(BC, RetpolineTag, BrInfo, R11Available);
225}
226
227void createBranchReplacement(BinaryContext &BC,
228 const IndirectBranchInfo &BrInfo,
229 bool R11Available,
230 InstructionListType &Replacement,
231 const MCSymbol *RetpolineSymbol) {
232 auto &MIB = *BC.MIB;
233 // Load the branch address in r11 if available
234 if (BrInfo.isMem() && R11Available) {
235 const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
236 MCInst LoadCalleeAddrs;
237 MIB.createLoad(Inst&: LoadCalleeAddrs, BaseReg: MemRef.BaseRegNum, Scale: MemRef.ScaleImm,
238 IndexReg: MemRef.IndexRegNum, Offset: MemRef.DispImm, OffsetExpr: MemRef.DispExpr,
239 AddrSegmentReg: MemRef.SegRegNum, DstReg: MIB.getX86R11(), Size: 8);
240 Replacement.push_back(x: LoadCalleeAddrs);
241 }
242
243 // Call the retpoline
244 MCInst RetpolineCall;
245 MIB.createDirectCall(Inst&: RetpolineCall, Target: RetpolineSymbol, Ctx: BC.Ctx.get(),
246 IsTailCall: BrInfo.isJump() || BrInfo.isTailCall());
247
248 Replacement.push_back(x: RetpolineCall);
249}
250
251IndirectBranchInfo::IndirectBranchInfo(MCInst &Inst, MCPlusBuilder &MIB) {
252 IsCall = MIB.isCall(Inst);
253 IsTailCall = MIB.isTailCall(Inst);
254
255 if (MIB.isBranchOnMem(Inst)) {
256 IsMem = true;
257 std::optional<MCPlusBuilder::X86MemOperand> MO =
258 MIB.evaluateX86MemoryOperand(Inst);
259 if (!MO)
260 llvm_unreachable("not expected");
261 Memory = MO.value();
262 } else if (MIB.isBranchOnReg(Inst)) {
263 assert(MCPlus::getNumPrimeOperands(Inst) == 1 && "expect 1 operand");
264 BranchReg = Inst.getOperand(i: 0).getReg();
265 } else {
266 llvm_unreachable("unexpected instruction");
267 }
268}
269
270Error RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
271 if (!opts::InsertRetpolines)
272 return Error::success();
273
274 assert(BC.isX86() &&
275 "retpoline insertion not supported for target architecture");
276
277 assert(BC.HasRelocations && "retpoline mode not supported in non-reloc");
278
279 auto &MIB = *BC.MIB;
280 uint32_t RetpolinedBranches = 0;
281 for (auto &It : BC.getBinaryFunctions()) {
282 BinaryFunction &Function = It.second;
283 for (BinaryBasicBlock &BB : Function) {
284 for (auto It = BB.begin(); It != BB.end(); ++It) {
285 MCInst &Inst = *It;
286
287 if (!MIB.isIndirectCall(Inst) && !MIB.isIndirectBranch(Inst))
288 continue;
289
290 IndirectBranchInfo BrInfo(Inst, MIB);
291 bool R11Available = false;
292 BinaryFunction *TargetRetpoline;
293 InstructionListType Replacement;
294
295 // Determine if r11 is available before this instruction
296 if (BrInfo.isMem()) {
297 if (MIB.hasAnnotation(Inst, Name: "PLTCall"))
298 R11Available = true;
299 else if (opts::R11Availability == AvailabilityOptions::ALWAYS)
300 R11Available = true;
301 else if (opts::R11Availability == AvailabilityOptions::ABI)
302 R11Available = BrInfo.isCall();
303 }
304
305 // If the instruction addressing pattern uses rsp and the retpoline
306 // loads the callee address then displacement needs to be updated
307 if (BrInfo.isMem() && !R11Available) {
308 IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
309 int Addend = (BrInfo.isJump() || BrInfo.isTailCall()) ? 8 : 16;
310 if (MemRef.BaseRegNum == MIB.getStackPointer())
311 MemRef.DispImm += Addend;
312 if (MemRef.IndexRegNum == MIB.getStackPointer())
313 MemRef.DispImm += Addend * MemRef.ScaleImm;
314 }
315
316 TargetRetpoline = getOrCreateRetpoline(BC, BrInfo, R11Available);
317
318 createBranchReplacement(BC, BrInfo, R11Available, Replacement,
319 RetpolineSymbol: TargetRetpoline->getSymbol());
320
321 It = BB.replaceInstruction(II: It, Begin: Replacement.begin(), End: Replacement.end());
322 RetpolinedBranches++;
323 }
324 }
325 }
326 BC.outs() << "BOLT-INFO: The number of created retpoline functions is : "
327 << CreatedRetpolines.size()
328 << "\nBOLT-INFO: The number of retpolined branches is : "
329 << RetpolinedBranches << "\n";
330 return Error::success();
331}
332
333} // namespace bolt
334} // namespace llvm
335

source code of bolt/lib/Passes/RetpolineInsertion.cpp