1//===- bolt/Passes/RetpolineInsertion.cpp ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements RetpolineInsertion class, which replaces indirect
10// branches (calls and jumps) with calls to retpolines to protect against branch
11// target injection attacks.
12// A unique retpoline is created for each register holding the address of the
13// callee, if the callee address is in memory %r11 is used if available to
14// hold the address of the callee before calling the retpoline, otherwise an
15// address pattern specific retpoline is called where the callee address is
16// loaded inside the retpoline.
17// The user can determine when to assume %r11 available using r11-availability
18// option, by default %r11 is assumed not available.
19// Adding lfence instruction to the body of the speculate code is enabled by
20// default and can be controlled by the user using retpoline-lfence option.
21//
22//===----------------------------------------------------------------------===//
23
24#include "bolt/Passes/RetpolineInsertion.h"
25#include "llvm/MC/MCInstPrinter.h"
26#include "llvm/Support/raw_ostream.h"
27
28#define DEBUG_TYPE "bolt-retpoline"
29
30using namespace llvm;
31using namespace bolt;
32namespace opts {
33
34extern cl::OptionCategory BoltCategory;
35
36llvm::cl::opt<bool> InsertRetpolines("insert-retpolines",
37 cl::desc("run retpoline insertion pass"),
38 cl::cat(BoltCategory));
39
40llvm::cl::opt<bool>
41RetpolineLfence("retpoline-lfence",
42 cl::desc("determine if lfence instruction should exist in the retpoline"),
43 cl::init(Val: true),
44 cl::ZeroOrMore,
45 cl::Hidden,
46 cl::cat(BoltCategory));
47
48cl::opt<RetpolineInsertion::AvailabilityOptions> R11Availability(
49 "r11-availability",
50 cl::desc("determine the availability of r11 before indirect branches"),
51 cl::init(Val: RetpolineInsertion::AvailabilityOptions::NEVER),
52 cl::values(clEnumValN(RetpolineInsertion::AvailabilityOptions::NEVER,
53 "never", "r11 not available"),
54 clEnumValN(RetpolineInsertion::AvailabilityOptions::ALWAYS,
55 "always", "r11 available before calls and jumps"),
56 clEnumValN(RetpolineInsertion::AvailabilityOptions::ABI, "abi",
57 "r11 available before calls but not before jumps")),
58 cl::ZeroOrMore, cl::cat(BoltCategory));
59
60} // namespace opts
61
62namespace llvm {
63namespace bolt {
64
65// Retpoline function structure:
66// BB0: call BB2
67// BB1: pause
68// lfence
69// jmp BB1
70// BB2: mov %reg, (%rsp)
71// ret
72// or
73// BB2: push %r11
74// mov Address, %r11
75// mov %r11, 8(%rsp)
76// pop %r11
77// ret
78BinaryFunction *createNewRetpoline(BinaryContext &BC,
79 const std::string &RetpolineTag,
80 const IndirectBranchInfo &BrInfo,
81 bool R11Available) {
82 auto &MIB = *BC.MIB;
83 MCContext &Ctx = *BC.Ctx.get();
84 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Creating a new retpoline function["
85 << RetpolineTag << "]\n");
86
87 BinaryFunction *NewRetpoline =
88 BC.createInjectedBinaryFunction(Name: RetpolineTag, IsSimple: true);
89 std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks(3);
90 for (int I = 0; I < 3; I++) {
91 MCSymbol *Symbol =
92 Ctx.createNamedTempSymbol(Name: Twine(RetpolineTag + "_BB" + to_string(Value: I)));
93 NewBlocks[I] = NewRetpoline->createBasicBlock(Label: Symbol);
94 NewBlocks[I].get()->setCFIState(0);
95 }
96
97 BinaryBasicBlock &BB0 = *NewBlocks[0].get();
98 BinaryBasicBlock &BB1 = *NewBlocks[1].get();
99 BinaryBasicBlock &BB2 = *NewBlocks[2].get();
100
101 BB0.addSuccessor(Succ: &BB2, Count: 0, MispredictedCount: 0);
102 BB1.addSuccessor(Succ: &BB1, Count: 0, MispredictedCount: 0);
103
104 // Build BB0
105 MCInst DirectCall;
106 MIB.createDirectCall(Inst&: DirectCall, Target: BB2.getLabel(), Ctx: &Ctx, /*IsTailCall*/ false);
107 BB0.addInstruction(Inst: DirectCall);
108
109 // Build BB1
110 MCInst Pause;
111 MIB.createPause(Inst&: Pause);
112 BB1.addInstruction(Inst: Pause);
113
114 if (opts::RetpolineLfence) {
115 MCInst Lfence;
116 MIB.createLfence(Inst&: Lfence);
117 BB1.addInstruction(Inst: Lfence);
118 }
119
120 InstructionListType Seq;
121 MIB.createShortJmp(Seq, Target: BB1.getLabel(), Ctx: &Ctx);
122 BB1.addInstructions(Begin: Seq.begin(), End: Seq.end());
123
124 // Build BB2
125 if (BrInfo.isMem()) {
126 if (R11Available) {
127 MCInst StoreToStack;
128 MIB.createSaveToStack(Inst&: StoreToStack, StackReg: MIB.getStackPointer(), Offset: 0,
129 SrcReg: MIB.getX86R11(), Size: 8);
130 BB2.addInstruction(Inst: StoreToStack);
131 } else {
132 MCInst PushR11;
133 MIB.createPushRegister(Inst&: PushR11, Reg: MIB.getX86R11(), Size: 8);
134 BB2.addInstruction(Inst: PushR11);
135
136 MCInst LoadCalleeAddrs;
137 const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
138 MIB.createLoad(Inst&: LoadCalleeAddrs, BaseReg: MemRef.BaseRegNum, Scale: MemRef.ScaleImm,
139 IndexReg: MemRef.IndexRegNum, Offset: MemRef.DispImm, OffsetExpr: MemRef.DispExpr,
140 AddrSegmentReg: MemRef.SegRegNum, DstReg: MIB.getX86R11(), Size: 8);
141
142 BB2.addInstruction(Inst: LoadCalleeAddrs);
143
144 MCInst StoreToStack;
145 MIB.createSaveToStack(Inst&: StoreToStack, StackReg: MIB.getStackPointer(), Offset: 8,
146 SrcReg: MIB.getX86R11(), Size: 8);
147 BB2.addInstruction(Inst: StoreToStack);
148
149 MCInst PopR11;
150 MIB.createPopRegister(Inst&: PopR11, Reg: MIB.getX86R11(), Size: 8);
151 BB2.addInstruction(Inst: PopR11);
152 }
153 } else if (BrInfo.isReg()) {
154 MCInst StoreToStack;
155 MIB.createSaveToStack(Inst&: StoreToStack, StackReg: MIB.getStackPointer(), Offset: 0,
156 SrcReg: BrInfo.BranchReg, Size: 8);
157 BB2.addInstruction(Inst: StoreToStack);
158 } else {
159 llvm_unreachable("not expected");
160 }
161
162 // return
163 MCInst Return;
164 MIB.createReturn(Inst&: Return);
165 BB2.addInstruction(Inst: Return);
166 NewRetpoline->insertBasicBlocks(Start: nullptr, NewBBs: std::move(NewBlocks),
167 /* UpdateLayout */ true,
168 /* UpdateCFIState */ false);
169
170 NewRetpoline->updateState(State: BinaryFunction::State::CFG_Finalized);
171 return NewRetpoline;
172}
173
174std::string createRetpolineFunctionTag(BinaryContext &BC,
175 const IndirectBranchInfo &BrInfo,
176 bool R11Available) {
177 std::string Tag;
178 llvm::raw_string_ostream TagOS(Tag);
179 TagOS << "__retpoline_";
180
181 if (BrInfo.isReg()) {
182 BC.InstPrinter->printRegName(OS&: TagOS, Reg: BrInfo.BranchReg);
183 TagOS << "_";
184 TagOS.flush();
185 return Tag;
186 }
187
188 // Memory Branch
189 if (R11Available)
190 return "__retpoline_r11";
191
192 const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
193
194 TagOS << "mem_";
195
196 if (MemRef.BaseRegNum != BC.MIB->getNoRegister())
197 BC.InstPrinter->printRegName(OS&: TagOS, Reg: MemRef.BaseRegNum);
198
199 TagOS << "+";
200 if (MemRef.DispExpr)
201 MemRef.DispExpr->print(OS&: TagOS, MAI: BC.AsmInfo.get());
202 else
203 TagOS << MemRef.DispImm;
204
205 if (MemRef.IndexRegNum != BC.MIB->getNoRegister()) {
206 TagOS << "+" << MemRef.ScaleImm << "*";
207 BC.InstPrinter->printRegName(OS&: TagOS, Reg: MemRef.IndexRegNum);
208 }
209
210 if (MemRef.SegRegNum != BC.MIB->getNoRegister()) {
211 TagOS << "_seg_";
212 BC.InstPrinter->printRegName(OS&: TagOS, Reg: MemRef.SegRegNum);
213 }
214
215 TagOS.flush();
216 return Tag;
217}
218
219BinaryFunction *RetpolineInsertion::getOrCreateRetpoline(
220 BinaryContext &BC, const IndirectBranchInfo &BrInfo, bool R11Available) {
221 const std::string RetpolineTag =
222 createRetpolineFunctionTag(BC, BrInfo, R11Available);
223
224 if (CreatedRetpolines.count(x: RetpolineTag))
225 return CreatedRetpolines[RetpolineTag];
226
227 return CreatedRetpolines[RetpolineTag] =
228 createNewRetpoline(BC, RetpolineTag, BrInfo, R11Available);
229}
230
231void createBranchReplacement(BinaryContext &BC,
232 const IndirectBranchInfo &BrInfo,
233 bool R11Available,
234 InstructionListType &Replacement,
235 const MCSymbol *RetpolineSymbol) {
236 auto &MIB = *BC.MIB;
237 // Load the branch address in r11 if available
238 if (BrInfo.isMem() && R11Available) {
239 const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
240 MCInst LoadCalleeAddrs;
241 MIB.createLoad(Inst&: LoadCalleeAddrs, BaseReg: MemRef.BaseRegNum, Scale: MemRef.ScaleImm,
242 IndexReg: MemRef.IndexRegNum, Offset: MemRef.DispImm, OffsetExpr: MemRef.DispExpr,
243 AddrSegmentReg: MemRef.SegRegNum, DstReg: MIB.getX86R11(), Size: 8);
244 Replacement.push_back(x: LoadCalleeAddrs);
245 }
246
247 // Call the retpoline
248 MCInst RetpolineCall;
249 MIB.createDirectCall(Inst&: RetpolineCall, Target: RetpolineSymbol, Ctx: BC.Ctx.get(),
250 IsTailCall: BrInfo.isJump() || BrInfo.isTailCall());
251
252 Replacement.push_back(x: RetpolineCall);
253}
254
255IndirectBranchInfo::IndirectBranchInfo(MCInst &Inst, MCPlusBuilder &MIB) {
256 IsCall = MIB.isCall(Inst);
257 IsTailCall = MIB.isTailCall(Inst);
258
259 if (MIB.isBranchOnMem(Inst)) {
260 IsMem = true;
261 std::optional<MCPlusBuilder::X86MemOperand> MO =
262 MIB.evaluateX86MemoryOperand(Inst);
263 if (!MO)
264 llvm_unreachable("not expected");
265 Memory = MO.value();
266 } else if (MIB.isBranchOnReg(Inst)) {
267 assert(MCPlus::getNumPrimeOperands(Inst) == 1 && "expect 1 operand");
268 BranchReg = Inst.getOperand(i: 0).getReg();
269 } else {
270 llvm_unreachable("unexpected instruction");
271 }
272}
273
274Error RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
275 if (!opts::InsertRetpolines)
276 return Error::success();
277
278 assert(BC.isX86() &&
279 "retpoline insertion not supported for target architecture");
280
281 assert(BC.HasRelocations && "retpoline mode not supported in non-reloc");
282
283 auto &MIB = *BC.MIB;
284 uint32_t RetpolinedBranches = 0;
285 for (auto &It : BC.getBinaryFunctions()) {
286 BinaryFunction &Function = It.second;
287 for (BinaryBasicBlock &BB : Function) {
288 for (auto It = BB.begin(); It != BB.end(); ++It) {
289 MCInst &Inst = *It;
290
291 if (!MIB.isIndirectCall(Inst) && !MIB.isIndirectBranch(Inst))
292 continue;
293
294 IndirectBranchInfo BrInfo(Inst, MIB);
295 bool R11Available = false;
296 BinaryFunction *TargetRetpoline;
297 InstructionListType Replacement;
298
299 // Determine if r11 is available before this instruction
300 if (BrInfo.isMem()) {
301 if (MIB.hasAnnotation(Inst, Name: "PLTCall"))
302 R11Available = true;
303 else if (opts::R11Availability == AvailabilityOptions::ALWAYS)
304 R11Available = true;
305 else if (opts::R11Availability == AvailabilityOptions::ABI)
306 R11Available = BrInfo.isCall();
307 }
308
309 // If the instruction addressing pattern uses rsp and the retpoline
310 // loads the callee address then displacement needs to be updated
311 if (BrInfo.isMem() && !R11Available) {
312 IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
313 int Addend = (BrInfo.isJump() || BrInfo.isTailCall()) ? 8 : 16;
314 if (MemRef.BaseRegNum == MIB.getStackPointer())
315 MemRef.DispImm += Addend;
316 if (MemRef.IndexRegNum == MIB.getStackPointer())
317 MemRef.DispImm += Addend * MemRef.ScaleImm;
318 }
319
320 TargetRetpoline = getOrCreateRetpoline(BC, BrInfo, R11Available);
321
322 createBranchReplacement(BC, BrInfo, R11Available, Replacement,
323 RetpolineSymbol: TargetRetpoline->getSymbol());
324
325 It = BB.replaceInstruction(II: It, Begin: Replacement.begin(), End: Replacement.end());
326 RetpolinedBranches++;
327 }
328 }
329 }
330 BC.outs() << "BOLT-INFO: The number of created retpoline functions is : "
331 << CreatedRetpolines.size()
332 << "\nBOLT-INFO: The number of retpolined branches is : "
333 << RetpolinedBranches << "\n";
334 return Error::success();
335}
336
337} // namespace bolt
338} // namespace llvm
339

source code of bolt/lib/Passes/RetpolineInsertion.cpp