1 | //===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains a pass to insert code to mitigate against side channel |
10 | // vulnerabilities that may happen under straight line miss-speculation. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "ARM.h" |
15 | #include "ARMInstrInfo.h" |
16 | #include "ARMSubtarget.h" |
17 | #include "llvm/CodeGen/IndirectThunks.h" |
18 | #include "llvm/CodeGen/MachineBasicBlock.h" |
19 | #include "llvm/CodeGen/MachineFunction.h" |
20 | #include "llvm/CodeGen/MachineFunctionPass.h" |
21 | #include "llvm/CodeGen/MachineInstr.h" |
22 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
23 | #include "llvm/CodeGen/MachineOperand.h" |
24 | #include "llvm/IR/DebugLoc.h" |
25 | #include <cassert> |
26 | |
27 | using namespace llvm; |
28 | |
29 | #define DEBUG_TYPE "arm-sls-hardening" |
30 | |
31 | #define ARM_SLS_HARDENING_NAME "ARM sls hardening pass" |
32 | |
33 | namespace { |
34 | |
35 | class ARMSLSHardening : public MachineFunctionPass { |
36 | public: |
37 | const TargetInstrInfo *TII; |
38 | const ARMSubtarget *ST; |
39 | |
40 | static char ID; |
41 | |
42 | ARMSLSHardening() : MachineFunctionPass(ID) { |
43 | initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry()); |
44 | } |
45 | |
46 | bool runOnMachineFunction(MachineFunction &Fn) override; |
47 | |
48 | StringRef getPassName() const override { return ARM_SLS_HARDENING_NAME; } |
49 | |
50 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
51 | AU.setPreservesCFG(); |
52 | MachineFunctionPass::getAnalysisUsage(AU); |
53 | } |
54 | |
55 | private: |
56 | bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const; |
57 | bool hardenIndirectCalls(MachineBasicBlock &MBB) const; |
58 | MachineBasicBlock & |
59 | ConvertIndirectCallToIndirectJump(MachineBasicBlock &MBB, |
60 | MachineBasicBlock::iterator) const; |
61 | }; |
62 | |
63 | } // end anonymous namespace |
64 | |
65 | char ARMSLSHardening::ID = 0; |
66 | |
67 | INITIALIZE_PASS(ARMSLSHardening, "arm-sls-hardening" , |
68 | ARM_SLS_HARDENING_NAME, false, false) |
69 | |
70 | static void insertSpeculationBarrier(const ARMSubtarget *ST, |
71 | MachineBasicBlock &MBB, |
72 | MachineBasicBlock::iterator MBBI, |
73 | DebugLoc DL, |
74 | bool AlwaysUseISBDSB = false) { |
75 | assert(MBBI != MBB.begin() && |
76 | "Must not insert SpeculationBarrierEndBB as only instruction in MBB." ); |
77 | assert(std::prev(MBBI)->isBarrier() && |
78 | "SpeculationBarrierEndBB must only follow unconditional control flow " |
79 | "instructions." ); |
80 | assert(std::prev(MBBI)->isTerminator() && |
81 | "SpeculationBarrierEndBB must only follow terminators." ); |
82 | const TargetInstrInfo *TII = ST->getInstrInfo(); |
83 | assert(ST->hasDataBarrier() || ST->hasSB()); |
84 | bool ProduceSB = ST->hasSB() && !AlwaysUseISBDSB; |
85 | unsigned BarrierOpc = |
86 | ProduceSB ? (ST->isThumb() ? ARM::t2SpeculationBarrierSBEndBB |
87 | : ARM::SpeculationBarrierSBEndBB) |
88 | : (ST->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB |
89 | : ARM::SpeculationBarrierISBDSBEndBB); |
90 | if (MBBI == MBB.end() || !isSpeculationBarrierEndBBOpcode(Opc: MBBI->getOpcode())) |
91 | BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: BarrierOpc)); |
92 | } |
93 | |
94 | bool ARMSLSHardening::runOnMachineFunction(MachineFunction &MF) { |
95 | ST = &MF.getSubtarget<ARMSubtarget>(); |
96 | TII = MF.getSubtarget().getInstrInfo(); |
97 | |
98 | bool Modified = false; |
99 | for (auto &MBB : MF) { |
100 | Modified |= hardenReturnsAndBRs(MBB); |
101 | Modified |= hardenIndirectCalls(MBB); |
102 | } |
103 | |
104 | return Modified; |
105 | } |
106 | |
107 | bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const { |
108 | if (!ST->hardenSlsRetBr()) |
109 | return false; |
110 | assert(!ST->isThumb1Only()); |
111 | bool Modified = false; |
112 | MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end(); |
113 | MachineBasicBlock::iterator NextMBBI; |
114 | for (; MBBI != E; MBBI = NextMBBI) { |
115 | MachineInstr &MI = *MBBI; |
116 | NextMBBI = std::next(x: MBBI); |
117 | if (isIndirectControlFlowNotComingBack(MI)) { |
118 | assert(MI.isTerminator()); |
119 | assert(!TII->isPredicated(MI)); |
120 | insertSpeculationBarrier(ST, MBB, MBBI: std::next(x: MBBI), DL: MI.getDebugLoc()); |
121 | Modified = true; |
122 | } |
123 | } |
124 | return Modified; |
125 | } |
126 | |
127 | static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_" ; |
128 | |
129 | static const struct ThunkNameRegMode { |
130 | const char* Name; |
131 | Register Reg; |
132 | bool isThumb; |
133 | } SLSBLRThunks[] = { |
134 | {"__llvm_slsblr_thunk_arm_r0" , ARM::R0, false}, |
135 | {"__llvm_slsblr_thunk_arm_r1" , ARM::R1, false}, |
136 | {"__llvm_slsblr_thunk_arm_r2" , ARM::R2, false}, |
137 | {"__llvm_slsblr_thunk_arm_r3" , ARM::R3, false}, |
138 | {"__llvm_slsblr_thunk_arm_r4" , ARM::R4, false}, |
139 | {"__llvm_slsblr_thunk_arm_r5" , ARM::R5, false}, |
140 | {"__llvm_slsblr_thunk_arm_r6" , ARM::R6, false}, |
141 | {"__llvm_slsblr_thunk_arm_r7" , ARM::R7, false}, |
142 | {"__llvm_slsblr_thunk_arm_r8" , ARM::R8, false}, |
143 | {"__llvm_slsblr_thunk_arm_r9" , ARM::R9, false}, |
144 | {"__llvm_slsblr_thunk_arm_r10" , ARM::R10, false}, |
145 | {"__llvm_slsblr_thunk_arm_r11" , ARM::R11, false}, |
146 | {"__llvm_slsblr_thunk_arm_sp" , ARM::SP, false}, |
147 | {"__llvm_slsblr_thunk_arm_pc" , ARM::PC, false}, |
148 | {"__llvm_slsblr_thunk_thumb_r0" , ARM::R0, true}, |
149 | {"__llvm_slsblr_thunk_thumb_r1" , ARM::R1, true}, |
150 | {"__llvm_slsblr_thunk_thumb_r2" , ARM::R2, true}, |
151 | {"__llvm_slsblr_thunk_thumb_r3" , ARM::R3, true}, |
152 | {"__llvm_slsblr_thunk_thumb_r4" , ARM::R4, true}, |
153 | {"__llvm_slsblr_thunk_thumb_r5" , ARM::R5, true}, |
154 | {"__llvm_slsblr_thunk_thumb_r6" , ARM::R6, true}, |
155 | {"__llvm_slsblr_thunk_thumb_r7" , ARM::R7, true}, |
156 | {"__llvm_slsblr_thunk_thumb_r8" , ARM::R8, true}, |
157 | {"__llvm_slsblr_thunk_thumb_r9" , ARM::R9, true}, |
158 | {"__llvm_slsblr_thunk_thumb_r10" , ARM::R10, true}, |
159 | {"__llvm_slsblr_thunk_thumb_r11" , ARM::R11, true}, |
160 | {"__llvm_slsblr_thunk_thumb_sp" , ARM::SP, true}, |
161 | {"__llvm_slsblr_thunk_thumb_pc" , ARM::PC, true}, |
162 | }; |
163 | |
164 | // An enum for tracking whether Arm and Thumb thunks have been inserted into the |
165 | // current module so far. |
166 | enum ArmInsertedThunks { ArmThunk = 1, ThumbThunk = 2 }; |
167 | |
168 | inline ArmInsertedThunks &operator|=(ArmInsertedThunks &X, |
169 | ArmInsertedThunks Y) { |
170 | return X = static_cast<ArmInsertedThunks>(X | Y); |
171 | } |
172 | |
173 | namespace { |
174 | struct SLSBLRThunkInserter |
175 | : ThunkInserter<SLSBLRThunkInserter, ArmInsertedThunks> { |
176 | const char *getThunkPrefix() { return SLSBLRNamePrefix; } |
177 | bool mayUseThunk(const MachineFunction &MF, |
178 | ArmInsertedThunks InsertedThunks) { |
179 | if ((InsertedThunks & ArmThunk && |
180 | !MF.getSubtarget<ARMSubtarget>().isThumb()) || |
181 | (InsertedThunks & ThumbThunk && |
182 | MF.getSubtarget<ARMSubtarget>().isThumb())) |
183 | return false; |
184 | ComdatThunks &= !MF.getSubtarget<ARMSubtarget>().hardenSlsNoComdat(); |
185 | // FIXME: This could also check if there are any indirect calls in the |
186 | // function to more accurately reflect if a thunk will be needed. |
187 | return MF.getSubtarget<ARMSubtarget>().hardenSlsBlr(); |
188 | } |
189 | ArmInsertedThunks insertThunks(MachineModuleInfo &MMI, MachineFunction &MF); |
190 | void populateThunk(MachineFunction &MF); |
191 | |
192 | private: |
193 | bool ComdatThunks = true; |
194 | }; |
195 | } // namespace |
196 | |
197 | ArmInsertedThunks SLSBLRThunkInserter::insertThunks(MachineModuleInfo &MMI, |
198 | MachineFunction &MF) { |
199 | // FIXME: It probably would be possible to filter which thunks to produce |
200 | // based on which registers are actually used in indirect calls in this |
201 | // function. But would that be a worthwhile optimization? |
202 | const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>(); |
203 | for (auto T : SLSBLRThunks) |
204 | if (ST->isThumb() == T.isThumb) |
205 | createThunkFunction(MMI, T.Name, ComdatThunks, |
206 | T.isThumb ? "+thumb-mode" : "" ); |
207 | return ST->isThumb() ? ThumbThunk : ArmThunk; |
208 | } |
209 | |
210 | void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) { |
211 | // FIXME: How to better communicate Register number, rather than through |
212 | // name and lookup table? |
213 | assert(MF.getName().starts_with(getThunkPrefix())); |
214 | auto ThunkIt = llvm::find_if( |
215 | SLSBLRThunks, [&MF](auto T) { return T.Name == MF.getName(); }); |
216 | assert(ThunkIt != std::end(SLSBLRThunks)); |
217 | Register ThunkReg = ThunkIt->Reg; |
218 | bool isThumb = ThunkIt->isThumb; |
219 | |
220 | const TargetInstrInfo *TII = MF.getSubtarget<ARMSubtarget>().getInstrInfo(); |
221 | MachineBasicBlock *Entry = &MF.front(); |
222 | Entry->clear(); |
223 | |
224 | // These thunks need to consist of the following instructions: |
225 | // __llvm_slsblr_thunk_(arm/thumb)_rN: |
226 | // bx rN |
227 | // barrierInsts |
228 | Entry->addLiveIn(PhysReg: ThunkReg); |
229 | if (isThumb) |
230 | BuildMI(Entry, DebugLoc(), TII->get(ARM::tBX)) |
231 | .addReg(ThunkReg) |
232 | .add(predOps(ARMCC::AL)); |
233 | else |
234 | BuildMI(Entry, DebugLoc(), TII->get(ARM::BX)) |
235 | .addReg(ThunkReg); |
236 | |
237 | // Make sure the thunks do not make use of the SB extension in case there is |
238 | // a function somewhere that will call to it that for some reason disabled |
239 | // the SB extension locally on that function, even though it's enabled for |
240 | // the module otherwise. Therefore set AlwaysUseISBSDB to true. |
241 | insertSpeculationBarrier(ST: &MF.getSubtarget<ARMSubtarget>(), MBB&: *Entry, |
242 | MBBI: Entry->end(), DL: DebugLoc(), AlwaysUseISBDSB: true /*AlwaysUseISBDSB*/); |
243 | } |
244 | |
245 | MachineBasicBlock &ARMSLSHardening::ConvertIndirectCallToIndirectJump( |
246 | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { |
247 | // Transform an indirect call to an indirect jump as follows: |
248 | // Before: |
249 | // |-----------------------------| |
250 | // | ... | |
251 | // | instI | |
252 | // | BLX rN | |
253 | // | instJ | |
254 | // | ... | |
255 | // |-----------------------------| |
256 | // |
257 | // After: |
258 | // |---------- -------------------------| |
259 | // | ... | |
260 | // | instI | |
261 | // | *call* __llvm_slsblr_thunk_mode_xN | |
262 | // | instJ | |
263 | // | ... | |
264 | // |--------------------------------------| |
265 | // |
266 | // __llvm_slsblr_thunk_mode_xN: |
267 | // |-----------------------------| |
268 | // | BX rN | |
269 | // | barrierInsts | |
270 | // |-----------------------------| |
271 | // |
272 | // The __llvm_slsblr_thunk_mode_xN thunks are created by the |
273 | // SLSBLRThunkInserter. |
274 | // This function merely needs to transform an indirect call to a direct call |
275 | // to __llvm_slsblr_thunk_xN. |
276 | MachineInstr &IndirectCall = *MBBI; |
277 | assert(isIndirectCall(IndirectCall) && !IndirectCall.isReturn()); |
278 | int RegOpIdxOnIndirectCall = -1; |
279 | bool isThumb; |
280 | switch (IndirectCall.getOpcode()) { |
281 | case ARM::BLX: // !isThumb2 |
282 | case ARM::BLX_noip: // !isThumb2 |
283 | isThumb = false; |
284 | RegOpIdxOnIndirectCall = 0; |
285 | break; |
286 | case ARM::tBLXr: // isThumb2 |
287 | case ARM::tBLXr_noip: // isThumb2 |
288 | isThumb = true; |
289 | RegOpIdxOnIndirectCall = 2; |
290 | break; |
291 | default: |
292 | llvm_unreachable("unhandled Indirect Call" ); |
293 | } |
294 | |
295 | Register Reg = IndirectCall.getOperand(i: RegOpIdxOnIndirectCall).getReg(); |
296 | // Since linkers are allowed to clobber R12 on function calls, the above |
297 | // mitigation only works if the original indirect call instruction was not |
298 | // using R12. Code generation before must make sure that no indirect call |
299 | // using R12 was produced if the mitigation is enabled. |
300 | // Also, the transformation is incorrect if the indirect call uses LR, so |
301 | // also have to avoid that. |
302 | assert(Reg != ARM::R12 && Reg != ARM::LR); |
303 | bool RegIsKilled = IndirectCall.getOperand(i: RegOpIdxOnIndirectCall).isKill(); |
304 | |
305 | DebugLoc DL = IndirectCall.getDebugLoc(); |
306 | |
307 | MachineFunction &MF = *MBBI->getMF(); |
308 | auto ThunkIt = llvm::find_if(SLSBLRThunks, [Reg, isThumb](auto T) { |
309 | return T.Reg == Reg && T.isThumb == isThumb; |
310 | }); |
311 | assert(ThunkIt != std::end(SLSBLRThunks)); |
312 | Module *M = MF.getFunction().getParent(); |
313 | const GlobalValue *GV = cast<GlobalValue>(M->getNamedValue(Name: ThunkIt->Name)); |
314 | |
315 | MachineInstr *BL = |
316 | isThumb ? BuildMI(MBB, MBBI, DL, TII->get(ARM::tBL)) |
317 | .addImm(IndirectCall.getOperand(0).getImm()) |
318 | .addReg(IndirectCall.getOperand(1).getReg()) |
319 | .addGlobalAddress(GV) |
320 | : BuildMI(MBB, MBBI, DL, TII->get(ARM::BL)).addGlobalAddress(GV); |
321 | |
322 | // Now copy the implicit operands from IndirectCall to BL and copy other |
323 | // necessary info. |
324 | // However, both IndirectCall and BL instructions implictly use SP and |
325 | // implicitly define LR. Blindly copying implicit operands would result in SP |
326 | // and LR operands to be present multiple times. While this may not be too |
327 | // much of an issue, let's avoid that for cleanliness, by removing those |
328 | // implicit operands from the BL created above before we copy over all |
329 | // implicit operands from the IndirectCall. |
330 | int ImpLROpIdx = -1; |
331 | int ImpSPOpIdx = -1; |
332 | for (unsigned OpIdx = BL->getNumExplicitOperands(); |
333 | OpIdx < BL->getNumOperands(); OpIdx++) { |
334 | MachineOperand Op = BL->getOperand(i: OpIdx); |
335 | if (!Op.isReg()) |
336 | continue; |
337 | if (Op.getReg() == ARM::LR && Op.isDef()) |
338 | ImpLROpIdx = OpIdx; |
339 | if (Op.getReg() == ARM::SP && !Op.isDef()) |
340 | ImpSPOpIdx = OpIdx; |
341 | } |
342 | assert(ImpLROpIdx != -1); |
343 | assert(ImpSPOpIdx != -1); |
344 | int FirstOpIdxToRemove = std::max(a: ImpLROpIdx, b: ImpSPOpIdx); |
345 | int SecondOpIdxToRemove = std::min(a: ImpLROpIdx, b: ImpSPOpIdx); |
346 | BL->removeOperand(OpNo: FirstOpIdxToRemove); |
347 | BL->removeOperand(OpNo: SecondOpIdxToRemove); |
348 | // Now copy over the implicit operands from the original IndirectCall |
349 | BL->copyImplicitOps(MF, MI: IndirectCall); |
350 | MF.moveCallSiteInfo(Old: &IndirectCall, New: BL); |
351 | // Also add the register called in the IndirectCall as being used in the |
352 | // called thunk. |
353 | BL->addOperand(Op: MachineOperand::CreateReg(Reg, isDef: false /*isDef*/, isImp: true /*isImp*/, |
354 | isKill: RegIsKilled /*isKill*/)); |
355 | // Remove IndirectCallinstruction |
356 | MBB.erase(I: MBBI); |
357 | return MBB; |
358 | } |
359 | |
360 | bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock &MBB) const { |
361 | if (!ST->hardenSlsBlr()) |
362 | return false; |
363 | bool Modified = false; |
364 | MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); |
365 | MachineBasicBlock::iterator NextMBBI; |
366 | for (; MBBI != E; MBBI = NextMBBI) { |
367 | MachineInstr &MI = *MBBI; |
368 | NextMBBI = std::next(x: MBBI); |
369 | // Tail calls are both indirect calls and "returns". |
370 | // They are also indirect jumps, so should be handled by sls-harden-retbr, |
371 | // rather than sls-harden-blr. |
372 | if (isIndirectCall(MI) && !MI.isReturn()) { |
373 | ConvertIndirectCallToIndirectJump(MBB, MBBI); |
374 | Modified = true; |
375 | } |
376 | } |
377 | return Modified; |
378 | } |
379 | |
380 | |
381 | |
382 | FunctionPass *llvm::createARMSLSHardeningPass() { |
383 | return new ARMSLSHardening(); |
384 | } |
385 | |
386 | namespace { |
387 | class ARMIndirectThunks : public MachineFunctionPass { |
388 | public: |
389 | static char ID; |
390 | |
391 | ARMIndirectThunks() : MachineFunctionPass(ID) {} |
392 | |
393 | StringRef getPassName() const override { return "ARM Indirect Thunks" ; } |
394 | |
395 | bool doInitialization(Module &M) override; |
396 | bool runOnMachineFunction(MachineFunction &MF) override; |
397 | |
398 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
399 | MachineFunctionPass::getAnalysisUsage(AU); |
400 | AU.addRequired<MachineModuleInfoWrapperPass>(); |
401 | AU.addPreserved<MachineModuleInfoWrapperPass>(); |
402 | } |
403 | |
404 | private: |
405 | std::tuple<SLSBLRThunkInserter> TIs; |
406 | |
407 | template <typename... ThunkInserterT> |
408 | static void initTIs(Module &M, |
409 | std::tuple<ThunkInserterT...> &ThunkInserters) { |
410 | (..., std::get<ThunkInserterT>(ThunkInserters).init(M)); |
411 | } |
412 | template <typename... ThunkInserterT> |
413 | static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF, |
414 | std::tuple<ThunkInserterT...> &ThunkInserters) { |
415 | return (0 | ... | std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)); |
416 | } |
417 | }; |
418 | |
419 | } // end anonymous namespace |
420 | |
421 | char ARMIndirectThunks::ID = 0; |
422 | |
423 | FunctionPass *llvm::createARMIndirectThunks() { |
424 | return new ARMIndirectThunks(); |
425 | } |
426 | |
427 | bool ARMIndirectThunks::doInitialization(Module &M) { |
428 | initTIs(M, ThunkInserters&: TIs); |
429 | return false; |
430 | } |
431 | |
432 | bool ARMIndirectThunks::runOnMachineFunction(MachineFunction &MF) { |
433 | LLVM_DEBUG(dbgs() << getPassName() << '\n'); |
434 | auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); |
435 | return runTIs(MMI, MF, ThunkInserters&: TIs); |
436 | } |
437 | |