1 | //===-- AArch64PointerAuth.cpp -- Harden code using PAuth ------------------==// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "AArch64PointerAuth.h" |
10 | |
11 | #include "AArch64.h" |
12 | #include "AArch64InstrInfo.h" |
13 | #include "AArch64MachineFunctionInfo.h" |
14 | #include "AArch64Subtarget.h" |
15 | #include "Utils/AArch64BaseInfo.h" |
16 | #include "llvm/CodeGen/MachineBasicBlock.h" |
17 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
18 | #include "llvm/CodeGen/MachineModuleInfo.h" |
19 | |
20 | using namespace llvm; |
21 | using namespace llvm::AArch64PAuth; |
22 | |
23 | #define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication" |
24 | |
25 | namespace { |
26 | |
27 | class AArch64PointerAuth : public MachineFunctionPass { |
28 | public: |
29 | static char ID; |
30 | |
31 | AArch64PointerAuth() : MachineFunctionPass(ID) {} |
32 | |
33 | bool runOnMachineFunction(MachineFunction &MF) override; |
34 | |
35 | StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; } |
36 | |
37 | private: |
38 | /// An immediate operand passed to BRK instruction, if it is ever emitted. |
39 | static unsigned BrkOperandForKey(AArch64PACKey::ID KeyId) { |
40 | const unsigned BrkOperandBase = 0xc470; |
41 | return BrkOperandBase + KeyId; |
42 | } |
43 | |
44 | const AArch64Subtarget *Subtarget = nullptr; |
45 | const AArch64InstrInfo *TII = nullptr; |
46 | const AArch64RegisterInfo *TRI = nullptr; |
47 | |
48 | void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const; |
49 | |
50 | void authenticateLR(MachineFunction &MF, |
51 | MachineBasicBlock::iterator MBBI) const; |
52 | |
53 | /// Stores blend(AddrDisc, IntDisc) to the Result register. |
54 | void emitBlend(MachineBasicBlock::iterator MBBI, Register Result, |
55 | Register AddrDisc, unsigned IntDisc) const; |
56 | |
57 | /// Expands PAUTH_BLEND pseudo instruction. |
58 | void expandPAuthBlend(MachineBasicBlock::iterator MBBI) const; |
59 | |
60 | bool checkAuthenticatedLR(MachineBasicBlock::iterator TI) const; |
61 | }; |
62 | |
63 | } // end anonymous namespace |
64 | |
65 | INITIALIZE_PASS(AArch64PointerAuth, "aarch64-ptrauth" , |
66 | AARCH64_POINTER_AUTH_NAME, false, false) |
67 | |
68 | FunctionPass *llvm::createAArch64PointerAuthPass() { |
69 | return new AArch64PointerAuth(); |
70 | } |
71 | |
72 | char AArch64PointerAuth::ID = 0; |
73 | |
74 | // Where PAuthLR support is not known at compile time, it is supported using |
75 | // PACM. PACM is in the hint space so has no effect when PAuthLR is not |
76 | // supported by the hardware, but will alter the behaviour of PACI*SP, AUTI*SP |
77 | // and RETAA/RETAB if the hardware supports PAuthLR. |
78 | static void BuildPACM(const AArch64Subtarget &Subtarget, MachineBasicBlock &MBB, |
79 | MachineBasicBlock::iterator MBBI, DebugLoc DL, |
80 | MachineInstr::MIFlag Flags, MCSymbol *PACSym = nullptr) { |
81 | const TargetInstrInfo *TII = Subtarget.getInstrInfo(); |
82 | auto &MFnI = *MBB.getParent()->getInfo<AArch64FunctionInfo>(); |
83 | |
84 | // ADR X16,<address_of_PACIASP> |
85 | if (PACSym) { |
86 | assert(Flags == MachineInstr::FrameDestroy); |
87 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::Opcode: ADR)) |
88 | .addReg(AArch64::X16, RegState::Define) |
89 | .addSym(PACSym); |
90 | } |
91 | |
92 | // Only emit PACM if -mbranch-protection has +pc and the target does not |
93 | // have feature +pauth-lr. |
94 | if (MFnI.branchProtectionPAuthLR() && !Subtarget.hasPAuthLR()) |
95 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::Opcode: PACM)).setMIFlag(Flags); |
96 | } |
97 | |
98 | void AArch64PointerAuth::signLR(MachineFunction &MF, |
99 | MachineBasicBlock::iterator MBBI) const { |
100 | auto &MFnI = *MF.getInfo<AArch64FunctionInfo>(); |
101 | bool UseBKey = MFnI.shouldSignWithBKey(); |
102 | bool EmitCFI = MFnI.needsDwarfUnwindInfo(MF); |
103 | bool NeedsWinCFI = MF.hasWinCFI(); |
104 | |
105 | MachineBasicBlock &MBB = *MBBI->getParent(); |
106 | |
107 | // Debug location must be unknown, see AArch64FrameLowering::emitPrologue. |
108 | DebugLoc DL; |
109 | |
110 | if (UseBKey) { |
111 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY)) |
112 | .setMIFlag(MachineInstr::FrameSetup); |
113 | } |
114 | |
115 | // PAuthLR authentication instructions need to know the value of PC at the |
116 | // point of signing (PACI*). |
117 | if (MFnI.branchProtectionPAuthLR()) { |
118 | MCSymbol *PACSym = MF.getMMI().getContext().createTempSymbol(); |
119 | MFnI.setSigningInstrLabel(PACSym); |
120 | } |
121 | |
122 | // No SEH opcode for this one; it doesn't materialize into an |
123 | // instruction on Windows. |
124 | if (MFnI.branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
125 | BuildMI(MBB, MBBI, DL, |
126 | TII->get(MFnI.shouldSignWithBKey() ? AArch64::PACIBSPPC |
127 | : AArch64::PACIASPPC)) |
128 | .setMIFlag(MachineInstr::FrameSetup) |
129 | ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel()); |
130 | } else { |
131 | BuildPACM(Subtarget: *Subtarget, MBB, MBBI, DL, Flags: MachineInstr::FrameSetup); |
132 | BuildMI(MBB, MBBI, DL, |
133 | TII->get(MFnI.shouldSignWithBKey() ? AArch64::PACIBSP |
134 | : AArch64::PACIASP)) |
135 | .setMIFlag(MachineInstr::FrameSetup) |
136 | ->setPreInstrSymbol(MF, MFnI.getSigningInstrLabel()); |
137 | } |
138 | |
139 | if (EmitCFI) { |
140 | unsigned CFIIndex = |
141 | MF.addFrameInst(Inst: MCCFIInstruction::createNegateRAState(L: nullptr)); |
142 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
143 | .addCFIIndex(CFIIndex) |
144 | .setMIFlags(MachineInstr::FrameSetup); |
145 | } else if (NeedsWinCFI) { |
146 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PACSignLR)) |
147 | .setMIFlag(MachineInstr::FrameSetup); |
148 | } |
149 | } |
150 | |
151 | void AArch64PointerAuth::authenticateLR( |
152 | MachineFunction &MF, MachineBasicBlock::iterator MBBI) const { |
153 | const AArch64FunctionInfo *MFnI = MF.getInfo<AArch64FunctionInfo>(); |
154 | bool UseBKey = MFnI->shouldSignWithBKey(); |
155 | bool EmitAsyncCFI = MFnI->needsAsyncDwarfUnwindInfo(MF); |
156 | bool NeedsWinCFI = MF.hasWinCFI(); |
157 | |
158 | MachineBasicBlock &MBB = *MBBI->getParent(); |
159 | DebugLoc DL = MBBI->getDebugLoc(); |
160 | // MBBI points to a PAUTH_EPILOGUE instruction to be replaced and |
161 | // TI points to a terminator instruction that may or may not be combined. |
162 | // Note that inserting new instructions "before MBBI" and "before TI" is |
163 | // not the same because if ShadowCallStack is enabled, its instructions |
164 | // are placed between MBBI and TI. |
165 | MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator(); |
166 | |
167 | // The AUTIASP instruction assembles to a hint instruction before v8.3a so |
168 | // this instruction can safely used for any v8a architecture. |
169 | // From v8.3a onwards there are optimised authenticate LR and return |
170 | // instructions, namely RETA{A,B}, that can be used instead. In this case the |
171 | // DW_CFA_AARCH64_negate_ra_state can't be emitted. |
172 | bool TerminatorIsCombinable = |
173 | TI != MBB.end() && TI->getOpcode() == AArch64::RET; |
174 | MCSymbol *PACSym = MFnI->getSigningInstrLabel(); |
175 | |
176 | if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI && |
177 | !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) { |
178 | if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
179 | assert(PACSym && "No PAC instruction to refer to" ); |
180 | BuildMI(MBB, TI, DL, |
181 | TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi)) |
182 | .addSym(PACSym) |
183 | .copyImplicitOps(*MBBI) |
184 | .setMIFlag(MachineInstr::FrameDestroy); |
185 | } else { |
186 | BuildPACM(Subtarget: *Subtarget, MBB, MBBI: TI, DL, Flags: MachineInstr::FrameDestroy, PACSym); |
187 | BuildMI(MBB, TI, DL, TII->get(UseBKey ? AArch64::RETAB : AArch64::RETAA)) |
188 | .copyImplicitOps(*MBBI) |
189 | .setMIFlag(MachineInstr::FrameDestroy); |
190 | } |
191 | MBB.erase(I: TI); |
192 | } else { |
193 | if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) { |
194 | assert(PACSym && "No PAC instruction to refer to" ); |
195 | BuildMI(MBB, MBBI, DL, |
196 | TII->get(UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi)) |
197 | .addSym(PACSym) |
198 | .setMIFlag(MachineInstr::FrameDestroy); |
199 | } else { |
200 | BuildPACM(Subtarget: *Subtarget, MBB, MBBI, DL, Flags: MachineInstr::FrameDestroy, PACSym); |
201 | BuildMI(MBB, MBBI, DL, |
202 | TII->get(UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP)) |
203 | .setMIFlag(MachineInstr::FrameDestroy); |
204 | } |
205 | |
206 | if (EmitAsyncCFI) { |
207 | unsigned CFIIndex = |
208 | MF.addFrameInst(Inst: MCCFIInstruction::createNegateRAState(L: nullptr)); |
209 | BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
210 | .addCFIIndex(CFIIndex) |
211 | .setMIFlags(MachineInstr::FrameDestroy); |
212 | } |
213 | if (NeedsWinCFI) { |
214 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PACSignLR)) |
215 | .setMIFlag(MachineInstr::FrameDestroy); |
216 | } |
217 | } |
218 | } |
219 | |
220 | namespace { |
221 | |
222 | // Mark dummy LDR instruction as volatile to prevent removing it as dead code. |
223 | MachineMemOperand *createCheckMemOperand(MachineFunction &MF, |
224 | const AArch64Subtarget &Subtarget) { |
225 | MachinePointerInfo PointerInfo(Subtarget.getAddressCheckPSV()); |
226 | auto MOVolatileLoad = |
227 | MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; |
228 | |
229 | return MF.getMachineMemOperand(PtrInfo: PointerInfo, F: MOVolatileLoad, Size: 4, BaseAlignment: Align(4)); |
230 | } |
231 | |
232 | } // namespace |
233 | |
234 | MachineBasicBlock &llvm::AArch64PAuth::checkAuthenticatedRegister( |
235 | MachineBasicBlock::iterator MBBI, AuthCheckMethod Method, |
236 | Register AuthenticatedReg, Register TmpReg, bool UseIKey, unsigned BrkImm) { |
237 | |
238 | MachineBasicBlock &MBB = *MBBI->getParent(); |
239 | MachineFunction &MF = *MBB.getParent(); |
240 | const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>(); |
241 | const AArch64InstrInfo *TII = Subtarget.getInstrInfo(); |
242 | DebugLoc DL = MBBI->getDebugLoc(); |
243 | |
244 | // First, handle the methods not requiring creating extra MBBs. |
245 | switch (Method) { |
246 | default: |
247 | break; |
248 | case AuthCheckMethod::None: |
249 | return MBB; |
250 | case AuthCheckMethod::DummyLoad: |
251 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRWui), getWRegFromXReg(Reg: TmpReg)) |
252 | .addReg(AuthenticatedReg) |
253 | .addImm(0) |
254 | .addMemOperand(createCheckMemOperand(MF, Subtarget)); |
255 | return MBB; |
256 | } |
257 | |
258 | // Control flow has to be changed, so arrange new MBBs. |
259 | |
260 | // At now, at least an AUT* instruction is expected before MBBI |
261 | assert(MBBI != MBB.begin() && |
262 | "Cannot insert the check at the very beginning of MBB" ); |
263 | // The block to insert check into. |
264 | MachineBasicBlock *CheckBlock = &MBB; |
265 | // The remaining part of the original MBB that is executed on success. |
266 | MachineBasicBlock *SuccessBlock = MBB.splitAt(SplitInst&: *std::prev(x: MBBI)); |
267 | |
268 | // The block that explicitly generates a break-point exception on failure. |
269 | MachineBasicBlock *BreakBlock = |
270 | MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock()); |
271 | MF.push_back(MBB: BreakBlock); |
272 | MBB.splitSuccessor(Old: SuccessBlock, New: BreakBlock); |
273 | |
274 | assert(CheckBlock->getFallThrough() == SuccessBlock); |
275 | BuildMI(BreakBlock, DL, TII->get(AArch64::BRK)).addImm(BrkImm); |
276 | |
277 | switch (Method) { |
278 | case AuthCheckMethod::None: |
279 | case AuthCheckMethod::DummyLoad: |
280 | llvm_unreachable("Should be handled above" ); |
281 | case AuthCheckMethod::HighBitsNoTBI: |
282 | BuildMI(CheckBlock, DL, TII->get(AArch64::EORXrs), TmpReg) |
283 | .addReg(AuthenticatedReg) |
284 | .addReg(AuthenticatedReg) |
285 | .addImm(1); |
286 | BuildMI(CheckBlock, DL, TII->get(AArch64::TBNZX)) |
287 | .addReg(TmpReg) |
288 | .addImm(62) |
289 | .addMBB(BreakBlock); |
290 | return *SuccessBlock; |
291 | case AuthCheckMethod::XPACHint: |
292 | assert(AuthenticatedReg == AArch64::LR && |
293 | "XPACHint mode is only compatible with checking the LR register" ); |
294 | assert(UseIKey && "XPACHint mode is only compatible with I-keys" ); |
295 | BuildMI(CheckBlock, DL, TII->get(AArch64::ORRXrs), TmpReg) |
296 | .addReg(AArch64::XZR) |
297 | .addReg(AArch64::LR) |
298 | .addImm(0); |
299 | BuildMI(CheckBlock, DL, TII->get(AArch64::XPACLRI)); |
300 | BuildMI(CheckBlock, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR) |
301 | .addReg(TmpReg) |
302 | .addReg(AArch64::LR) |
303 | .addImm(0); |
304 | BuildMI(CheckBlock, DL, TII->get(AArch64::Bcc)) |
305 | .addImm(AArch64CC::NE) |
306 | .addMBB(BreakBlock); |
307 | return *SuccessBlock; |
308 | } |
309 | llvm_unreachable("Unknown AuthCheckMethod enum" ); |
310 | } |
311 | |
312 | unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) { |
313 | switch (Method) { |
314 | case AuthCheckMethod::None: |
315 | return 0; |
316 | case AuthCheckMethod::DummyLoad: |
317 | return 4; |
318 | case AuthCheckMethod::HighBitsNoTBI: |
319 | return 12; |
320 | case AuthCheckMethod::XPACHint: |
321 | return 20; |
322 | } |
323 | llvm_unreachable("Unknown AuthCheckMethod enum" ); |
324 | } |
325 | |
326 | bool AArch64PointerAuth::checkAuthenticatedLR( |
327 | MachineBasicBlock::iterator TI) const { |
328 | const AArch64FunctionInfo *MFnI = TI->getMF()->getInfo<AArch64FunctionInfo>(); |
329 | AArch64PACKey::ID KeyId = |
330 | MFnI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA; |
331 | |
332 | AuthCheckMethod Method = Subtarget->getAuthenticatedLRCheckMethod(); |
333 | |
334 | if (Method == AuthCheckMethod::None) |
335 | return false; |
336 | |
337 | // FIXME If FEAT_FPAC is implemented by the CPU, this check can be skipped. |
338 | |
339 | assert(!TI->getMF()->hasWinCFI() && "WinCFI is not yet supported" ); |
340 | |
341 | // The following code may create a signing oracle: |
342 | // |
343 | // <authenticate LR> |
344 | // TCRETURN ; the callee may sign and spill the LR in its prologue |
345 | // |
346 | // To avoid generating a signing oracle, check the authenticated value |
347 | // before possibly re-signing it in the callee, as follows: |
348 | // |
349 | // <authenticate LR> |
350 | // <check if LR contains a valid address> |
351 | // b.<cond> break_block |
352 | // ret_block: |
353 | // TCRETURN |
354 | // break_block: |
355 | // brk <BrkOperand> |
356 | // |
357 | // or just |
358 | // |
359 | // <authenticate LR> |
360 | // ldr tmp, [lr] |
361 | // TCRETURN |
362 | |
363 | // TmpReg is chosen assuming X16 and X17 are dead after TI. |
364 | assert(AArch64InstrInfo::isTailCallReturnInst(*TI) && |
365 | "Tail call is expected" ); |
366 | Register TmpReg = |
367 | TI->readsRegister(AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16; |
368 | assert(!TI->readsRegister(TmpReg, TRI) && |
369 | "More than a single register is used by TCRETURN" ); |
370 | |
371 | checkAuthenticatedRegister(TI, Method, AArch64::LR, TmpReg, /*UseIKey=*/true, |
372 | BrkOperandForKey(KeyId)); |
373 | |
374 | return true; |
375 | } |
376 | |
377 | void AArch64PointerAuth::emitBlend(MachineBasicBlock::iterator MBBI, |
378 | Register Result, Register AddrDisc, |
379 | unsigned IntDisc) const { |
380 | MachineBasicBlock &MBB = *MBBI->getParent(); |
381 | DebugLoc DL = MBBI->getDebugLoc(); |
382 | |
383 | if (Result != AddrDisc) |
384 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), Result) |
385 | .addReg(AArch64::XZR) |
386 | .addReg(AddrDisc) |
387 | .addImm(0); |
388 | |
389 | BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), Result) |
390 | .addReg(Result) |
391 | .addImm(IntDisc) |
392 | .addImm(48); |
393 | } |
394 | |
395 | void AArch64PointerAuth::expandPAuthBlend( |
396 | MachineBasicBlock::iterator MBBI) const { |
397 | Register ResultReg = MBBI->getOperand(i: 0).getReg(); |
398 | Register AddrDisc = MBBI->getOperand(i: 1).getReg(); |
399 | unsigned IntDisc = MBBI->getOperand(i: 2).getImm(); |
400 | emitBlend(MBBI, Result: ResultReg, AddrDisc, IntDisc); |
401 | } |
402 | |
403 | bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) { |
404 | const auto *MFnI = MF.getInfo<AArch64FunctionInfo>(); |
405 | |
406 | Subtarget = &MF.getSubtarget<AArch64Subtarget>(); |
407 | TII = Subtarget->getInstrInfo(); |
408 | TRI = Subtarget->getRegisterInfo(); |
409 | |
410 | SmallVector<MachineBasicBlock::instr_iterator> PAuthPseudoInstrs; |
411 | SmallVector<MachineBasicBlock::instr_iterator> TailCallInstrs; |
412 | |
413 | bool Modified = false; |
414 | bool HasAuthenticationInstrs = false; |
415 | |
416 | for (auto &MBB : MF) { |
417 | // Using instr_iterator to catch unsupported bundled TCRETURN* instructions |
418 | // instead of just skipping them. |
419 | for (auto &MI : MBB.instrs()) { |
420 | switch (MI.getOpcode()) { |
421 | default: |
422 | // Bundled TCRETURN* instructions (such as created by KCFI) |
423 | // are not supported yet, but no support is required if no |
424 | // PAUTH_EPILOGUE instructions exist in the same function. |
425 | // Skip the BUNDLE instruction itself (actual bundled instructions |
426 | // follow it in the instruction list). |
427 | if (MI.isBundle()) |
428 | continue; |
429 | if (AArch64InstrInfo::isTailCallReturnInst(MI)) |
430 | TailCallInstrs.push_back(Elt: MI.getIterator()); |
431 | break; |
432 | case AArch64::PAUTH_PROLOGUE: |
433 | case AArch64::PAUTH_EPILOGUE: |
434 | case AArch64::PAUTH_BLEND: |
435 | assert(!MI.isBundled()); |
436 | PAuthPseudoInstrs.push_back(Elt: MI.getIterator()); |
437 | break; |
438 | } |
439 | } |
440 | } |
441 | |
442 | for (auto It : PAuthPseudoInstrs) { |
443 | switch (It->getOpcode()) { |
444 | case AArch64::PAUTH_PROLOGUE: |
445 | signLR(MF, MBBI: It); |
446 | break; |
447 | case AArch64::PAUTH_EPILOGUE: |
448 | authenticateLR(MF, MBBI: It); |
449 | HasAuthenticationInstrs = true; |
450 | break; |
451 | case AArch64::PAUTH_BLEND: |
452 | expandPAuthBlend(MBBI: It); |
453 | break; |
454 | default: |
455 | llvm_unreachable("Unhandled opcode" ); |
456 | } |
457 | It->eraseFromParent(); |
458 | Modified = true; |
459 | } |
460 | |
461 | // FIXME Do we need to emit any PAuth-related epilogue code at all |
462 | // when SCS is enabled? |
463 | if (HasAuthenticationInstrs && |
464 | !MFnI->needsShadowCallStackPrologueEpilogue(MF)) { |
465 | for (auto TailCall : TailCallInstrs) { |
466 | assert(!TailCall->isBundled() && "Not yet supported" ); |
467 | Modified |= checkAuthenticatedLR(TI: TailCall); |
468 | } |
469 | } |
470 | |
471 | return Modified; |
472 | } |
473 | |