1 | //===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Coverage instrumentation done on LLVM IR level, works with Sanitizers. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" |
14 | #include "llvm/ADT/ArrayRef.h" |
15 | #include "llvm/ADT/SmallVector.h" |
16 | #include "llvm/Analysis/GlobalsModRef.h" |
17 | #include "llvm/Analysis/PostDominators.h" |
18 | #include "llvm/IR/Constant.h" |
19 | #include "llvm/IR/DataLayout.h" |
20 | #include "llvm/IR/Dominators.h" |
21 | #include "llvm/IR/EHPersonalities.h" |
22 | #include "llvm/IR/Function.h" |
23 | #include "llvm/IR/GlobalVariable.h" |
24 | #include "llvm/IR/IRBuilder.h" |
25 | #include "llvm/IR/IntrinsicInst.h" |
26 | #include "llvm/IR/Intrinsics.h" |
27 | #include "llvm/IR/LLVMContext.h" |
28 | #include "llvm/IR/Module.h" |
29 | #include "llvm/IR/Type.h" |
30 | #include "llvm/Support/CommandLine.h" |
31 | #include "llvm/Support/SpecialCaseList.h" |
32 | #include "llvm/Support/VirtualFileSystem.h" |
33 | #include "llvm/TargetParser/Triple.h" |
34 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
35 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
36 | |
37 | using namespace llvm; |
38 | |
39 | #define DEBUG_TYPE "sancov" |
40 | |
41 | const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir" ; |
42 | const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc" ; |
43 | const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1" ; |
44 | const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2" ; |
45 | const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4" ; |
46 | const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8" ; |
47 | const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1" ; |
48 | const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2" ; |
49 | const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4" ; |
50 | const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8" ; |
51 | const char SanCovLoad1[] = "__sanitizer_cov_load1" ; |
52 | const char SanCovLoad2[] = "__sanitizer_cov_load2" ; |
53 | const char SanCovLoad4[] = "__sanitizer_cov_load4" ; |
54 | const char SanCovLoad8[] = "__sanitizer_cov_load8" ; |
55 | const char SanCovLoad16[] = "__sanitizer_cov_load16" ; |
56 | const char SanCovStore1[] = "__sanitizer_cov_store1" ; |
57 | const char SanCovStore2[] = "__sanitizer_cov_store2" ; |
58 | const char SanCovStore4[] = "__sanitizer_cov_store4" ; |
59 | const char SanCovStore8[] = "__sanitizer_cov_store8" ; |
60 | const char SanCovStore16[] = "__sanitizer_cov_store16" ; |
61 | const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4" ; |
62 | const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8" ; |
63 | const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep" ; |
64 | const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch" ; |
65 | const char SanCovModuleCtorTracePcGuardName[] = |
66 | "sancov.module_ctor_trace_pc_guard" ; |
67 | const char SanCovModuleCtor8bitCountersName[] = |
68 | "sancov.module_ctor_8bit_counters" ; |
69 | const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag" ; |
70 | static const uint64_t SanCtorAndDtorPriority = 2; |
71 | |
72 | const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard" ; |
73 | const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init" ; |
74 | const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init" ; |
75 | const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init" ; |
76 | const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init" ; |
77 | const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init" ; |
78 | |
79 | const char SanCovGuardsSectionName[] = "sancov_guards" ; |
80 | const char [] = "sancov_cntrs" ; |
81 | const char SanCovBoolFlagSectionName[] = "sancov_bools" ; |
82 | const char SanCovPCsSectionName[] = "sancov_pcs" ; |
83 | const char SanCovCFsSectionName[] = "sancov_cfs" ; |
84 | |
85 | const char SanCovLowestStackName[] = "__sancov_lowest_stack" ; |
86 | |
87 | static cl::opt<int> ClCoverageLevel( |
88 | "sanitizer-coverage-level" , |
89 | cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, " |
90 | "3: all blocks and critical edges" ), |
91 | cl::Hidden, cl::init(Val: 0)); |
92 | |
93 | static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc" , |
94 | cl::desc("Experimental pc tracing" ), cl::Hidden, |
95 | cl::init(Val: false)); |
96 | |
97 | static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard" , |
98 | cl::desc("pc tracing with a guard" ), |
99 | cl::Hidden, cl::init(Val: false)); |
100 | |
101 | // If true, we create a global variable that contains PCs of all instrumented |
102 | // BBs, put this global into a named section, and pass this section's bounds |
103 | // to __sanitizer_cov_pcs_init. |
104 | // This way the coverage instrumentation does not need to acquire the PCs |
105 | // at run-time. Works with trace-pc-guard, inline-8bit-counters, and |
106 | // inline-bool-flag. |
107 | static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table" , |
108 | cl::desc("create a static PC table" ), |
109 | cl::Hidden, cl::init(Val: false)); |
110 | |
111 | static cl::opt<bool> |
112 | ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters" , |
113 | cl::desc("increments 8-bit counter for every edge" ), |
114 | cl::Hidden, cl::init(Val: false)); |
115 | |
116 | static cl::opt<bool> |
117 | ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag" , |
118 | cl::desc("sets a boolean flag for every edge" ), cl::Hidden, |
119 | cl::init(Val: false)); |
120 | |
121 | static cl::opt<bool> |
122 | ClCMPTracing("sanitizer-coverage-trace-compares" , |
123 | cl::desc("Tracing of CMP and similar instructions" ), |
124 | cl::Hidden, cl::init(Val: false)); |
125 | |
126 | static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs" , |
127 | cl::desc("Tracing of DIV instructions" ), |
128 | cl::Hidden, cl::init(Val: false)); |
129 | |
130 | static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads" , |
131 | cl::desc("Tracing of load instructions" ), |
132 | cl::Hidden, cl::init(Val: false)); |
133 | |
134 | static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores" , |
135 | cl::desc("Tracing of store instructions" ), |
136 | cl::Hidden, cl::init(Val: false)); |
137 | |
138 | static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps" , |
139 | cl::desc("Tracing of GEP instructions" ), |
140 | cl::Hidden, cl::init(Val: false)); |
141 | |
142 | static cl::opt<bool> |
143 | ClPruneBlocks("sanitizer-coverage-prune-blocks" , |
144 | cl::desc("Reduce the number of instrumented blocks" ), |
145 | cl::Hidden, cl::init(Val: true)); |
146 | |
147 | static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth" , |
148 | cl::desc("max stack depth tracing" ), |
149 | cl::Hidden, cl::init(Val: false)); |
150 | |
151 | static cl::opt<bool> |
152 | ClCollectCF("sanitizer-coverage-control-flow" , |
153 | cl::desc("collect control flow for each function" ), cl::Hidden, |
154 | cl::init(Val: false)); |
155 | |
156 | namespace { |
157 | |
158 | SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) { |
159 | SanitizerCoverageOptions Res; |
160 | switch (LegacyCoverageLevel) { |
161 | case 0: |
162 | Res.CoverageType = SanitizerCoverageOptions::SCK_None; |
163 | break; |
164 | case 1: |
165 | Res.CoverageType = SanitizerCoverageOptions::SCK_Function; |
166 | break; |
167 | case 2: |
168 | Res.CoverageType = SanitizerCoverageOptions::SCK_BB; |
169 | break; |
170 | case 3: |
171 | Res.CoverageType = SanitizerCoverageOptions::SCK_Edge; |
172 | break; |
173 | case 4: |
174 | Res.CoverageType = SanitizerCoverageOptions::SCK_Edge; |
175 | Res.IndirectCalls = true; |
176 | break; |
177 | } |
178 | return Res; |
179 | } |
180 | |
181 | SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) { |
182 | // Sets CoverageType and IndirectCalls. |
183 | SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: ClCoverageLevel); |
184 | Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType); |
185 | Options.IndirectCalls |= CLOpts.IndirectCalls; |
186 | Options.TraceCmp |= ClCMPTracing; |
187 | Options.TraceDiv |= ClDIVTracing; |
188 | Options.TraceGep |= ClGEPTracing; |
189 | Options.TracePC |= ClTracePC; |
190 | Options.TracePCGuard |= ClTracePCGuard; |
191 | Options.Inline8bitCounters |= ClInline8bitCounters; |
192 | Options.InlineBoolFlag |= ClInlineBoolFlag; |
193 | Options.PCTable |= ClCreatePCTable; |
194 | Options.NoPrune |= !ClPruneBlocks; |
195 | Options.StackDepth |= ClStackDepth; |
196 | Options.TraceLoads |= ClLoadTracing; |
197 | Options.TraceStores |= ClStoreTracing; |
198 | if (!Options.TracePCGuard && !Options.TracePC && |
199 | !Options.Inline8bitCounters && !Options.StackDepth && |
200 | !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores) |
201 | Options.TracePCGuard = true; // TracePCGuard is default. |
202 | Options.CollectControlFlow |= ClCollectCF; |
203 | return Options; |
204 | } |
205 | |
206 | using DomTreeCallback = function_ref<const DominatorTree *(Function &F)>; |
207 | using PostDomTreeCallback = |
208 | function_ref<const PostDominatorTree *(Function &F)>; |
209 | |
210 | class ModuleSanitizerCoverage { |
211 | public: |
212 | ModuleSanitizerCoverage( |
213 | const SanitizerCoverageOptions &Options = SanitizerCoverageOptions(), |
214 | const SpecialCaseList *Allowlist = nullptr, |
215 | const SpecialCaseList *Blocklist = nullptr) |
216 | : Options(OverrideFromCL(Options)), Allowlist(Allowlist), |
217 | Blocklist(Blocklist) {} |
218 | bool instrumentModule(Module &M, DomTreeCallback DTCallback, |
219 | PostDomTreeCallback PDTCallback); |
220 | |
221 | private: |
222 | void createFunctionControlFlow(Function &F); |
223 | void instrumentFunction(Function &F, DomTreeCallback DTCallback, |
224 | PostDomTreeCallback PDTCallback); |
225 | void InjectCoverageForIndirectCalls(Function &F, |
226 | ArrayRef<Instruction *> IndirCalls); |
227 | void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets); |
228 | void InjectTraceForDiv(Function &F, |
229 | ArrayRef<BinaryOperator *> DivTraceTargets); |
230 | void InjectTraceForGep(Function &F, |
231 | ArrayRef<GetElementPtrInst *> GepTraceTargets); |
232 | void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads, |
233 | ArrayRef<StoreInst *> Stores); |
234 | void InjectTraceForSwitch(Function &F, |
235 | ArrayRef<Instruction *> SwitchTraceTargets); |
236 | bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks, |
237 | bool IsLeafFunc = true); |
238 | GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements, |
239 | Function &F, Type *Ty, |
240 | const char *Section); |
241 | GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks); |
242 | void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks); |
243 | void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx, |
244 | bool IsLeafFunc = true); |
245 | Function *CreateInitCallsForSections(Module &M, const char *CtorName, |
246 | const char *InitFunctionName, Type *Ty, |
247 | const char *Section); |
248 | std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section, |
249 | Type *Ty); |
250 | |
251 | std::string getSectionName(const std::string &Section) const; |
252 | std::string getSectionStart(const std::string &Section) const; |
253 | std::string getSectionEnd(const std::string &Section) const; |
254 | FunctionCallee SanCovTracePCIndir; |
255 | FunctionCallee SanCovTracePC, SanCovTracePCGuard; |
256 | std::array<FunctionCallee, 4> SanCovTraceCmpFunction; |
257 | std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction; |
258 | std::array<FunctionCallee, 5> SanCovLoadFunction; |
259 | std::array<FunctionCallee, 5> SanCovStoreFunction; |
260 | std::array<FunctionCallee, 2> SanCovTraceDivFunction; |
261 | FunctionCallee SanCovTraceGepFunction; |
262 | FunctionCallee SanCovTraceSwitchFunction; |
263 | GlobalVariable *SanCovLowestStack; |
264 | Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty; |
265 | Module *CurModule; |
266 | std::string CurModuleUniqueId; |
267 | Triple TargetTriple; |
268 | LLVMContext *C; |
269 | const DataLayout *DL; |
270 | |
271 | GlobalVariable *FunctionGuardArray; // for trace-pc-guard. |
272 | GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters. |
273 | GlobalVariable *FunctionBoolArray; // for inline-bool-flag. |
274 | GlobalVariable *FunctionPCsArray; // for pc-table. |
275 | GlobalVariable *FunctionCFsArray; // for control flow table |
276 | SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed; |
277 | SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed; |
278 | |
279 | SanitizerCoverageOptions Options; |
280 | |
281 | const SpecialCaseList *Allowlist; |
282 | const SpecialCaseList *Blocklist; |
283 | }; |
284 | } // namespace |
285 | |
286 | PreservedAnalyses SanitizerCoveragePass::run(Module &M, |
287 | ModuleAnalysisManager &MAM) { |
288 | ModuleSanitizerCoverage ModuleSancov(Options, Allowlist.get(), |
289 | Blocklist.get()); |
290 | auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager(); |
291 | auto DTCallback = [&FAM](Function &F) -> const DominatorTree * { |
292 | return &FAM.getResult<DominatorTreeAnalysis>(IR&: F); |
293 | }; |
294 | auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree * { |
295 | return &FAM.getResult<PostDominatorTreeAnalysis>(IR&: F); |
296 | }; |
297 | if (!ModuleSancov.instrumentModule(M, DTCallback, PDTCallback)) |
298 | return PreservedAnalyses::all(); |
299 | |
300 | PreservedAnalyses PA = PreservedAnalyses::none(); |
301 | // GlobalsAA is considered stateless and does not get invalidated unless |
302 | // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers |
303 | // make changes that require GlobalsAA to be invalidated. |
304 | PA.abandon<GlobalsAA>(); |
305 | return PA; |
306 | } |
307 | |
308 | std::pair<Value *, Value *> |
309 | ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section, |
310 | Type *Ty) { |
311 | // Use ExternalWeak so that if all sections are discarded due to section |
312 | // garbage collection, the linker will not report undefined symbol errors. |
313 | // Windows defines the start/stop symbols in compiler-rt so no need for |
314 | // ExternalWeak. |
315 | GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF() |
316 | ? GlobalVariable::ExternalLinkage |
317 | : GlobalVariable::ExternalWeakLinkage; |
318 | GlobalVariable *SecStart = |
319 | new GlobalVariable(M, Ty, false, Linkage, nullptr, |
320 | getSectionStart(Section)); |
321 | SecStart->setVisibility(GlobalValue::HiddenVisibility); |
322 | GlobalVariable *SecEnd = |
323 | new GlobalVariable(M, Ty, false, Linkage, nullptr, |
324 | getSectionEnd(Section)); |
325 | SecEnd->setVisibility(GlobalValue::HiddenVisibility); |
326 | IRBuilder<> IRB(M.getContext()); |
327 | if (!TargetTriple.isOSBinFormatCOFF()) |
328 | return std::make_pair(x&: SecStart, y&: SecEnd); |
329 | |
330 | // Account for the fact that on windows-msvc __start_* symbols actually |
331 | // point to a uint64_t before the start of the array. |
332 | auto GEP = |
333 | IRB.CreatePtrAdd(Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t))); |
334 | return std::make_pair(x&: GEP, y&: SecEnd); |
335 | } |
336 | |
337 | Function *ModuleSanitizerCoverage::CreateInitCallsForSections( |
338 | Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty, |
339 | const char *Section) { |
340 | auto SecStartEnd = CreateSecStartEnd(M, Section, Ty); |
341 | auto SecStart = SecStartEnd.first; |
342 | auto SecEnd = SecStartEnd.second; |
343 | Function *CtorFunc; |
344 | std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions( |
345 | M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd}); |
346 | assert(CtorFunc->getName() == CtorName); |
347 | |
348 | if (TargetTriple.supportsCOMDAT()) { |
349 | // Use comdat to dedup CtorFunc. |
350 | CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName)); |
351 | appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc); |
352 | } else { |
353 | appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority); |
354 | } |
355 | |
356 | if (TargetTriple.isOSBinFormatCOFF()) { |
357 | // In COFF files, if the contructors are set as COMDAT (they are because |
358 | // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced |
359 | // functions and data) is used, the constructors get stripped. To prevent |
360 | // this, give the constructors weak ODR linkage and ensure the linker knows |
361 | // to include the sancov constructor. This way the linker can deduplicate |
362 | // the constructors but always leave one copy. |
363 | CtorFunc->setLinkage(GlobalValue::WeakODRLinkage); |
364 | } |
365 | return CtorFunc; |
366 | } |
367 | |
368 | bool ModuleSanitizerCoverage::instrumentModule( |
369 | Module &M, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { |
370 | if (Options.CoverageType == SanitizerCoverageOptions::SCK_None) |
371 | return false; |
372 | if (Allowlist && |
373 | !Allowlist->inSection(Section: "coverage" , Prefix: "src" , Query: M.getSourceFileName())) |
374 | return false; |
375 | if (Blocklist && |
376 | Blocklist->inSection(Section: "coverage" , Prefix: "src" , Query: M.getSourceFileName())) |
377 | return false; |
378 | C = &(M.getContext()); |
379 | DL = &M.getDataLayout(); |
380 | CurModule = &M; |
381 | CurModuleUniqueId = getUniqueModuleId(M: CurModule); |
382 | TargetTriple = Triple(M.getTargetTriple()); |
383 | FunctionGuardArray = nullptr; |
384 | Function8bitCounterArray = nullptr; |
385 | FunctionBoolArray = nullptr; |
386 | FunctionPCsArray = nullptr; |
387 | FunctionCFsArray = nullptr; |
388 | IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits()); |
389 | PtrTy = PointerType::getUnqual(C&: *C); |
390 | Type *VoidTy = Type::getVoidTy(C&: *C); |
391 | IRBuilder<> IRB(*C); |
392 | Int64Ty = IRB.getInt64Ty(); |
393 | Int32Ty = IRB.getInt32Ty(); |
394 | Int16Ty = IRB.getInt16Ty(); |
395 | Int8Ty = IRB.getInt8Ty(); |
396 | Int1Ty = IRB.getInt1Ty(); |
397 | |
398 | SanCovTracePCIndir = |
399 | M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy); |
400 | // Make sure smaller parameters are zero-extended to i64 if required by the |
401 | // target ABI. |
402 | AttributeList SanCovTraceCmpZeroExtAL; |
403 | SanCovTraceCmpZeroExtAL = |
404 | SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt); |
405 | SanCovTraceCmpZeroExtAL = |
406 | SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt); |
407 | |
408 | SanCovTraceCmpFunction[0] = |
409 | M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, |
410 | Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty()); |
411 | SanCovTraceCmpFunction[1] = |
412 | M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, |
413 | Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty()); |
414 | SanCovTraceCmpFunction[2] = |
415 | M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, |
416 | Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty()); |
417 | SanCovTraceCmpFunction[3] = |
418 | M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty); |
419 | |
420 | SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction( |
421 | Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty); |
422 | SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction( |
423 | Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty); |
424 | SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction( |
425 | Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty); |
426 | SanCovTraceConstCmpFunction[3] = |
427 | M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty); |
428 | |
429 | // Loads. |
430 | SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy); |
431 | SanCovLoadFunction[1] = |
432 | M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy); |
433 | SanCovLoadFunction[2] = |
434 | M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy); |
435 | SanCovLoadFunction[3] = |
436 | M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy); |
437 | SanCovLoadFunction[4] = |
438 | M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy); |
439 | // Stores. |
440 | SanCovStoreFunction[0] = |
441 | M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy); |
442 | SanCovStoreFunction[1] = |
443 | M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy); |
444 | SanCovStoreFunction[2] = |
445 | M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy); |
446 | SanCovStoreFunction[3] = |
447 | M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy); |
448 | SanCovStoreFunction[4] = |
449 | M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy); |
450 | |
451 | { |
452 | AttributeList AL; |
453 | AL = AL.addParamAttribute(*C, 0, Attribute::ZExt); |
454 | SanCovTraceDivFunction[0] = |
455 | M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty()); |
456 | } |
457 | SanCovTraceDivFunction[1] = |
458 | M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty); |
459 | SanCovTraceGepFunction = |
460 | M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy); |
461 | SanCovTraceSwitchFunction = |
462 | M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy); |
463 | |
464 | Constant *SanCovLowestStackConstant = |
465 | M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy); |
466 | SanCovLowestStack = dyn_cast<GlobalVariable>(Val: SanCovLowestStackConstant); |
467 | if (!SanCovLowestStack || SanCovLowestStack->getValueType() != IntptrTy) { |
468 | C->emitError(ErrorStr: StringRef("'" ) + SanCovLowestStackName + |
469 | "' should not be declared by the user" ); |
470 | return true; |
471 | } |
472 | SanCovLowestStack->setThreadLocalMode( |
473 | GlobalValue::ThreadLocalMode::InitialExecTLSModel); |
474 | if (Options.StackDepth && !SanCovLowestStack->isDeclaration()) |
475 | SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy)); |
476 | |
477 | SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy); |
478 | SanCovTracePCGuard = |
479 | M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy); |
480 | |
481 | for (auto &F : M) |
482 | instrumentFunction(F, DTCallback, PDTCallback); |
483 | |
484 | Function *Ctor = nullptr; |
485 | |
486 | if (FunctionGuardArray) |
487 | Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName, |
488 | InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty, |
489 | Section: SanCovGuardsSectionName); |
490 | if (Function8bitCounterArray) |
491 | Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName, |
492 | InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty, |
493 | Section: SanCovCountersSectionName); |
494 | if (FunctionBoolArray) { |
495 | Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName, |
496 | InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty, |
497 | Section: SanCovBoolFlagSectionName); |
498 | } |
499 | if (Ctor && Options.PCTable) { |
500 | auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy); |
501 | FunctionCallee InitFunction = declareSanitizerInitFunction( |
502 | M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy}); |
503 | IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator()); |
504 | IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second}); |
505 | } |
506 | |
507 | if (Ctor && Options.CollectControlFlow) { |
508 | auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy); |
509 | FunctionCallee InitFunction = declareSanitizerInitFunction( |
510 | M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy}); |
511 | IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator()); |
512 | IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second}); |
513 | } |
514 | |
515 | appendToUsed(M, Values: GlobalsToAppendToUsed); |
516 | appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed); |
517 | return true; |
518 | } |
519 | |
520 | // True if block has successors and it dominates all of them. |
521 | static bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) { |
522 | if (succ_empty(BB)) |
523 | return false; |
524 | |
525 | return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) { |
526 | return DT->dominates(A: BB, B: SUCC); |
527 | }); |
528 | } |
529 | |
530 | // True if block has predecessors and it postdominates all of them. |
531 | static bool isFullPostDominator(const BasicBlock *BB, |
532 | const PostDominatorTree *PDT) { |
533 | if (pred_empty(BB)) |
534 | return false; |
535 | |
536 | return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) { |
537 | return PDT->dominates(A: BB, B: PRED); |
538 | }); |
539 | } |
540 | |
541 | static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, |
542 | const DominatorTree *DT, |
543 | const PostDominatorTree *PDT, |
544 | const SanitizerCoverageOptions &Options) { |
545 | // Don't insert coverage for blocks containing nothing but unreachable: we |
546 | // will never call __sanitizer_cov() for them, so counting them in |
547 | // NumberOfInstrumentedBlocks() might complicate calculation of code coverage |
548 | // percentage. Also, unreachable instructions frequently have no debug |
549 | // locations. |
550 | if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrDbgOrLifetime())) |
551 | return false; |
552 | |
553 | // Don't insert coverage into blocks without a valid insertion point |
554 | // (catchswitch blocks). |
555 | if (BB->getFirstInsertionPt() == BB->end()) |
556 | return false; |
557 | |
558 | if (Options.NoPrune || &F.getEntryBlock() == BB) |
559 | return true; |
560 | |
561 | if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function && |
562 | &F.getEntryBlock() != BB) |
563 | return false; |
564 | |
565 | // Do not instrument full dominators, or full post-dominators with multiple |
566 | // predecessors. |
567 | return !isFullDominator(BB, DT) |
568 | && !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor()); |
569 | } |
570 | |
571 | |
572 | // Returns true iff From->To is a backedge. |
573 | // A twist here is that we treat From->To as a backedge if |
574 | // * To dominates From or |
575 | // * To->UniqueSuccessor dominates From |
576 | static bool IsBackEdge(BasicBlock *From, BasicBlock *To, |
577 | const DominatorTree *DT) { |
578 | if (DT->dominates(A: To, B: From)) |
579 | return true; |
580 | if (auto Next = To->getUniqueSuccessor()) |
581 | if (DT->dominates(A: Next, B: From)) |
582 | return true; |
583 | return false; |
584 | } |
585 | |
586 | // Prunes uninteresting Cmp instrumentation: |
587 | // * CMP instructions that feed into loop backedge branch. |
588 | // |
589 | // Note that Cmp pruning is controlled by the same flag as the |
590 | // BB pruning. |
591 | static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT, |
592 | const SanitizerCoverageOptions &Options) { |
593 | if (!Options.NoPrune) |
594 | if (CMP->hasOneUse()) |
595 | if (auto BR = dyn_cast<BranchInst>(Val: CMP->user_back())) |
596 | for (BasicBlock *B : BR->successors()) |
597 | if (IsBackEdge(From: BR->getParent(), To: B, DT)) |
598 | return false; |
599 | return true; |
600 | } |
601 | |
602 | void ModuleSanitizerCoverage::instrumentFunction( |
603 | Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { |
604 | if (F.empty()) |
605 | return; |
606 | if (F.getName().contains(Other: ".module_ctor" )) |
607 | return; // Should not instrument sanitizer init functions. |
608 | if (F.getName().starts_with(Prefix: "__sanitizer_" )) |
609 | return; // Don't instrument __sanitizer_* callbacks. |
610 | // Don't touch available_externally functions, their actual body is elewhere. |
611 | if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) |
612 | return; |
613 | // Don't instrument MSVC CRT configuration helpers. They may run before normal |
614 | // initialization. |
615 | if (F.getName() == "__local_stdio_printf_options" || |
616 | F.getName() == "__local_stdio_scanf_options" ) |
617 | return; |
618 | if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator())) |
619 | return; |
620 | // Don't instrument functions using SEH for now. Splitting basic blocks like |
621 | // we do for coverage breaks WinEHPrepare. |
622 | // FIXME: Remove this when SEH no longer uses landingpad pattern matching. |
623 | if (F.hasPersonalityFn() && |
624 | isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn()))) |
625 | return; |
626 | if (Allowlist && !Allowlist->inSection(Section: "coverage" , Prefix: "fun" , Query: F.getName())) |
627 | return; |
628 | if (Blocklist && Blocklist->inSection(Section: "coverage" , Prefix: "fun" , Query: F.getName())) |
629 | return; |
630 | if (F.hasFnAttribute(Attribute::NoSanitizeCoverage)) |
631 | return; |
632 | if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) |
633 | SplitAllCriticalEdges(F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests()); |
634 | SmallVector<Instruction *, 8> IndirCalls; |
635 | SmallVector<BasicBlock *, 16> BlocksToInstrument; |
636 | SmallVector<Instruction *, 8> CmpTraceTargets; |
637 | SmallVector<Instruction *, 8> SwitchTraceTargets; |
638 | SmallVector<BinaryOperator *, 8> DivTraceTargets; |
639 | SmallVector<GetElementPtrInst *, 8> GepTraceTargets; |
640 | SmallVector<LoadInst *, 8> Loads; |
641 | SmallVector<StoreInst *, 8> Stores; |
642 | |
643 | const DominatorTree *DT = DTCallback(F); |
644 | const PostDominatorTree *PDT = PDTCallback(F); |
645 | bool IsLeafFunc = true; |
646 | |
647 | for (auto &BB : F) { |
648 | if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options)) |
649 | BlocksToInstrument.push_back(Elt: &BB); |
650 | for (auto &Inst : BB) { |
651 | if (Options.IndirectCalls) { |
652 | CallBase *CB = dyn_cast<CallBase>(Val: &Inst); |
653 | if (CB && CB->isIndirectCall()) |
654 | IndirCalls.push_back(Elt: &Inst); |
655 | } |
656 | if (Options.TraceCmp) { |
657 | if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst)) |
658 | if (IsInterestingCmp(CMP, DT, Options)) |
659 | CmpTraceTargets.push_back(Elt: &Inst); |
660 | if (isa<SwitchInst>(Val: &Inst)) |
661 | SwitchTraceTargets.push_back(Elt: &Inst); |
662 | } |
663 | if (Options.TraceDiv) |
664 | if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst)) |
665 | if (BO->getOpcode() == Instruction::SDiv || |
666 | BO->getOpcode() == Instruction::UDiv) |
667 | DivTraceTargets.push_back(Elt: BO); |
668 | if (Options.TraceGep) |
669 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst)) |
670 | GepTraceTargets.push_back(Elt: GEP); |
671 | if (Options.TraceLoads) |
672 | if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst)) |
673 | Loads.push_back(Elt: LI); |
674 | if (Options.TraceStores) |
675 | if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst)) |
676 | Stores.push_back(Elt: SI); |
677 | if (Options.StackDepth) |
678 | if (isa<InvokeInst>(Val: Inst) || |
679 | (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst))) |
680 | IsLeafFunc = false; |
681 | } |
682 | } |
683 | |
684 | if (Options.CollectControlFlow) |
685 | createFunctionControlFlow(F); |
686 | |
687 | InjectCoverage(F, AllBlocks: BlocksToInstrument, IsLeafFunc); |
688 | InjectCoverageForIndirectCalls(F, IndirCalls); |
689 | InjectTraceForCmp(F, CmpTraceTargets); |
690 | InjectTraceForSwitch(F, SwitchTraceTargets); |
691 | InjectTraceForDiv(F, DivTraceTargets); |
692 | InjectTraceForGep(F, GepTraceTargets); |
693 | InjectTraceForLoadsAndStores(F, Loads, Stores); |
694 | } |
695 | |
696 | GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( |
697 | size_t NumElements, Function &F, Type *Ty, const char *Section) { |
698 | ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements); |
699 | auto Array = new GlobalVariable( |
700 | *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, |
701 | Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_" ); |
702 | |
703 | if (TargetTriple.supportsCOMDAT() && |
704 | (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) |
705 | if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple)) |
706 | Array->setComdat(Comdat); |
707 | Array->setSection(getSectionName(Section)); |
708 | Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue())); |
709 | |
710 | // sancov_pcs parallels the other metadata section(s). Optimizers (e.g. |
711 | // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other |
712 | // section(s) as a unit, so we conservatively retain all unconditionally in |
713 | // the compiler. |
714 | // |
715 | // With comdat (COFF/ELF), the linker can guarantee the associated sections |
716 | // will be retained or discarded as a unit, so llvm.compiler.used is |
717 | // sufficient. Otherwise, conservatively make all of them retained by the |
718 | // linker. |
719 | if (Array->hasComdat()) |
720 | GlobalsToAppendToCompilerUsed.push_back(Elt: Array); |
721 | else |
722 | GlobalsToAppendToUsed.push_back(Elt: Array); |
723 | |
724 | return Array; |
725 | } |
726 | |
727 | GlobalVariable * |
728 | ModuleSanitizerCoverage::CreatePCArray(Function &F, |
729 | ArrayRef<BasicBlock *> AllBlocks) { |
730 | size_t N = AllBlocks.size(); |
731 | assert(N); |
732 | SmallVector<Constant *, 32> PCs; |
733 | IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt()); |
734 | for (size_t i = 0; i < N; i++) { |
735 | if (&F.getEntryBlock() == AllBlocks[i]) { |
736 | PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy)); |
737 | PCs.push_back(Elt: (Constant *)IRB.CreateIntToPtr( |
738 | V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy)); |
739 | } else { |
740 | PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast( |
741 | V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy)); |
742 | PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy)); |
743 | } |
744 | } |
745 | auto *PCArray = CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy, |
746 | Section: SanCovPCsSectionName); |
747 | PCArray->setInitializer( |
748 | ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs)); |
749 | PCArray->setConstant(true); |
750 | |
751 | return PCArray; |
752 | } |
753 | |
754 | void ModuleSanitizerCoverage::CreateFunctionLocalArrays( |
755 | Function &F, ArrayRef<BasicBlock *> AllBlocks) { |
756 | if (Options.TracePCGuard) |
757 | FunctionGuardArray = CreateFunctionLocalArrayInSection( |
758 | NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName); |
759 | |
760 | if (Options.Inline8bitCounters) |
761 | Function8bitCounterArray = CreateFunctionLocalArrayInSection( |
762 | NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName); |
763 | if (Options.InlineBoolFlag) |
764 | FunctionBoolArray = CreateFunctionLocalArrayInSection( |
765 | NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName); |
766 | |
767 | if (Options.PCTable) |
768 | FunctionPCsArray = CreatePCArray(F, AllBlocks); |
769 | } |
770 | |
771 | bool ModuleSanitizerCoverage::InjectCoverage(Function &F, |
772 | ArrayRef<BasicBlock *> AllBlocks, |
773 | bool IsLeafFunc) { |
774 | if (AllBlocks.empty()) return false; |
775 | CreateFunctionLocalArrays(F, AllBlocks); |
776 | for (size_t i = 0, N = AllBlocks.size(); i < N; i++) |
777 | InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, IsLeafFunc); |
778 | return true; |
779 | } |
780 | |
781 | // On every indirect call we call a run-time function |
782 | // __sanitizer_cov_indir_call* with two parameters: |
783 | // - callee address, |
784 | // - global cache array that contains CacheSize pointers (zero-initialized). |
785 | // The cache is used to speed up recording the caller-callee pairs. |
786 | // The address of the caller is passed implicitly via caller PC. |
787 | // CacheSize is encoded in the name of the run-time function. |
788 | void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls( |
789 | Function &F, ArrayRef<Instruction *> IndirCalls) { |
790 | if (IndirCalls.empty()) |
791 | return; |
792 | assert(Options.TracePC || Options.TracePCGuard || |
793 | Options.Inline8bitCounters || Options.InlineBoolFlag); |
794 | for (auto *I : IndirCalls) { |
795 | InstrumentationIRBuilder IRB(I); |
796 | CallBase &CB = cast<CallBase>(Val&: *I); |
797 | Value *Callee = CB.getCalledOperand(); |
798 | if (isa<InlineAsm>(Val: Callee)) |
799 | continue; |
800 | IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy)); |
801 | } |
802 | } |
803 | |
804 | // For every switch statement we insert a call: |
805 | // __sanitizer_cov_trace_switch(CondValue, |
806 | // {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... }) |
807 | |
808 | void ModuleSanitizerCoverage::InjectTraceForSwitch( |
809 | Function &, ArrayRef<Instruction *> SwitchTraceTargets) { |
810 | for (auto *I : SwitchTraceTargets) { |
811 | if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) { |
812 | InstrumentationIRBuilder IRB(I); |
813 | SmallVector<Constant *, 16> Initializers; |
814 | Value *Cond = SI->getCondition(); |
815 | if (Cond->getType()->getScalarSizeInBits() > |
816 | Int64Ty->getScalarSizeInBits()) |
817 | continue; |
818 | Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases())); |
819 | Initializers.push_back( |
820 | Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits())); |
821 | if (Cond->getType()->getScalarSizeInBits() < |
822 | Int64Ty->getScalarSizeInBits()) |
823 | Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false); |
824 | for (auto It : SI->cases()) { |
825 | ConstantInt *C = It.getCaseValue(); |
826 | if (C->getType()->getScalarSizeInBits() < 64) |
827 | C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64)); |
828 | Initializers.push_back(Elt: C); |
829 | } |
830 | llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2), |
831 | Comp: [](const Constant *A, const Constant *B) { |
832 | return cast<ConstantInt>(Val: A)->getLimitedValue() < |
833 | cast<ConstantInt>(Val: B)->getLimitedValue(); |
834 | }); |
835 | ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size()); |
836 | GlobalVariable *GV = new GlobalVariable( |
837 | *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage, |
838 | ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers), |
839 | "__sancov_gen_cov_switch_values" ); |
840 | IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV}); |
841 | } |
842 | } |
843 | } |
844 | |
845 | void ModuleSanitizerCoverage::InjectTraceForDiv( |
846 | Function &, ArrayRef<BinaryOperator *> DivTraceTargets) { |
847 | for (auto *BO : DivTraceTargets) { |
848 | InstrumentationIRBuilder IRB(BO); |
849 | Value *A1 = BO->getOperand(i_nocapture: 1); |
850 | if (isa<ConstantInt>(Val: A1)) continue; |
851 | if (!A1->getType()->isIntegerTy()) |
852 | continue; |
853 | uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType()); |
854 | int CallbackIdx = TypeSize == 32 ? 0 : |
855 | TypeSize == 64 ? 1 : -1; |
856 | if (CallbackIdx < 0) continue; |
857 | auto Ty = Type::getIntNTy(C&: *C, N: TypeSize); |
858 | IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx], |
859 | Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)}); |
860 | } |
861 | } |
862 | |
863 | void ModuleSanitizerCoverage::InjectTraceForGep( |
864 | Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) { |
865 | for (auto *GEP : GepTraceTargets) { |
866 | InstrumentationIRBuilder IRB(GEP); |
867 | for (Use &Idx : GEP->indices()) |
868 | if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy()) |
869 | IRB.CreateCall(Callee: SanCovTraceGepFunction, |
870 | Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)}); |
871 | } |
872 | } |
873 | |
874 | void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores( |
875 | Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) { |
876 | auto CallbackIdx = [&](Type *ElementTy) -> int { |
877 | uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy); |
878 | return TypeSize == 8 ? 0 |
879 | : TypeSize == 16 ? 1 |
880 | : TypeSize == 32 ? 2 |
881 | : TypeSize == 64 ? 3 |
882 | : TypeSize == 128 ? 4 |
883 | : -1; |
884 | }; |
885 | for (auto *LI : Loads) { |
886 | InstrumentationIRBuilder IRB(LI); |
887 | auto Ptr = LI->getPointerOperand(); |
888 | int Idx = CallbackIdx(LI->getType()); |
889 | if (Idx < 0) |
890 | continue; |
891 | IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr); |
892 | } |
893 | for (auto *SI : Stores) { |
894 | InstrumentationIRBuilder IRB(SI); |
895 | auto Ptr = SI->getPointerOperand(); |
896 | int Idx = CallbackIdx(SI->getValueOperand()->getType()); |
897 | if (Idx < 0) |
898 | continue; |
899 | IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr); |
900 | } |
901 | } |
902 | |
903 | void ModuleSanitizerCoverage::InjectTraceForCmp( |
904 | Function &, ArrayRef<Instruction *> CmpTraceTargets) { |
905 | for (auto *I : CmpTraceTargets) { |
906 | if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) { |
907 | InstrumentationIRBuilder IRB(ICMP); |
908 | Value *A0 = ICMP->getOperand(i_nocapture: 0); |
909 | Value *A1 = ICMP->getOperand(i_nocapture: 1); |
910 | if (!A0->getType()->isIntegerTy()) |
911 | continue; |
912 | uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType()); |
913 | int CallbackIdx = TypeSize == 8 ? 0 : |
914 | TypeSize == 16 ? 1 : |
915 | TypeSize == 32 ? 2 : |
916 | TypeSize == 64 ? 3 : -1; |
917 | if (CallbackIdx < 0) continue; |
918 | // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1); |
919 | auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx]; |
920 | bool FirstIsConst = isa<ConstantInt>(Val: A0); |
921 | bool SecondIsConst = isa<ConstantInt>(Val: A1); |
922 | // If both are const, then we don't need such a comparison. |
923 | if (FirstIsConst && SecondIsConst) continue; |
924 | // If only one is const, then make it the first callback argument. |
925 | if (FirstIsConst || SecondIsConst) { |
926 | CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx]; |
927 | if (SecondIsConst) |
928 | std::swap(a&: A0, b&: A1); |
929 | } |
930 | |
931 | auto Ty = Type::getIntNTy(C&: *C, N: TypeSize); |
932 | IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true), |
933 | IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)}); |
934 | } |
935 | } |
936 | } |
937 | |
938 | void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, |
939 | size_t Idx, |
940 | bool IsLeafFunc) { |
941 | BasicBlock::iterator IP = BB.getFirstInsertionPt(); |
942 | bool IsEntryBB = &BB == &F.getEntryBlock(); |
943 | DebugLoc EntryLoc; |
944 | if (IsEntryBB) { |
945 | if (auto SP = F.getSubprogram()) |
946 | EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP); |
947 | // Keep static allocas and llvm.localescape calls in the entry block. Even |
948 | // if we aren't splitting the block, it's nice for allocas to be before |
949 | // calls. |
950 | IP = PrepareToSplitEntryBlock(BB, IP); |
951 | } |
952 | |
953 | InstrumentationIRBuilder IRB(&*IP); |
954 | if (EntryLoc) |
955 | IRB.SetCurrentDebugLocation(EntryLoc); |
956 | if (Options.TracePC) { |
957 | IRB.CreateCall(Callee: SanCovTracePC) |
958 | ->setCannotMerge(); // gets the PC using GET_CALLER_PC. |
959 | } |
960 | if (Options.TracePCGuard) { |
961 | auto GuardPtr = IRB.CreateIntToPtr( |
962 | V: IRB.CreateAdd(LHS: IRB.CreatePointerCast(V: FunctionGuardArray, DestTy: IntptrTy), |
963 | RHS: ConstantInt::get(Ty: IntptrTy, V: Idx * 4)), |
964 | DestTy: PtrTy); |
965 | IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge(); |
966 | } |
967 | if (Options.Inline8bitCounters) { |
968 | auto CounterPtr = IRB.CreateGEP( |
969 | Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray, |
970 | IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)}); |
971 | auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr); |
972 | auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1)); |
973 | auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr); |
974 | Load->setNoSanitizeMetadata(); |
975 | Store->setNoSanitizeMetadata(); |
976 | } |
977 | if (Options.InlineBoolFlag) { |
978 | auto FlagPtr = IRB.CreateGEP( |
979 | Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray, |
980 | IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)}); |
981 | auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr); |
982 | auto ThenTerm = |
983 | SplitBlockAndInsertIfThen(Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false); |
984 | IRBuilder<> ThenIRB(ThenTerm); |
985 | auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr); |
986 | Load->setNoSanitizeMetadata(); |
987 | Store->setNoSanitizeMetadata(); |
988 | } |
989 | if (Options.StackDepth && IsEntryBB && !IsLeafFunc) { |
990 | // Check stack depth. If it's the deepest so far, record it. |
991 | Module *M = F.getParent(); |
992 | Function *GetFrameAddr = Intrinsic::getDeclaration( |
993 | M, Intrinsic::id: frameaddress, |
994 | Tys: IRB.getPtrTy(AddrSpace: M->getDataLayout().getAllocaAddrSpace())); |
995 | auto FrameAddrPtr = |
996 | IRB.CreateCall(Callee: GetFrameAddr, Args: {Constant::getNullValue(Ty: Int32Ty)}); |
997 | auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy); |
998 | auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack); |
999 | auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack); |
1000 | auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false); |
1001 | IRBuilder<> ThenIRB(ThenTerm); |
1002 | auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack); |
1003 | LowestStack->setNoSanitizeMetadata(); |
1004 | Store->setNoSanitizeMetadata(); |
1005 | } |
1006 | } |
1007 | |
1008 | std::string |
1009 | ModuleSanitizerCoverage::getSectionName(const std::string &Section) const { |
1010 | if (TargetTriple.isOSBinFormatCOFF()) { |
1011 | if (Section == SanCovCountersSectionName) |
1012 | return ".SCOV$CM" ; |
1013 | if (Section == SanCovBoolFlagSectionName) |
1014 | return ".SCOV$BM" ; |
1015 | if (Section == SanCovPCsSectionName) |
1016 | return ".SCOVP$M" ; |
1017 | return ".SCOV$GM" ; // For SanCovGuardsSectionName. |
1018 | } |
1019 | if (TargetTriple.isOSBinFormatMachO()) |
1020 | return "__DATA,__" + Section; |
1021 | return "__" + Section; |
1022 | } |
1023 | |
1024 | std::string |
1025 | ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const { |
1026 | if (TargetTriple.isOSBinFormatMachO()) |
1027 | return "\1section$start$__DATA$__" + Section; |
1028 | return "__start___" + Section; |
1029 | } |
1030 | |
1031 | std::string |
1032 | ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const { |
1033 | if (TargetTriple.isOSBinFormatMachO()) |
1034 | return "\1section$end$__DATA$__" + Section; |
1035 | return "__stop___" + Section; |
1036 | } |
1037 | |
1038 | void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) { |
1039 | SmallVector<Constant *, 32> CFs; |
1040 | IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt()); |
1041 | |
1042 | for (auto &BB : F) { |
1043 | // blockaddress can not be used on function's entry block. |
1044 | if (&BB == &F.getEntryBlock()) |
1045 | CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy)); |
1046 | else |
1047 | CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB), |
1048 | DestTy: PtrTy)); |
1049 | |
1050 | for (auto SuccBB : successors(BB: &BB)) { |
1051 | assert(SuccBB != &F.getEntryBlock()); |
1052 | CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB), |
1053 | DestTy: PtrTy)); |
1054 | } |
1055 | |
1056 | CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy)); |
1057 | |
1058 | for (auto &Inst : BB) { |
1059 | if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) { |
1060 | if (CB->isIndirectCall()) { |
1061 | // TODO(navidem): handle indirect calls, for now mark its existence. |
1062 | CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr( |
1063 | V: ConstantInt::get(Ty: IntptrTy, V: -1), DestTy: PtrTy)); |
1064 | } else { |
1065 | auto CalledF = CB->getCalledFunction(); |
1066 | if (CalledF && !CalledF->isIntrinsic()) |
1067 | CFs.push_back( |
1068 | Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy)); |
1069 | } |
1070 | } |
1071 | } |
1072 | |
1073 | CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy)); |
1074 | } |
1075 | |
1076 | FunctionCFsArray = CreateFunctionLocalArrayInSection( |
1077 | NumElements: CFs.size(), F, Ty: PtrTy, Section: SanCovCFsSectionName); |
1078 | FunctionCFsArray->setInitializer( |
1079 | ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs)); |
1080 | FunctionCFsArray->setConstant(true); |
1081 | } |
1082 | |