1//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Coverage instrumentation done on LLVM IR level, works with Sanitizers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/Analysis/GlobalsModRef.h"
17#include "llvm/Analysis/PostDominators.h"
18#include "llvm/IR/Constant.h"
19#include "llvm/IR/DataLayout.h"
20#include "llvm/IR/Dominators.h"
21#include "llvm/IR/EHPersonalities.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/GlobalVariable.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/IntrinsicInst.h"
26#include "llvm/IR/Intrinsics.h"
27#include "llvm/IR/LLVMContext.h"
28#include "llvm/IR/Module.h"
29#include "llvm/IR/Type.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/SpecialCaseList.h"
32#include "llvm/Support/VirtualFileSystem.h"
33#include "llvm/TargetParser/Triple.h"
34#include "llvm/Transforms/Utils/BasicBlockUtils.h"
35#include "llvm/Transforms/Utils/ModuleUtils.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "sancov"
40
41const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
42const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
43const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
44const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
45const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
46const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
47const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
48const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
49const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
50const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
51const char SanCovLoad1[] = "__sanitizer_cov_load1";
52const char SanCovLoad2[] = "__sanitizer_cov_load2";
53const char SanCovLoad4[] = "__sanitizer_cov_load4";
54const char SanCovLoad8[] = "__sanitizer_cov_load8";
55const char SanCovLoad16[] = "__sanitizer_cov_load16";
56const char SanCovStore1[] = "__sanitizer_cov_store1";
57const char SanCovStore2[] = "__sanitizer_cov_store2";
58const char SanCovStore4[] = "__sanitizer_cov_store4";
59const char SanCovStore8[] = "__sanitizer_cov_store8";
60const char SanCovStore16[] = "__sanitizer_cov_store16";
61const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
62const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
63const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
64const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
65const char SanCovModuleCtorTracePcGuardName[] =
66 "sancov.module_ctor_trace_pc_guard";
67const char SanCovModuleCtor8bitCountersName[] =
68 "sancov.module_ctor_8bit_counters";
69const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
70static const uint64_t SanCtorAndDtorPriority = 2;
71
72const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
73const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
74const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
75const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
76const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
77const char SanCovCFsInitName[] = "__sanitizer_cov_cfs_init";
78
79const char SanCovGuardsSectionName[] = "sancov_guards";
80const char SanCovCountersSectionName[] = "sancov_cntrs";
81const char SanCovBoolFlagSectionName[] = "sancov_bools";
82const char SanCovPCsSectionName[] = "sancov_pcs";
83const char SanCovCFsSectionName[] = "sancov_cfs";
84
85const char SanCovLowestStackName[] = "__sancov_lowest_stack";
86
87static cl::opt<int> ClCoverageLevel(
88 "sanitizer-coverage-level",
89 cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
90 "3: all blocks and critical edges"),
91 cl::Hidden, cl::init(Val: 0));
92
93static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc",
94 cl::desc("Experimental pc tracing"), cl::Hidden,
95 cl::init(Val: false));
96
97static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard",
98 cl::desc("pc tracing with a guard"),
99 cl::Hidden, cl::init(Val: false));
100
101// If true, we create a global variable that contains PCs of all instrumented
102// BBs, put this global into a named section, and pass this section's bounds
103// to __sanitizer_cov_pcs_init.
104// This way the coverage instrumentation does not need to acquire the PCs
105// at run-time. Works with trace-pc-guard, inline-8bit-counters, and
106// inline-bool-flag.
107static cl::opt<bool> ClCreatePCTable("sanitizer-coverage-pc-table",
108 cl::desc("create a static PC table"),
109 cl::Hidden, cl::init(Val: false));
110
111static cl::opt<bool>
112 ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters",
113 cl::desc("increments 8-bit counter for every edge"),
114 cl::Hidden, cl::init(Val: false));
115
116static cl::opt<bool>
117 ClInlineBoolFlag("sanitizer-coverage-inline-bool-flag",
118 cl::desc("sets a boolean flag for every edge"), cl::Hidden,
119 cl::init(Val: false));
120
121static cl::opt<bool>
122 ClCMPTracing("sanitizer-coverage-trace-compares",
123 cl::desc("Tracing of CMP and similar instructions"),
124 cl::Hidden, cl::init(Val: false));
125
126static cl::opt<bool> ClDIVTracing("sanitizer-coverage-trace-divs",
127 cl::desc("Tracing of DIV instructions"),
128 cl::Hidden, cl::init(Val: false));
129
130static cl::opt<bool> ClLoadTracing("sanitizer-coverage-trace-loads",
131 cl::desc("Tracing of load instructions"),
132 cl::Hidden, cl::init(Val: false));
133
134static cl::opt<bool> ClStoreTracing("sanitizer-coverage-trace-stores",
135 cl::desc("Tracing of store instructions"),
136 cl::Hidden, cl::init(Val: false));
137
138static cl::opt<bool> ClGEPTracing("sanitizer-coverage-trace-geps",
139 cl::desc("Tracing of GEP instructions"),
140 cl::Hidden, cl::init(Val: false));
141
142static cl::opt<bool>
143 ClPruneBlocks("sanitizer-coverage-prune-blocks",
144 cl::desc("Reduce the number of instrumented blocks"),
145 cl::Hidden, cl::init(Val: true));
146
147static cl::opt<bool> ClStackDepth("sanitizer-coverage-stack-depth",
148 cl::desc("max stack depth tracing"),
149 cl::Hidden, cl::init(Val: false));
150
151static cl::opt<bool>
152 ClCollectCF("sanitizer-coverage-control-flow",
153 cl::desc("collect control flow for each function"), cl::Hidden,
154 cl::init(Val: false));
155
156namespace {
157
158SanitizerCoverageOptions getOptions(int LegacyCoverageLevel) {
159 SanitizerCoverageOptions Res;
160 switch (LegacyCoverageLevel) {
161 case 0:
162 Res.CoverageType = SanitizerCoverageOptions::SCK_None;
163 break;
164 case 1:
165 Res.CoverageType = SanitizerCoverageOptions::SCK_Function;
166 break;
167 case 2:
168 Res.CoverageType = SanitizerCoverageOptions::SCK_BB;
169 break;
170 case 3:
171 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
172 break;
173 case 4:
174 Res.CoverageType = SanitizerCoverageOptions::SCK_Edge;
175 Res.IndirectCalls = true;
176 break;
177 }
178 return Res;
179}
180
181SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) {
182 // Sets CoverageType and IndirectCalls.
183 SanitizerCoverageOptions CLOpts = getOptions(LegacyCoverageLevel: ClCoverageLevel);
184 Options.CoverageType = std::max(a: Options.CoverageType, b: CLOpts.CoverageType);
185 Options.IndirectCalls |= CLOpts.IndirectCalls;
186 Options.TraceCmp |= ClCMPTracing;
187 Options.TraceDiv |= ClDIVTracing;
188 Options.TraceGep |= ClGEPTracing;
189 Options.TracePC |= ClTracePC;
190 Options.TracePCGuard |= ClTracePCGuard;
191 Options.Inline8bitCounters |= ClInline8bitCounters;
192 Options.InlineBoolFlag |= ClInlineBoolFlag;
193 Options.PCTable |= ClCreatePCTable;
194 Options.NoPrune |= !ClPruneBlocks;
195 Options.StackDepth |= ClStackDepth;
196 Options.TraceLoads |= ClLoadTracing;
197 Options.TraceStores |= ClStoreTracing;
198 if (!Options.TracePCGuard && !Options.TracePC &&
199 !Options.Inline8bitCounters && !Options.StackDepth &&
200 !Options.InlineBoolFlag && !Options.TraceLoads && !Options.TraceStores)
201 Options.TracePCGuard = true; // TracePCGuard is default.
202 Options.CollectControlFlow |= ClCollectCF;
203 return Options;
204}
205
206using DomTreeCallback = function_ref<const DominatorTree *(Function &F)>;
207using PostDomTreeCallback =
208 function_ref<const PostDominatorTree *(Function &F)>;
209
210class ModuleSanitizerCoverage {
211public:
212 ModuleSanitizerCoverage(
213 const SanitizerCoverageOptions &Options = SanitizerCoverageOptions(),
214 const SpecialCaseList *Allowlist = nullptr,
215 const SpecialCaseList *Blocklist = nullptr)
216 : Options(OverrideFromCL(Options)), Allowlist(Allowlist),
217 Blocklist(Blocklist) {}
218 bool instrumentModule(Module &M, DomTreeCallback DTCallback,
219 PostDomTreeCallback PDTCallback);
220
221private:
222 void createFunctionControlFlow(Function &F);
223 void instrumentFunction(Function &F, DomTreeCallback DTCallback,
224 PostDomTreeCallback PDTCallback);
225 void InjectCoverageForIndirectCalls(Function &F,
226 ArrayRef<Instruction *> IndirCalls);
227 void InjectTraceForCmp(Function &F, ArrayRef<Instruction *> CmpTraceTargets);
228 void InjectTraceForDiv(Function &F,
229 ArrayRef<BinaryOperator *> DivTraceTargets);
230 void InjectTraceForGep(Function &F,
231 ArrayRef<GetElementPtrInst *> GepTraceTargets);
232 void InjectTraceForLoadsAndStores(Function &F, ArrayRef<LoadInst *> Loads,
233 ArrayRef<StoreInst *> Stores);
234 void InjectTraceForSwitch(Function &F,
235 ArrayRef<Instruction *> SwitchTraceTargets);
236 bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
237 bool IsLeafFunc = true);
238 GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements,
239 Function &F, Type *Ty,
240 const char *Section);
241 GlobalVariable *CreatePCArray(Function &F, ArrayRef<BasicBlock *> AllBlocks);
242 void CreateFunctionLocalArrays(Function &F, ArrayRef<BasicBlock *> AllBlocks);
243 void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx,
244 bool IsLeafFunc = true);
245 Function *CreateInitCallsForSections(Module &M, const char *CtorName,
246 const char *InitFunctionName, Type *Ty,
247 const char *Section);
248 std::pair<Value *, Value *> CreateSecStartEnd(Module &M, const char *Section,
249 Type *Ty);
250
251 std::string getSectionName(const std::string &Section) const;
252 std::string getSectionStart(const std::string &Section) const;
253 std::string getSectionEnd(const std::string &Section) const;
254 FunctionCallee SanCovTracePCIndir;
255 FunctionCallee SanCovTracePC, SanCovTracePCGuard;
256 std::array<FunctionCallee, 4> SanCovTraceCmpFunction;
257 std::array<FunctionCallee, 4> SanCovTraceConstCmpFunction;
258 std::array<FunctionCallee, 5> SanCovLoadFunction;
259 std::array<FunctionCallee, 5> SanCovStoreFunction;
260 std::array<FunctionCallee, 2> SanCovTraceDivFunction;
261 FunctionCallee SanCovTraceGepFunction;
262 FunctionCallee SanCovTraceSwitchFunction;
263 GlobalVariable *SanCovLowestStack;
264 Type *PtrTy, *IntptrTy, *Int64Ty, *Int32Ty, *Int16Ty, *Int8Ty, *Int1Ty;
265 Module *CurModule;
266 std::string CurModuleUniqueId;
267 Triple TargetTriple;
268 LLVMContext *C;
269 const DataLayout *DL;
270
271 GlobalVariable *FunctionGuardArray; // for trace-pc-guard.
272 GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters.
273 GlobalVariable *FunctionBoolArray; // for inline-bool-flag.
274 GlobalVariable *FunctionPCsArray; // for pc-table.
275 GlobalVariable *FunctionCFsArray; // for control flow table
276 SmallVector<GlobalValue *, 20> GlobalsToAppendToUsed;
277 SmallVector<GlobalValue *, 20> GlobalsToAppendToCompilerUsed;
278
279 SanitizerCoverageOptions Options;
280
281 const SpecialCaseList *Allowlist;
282 const SpecialCaseList *Blocklist;
283};
284} // namespace
285
286PreservedAnalyses SanitizerCoveragePass::run(Module &M,
287 ModuleAnalysisManager &MAM) {
288 ModuleSanitizerCoverage ModuleSancov(Options, Allowlist.get(),
289 Blocklist.get());
290 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
291 auto DTCallback = [&FAM](Function &F) -> const DominatorTree * {
292 return &FAM.getResult<DominatorTreeAnalysis>(IR&: F);
293 };
294 auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree * {
295 return &FAM.getResult<PostDominatorTreeAnalysis>(IR&: F);
296 };
297 if (!ModuleSancov.instrumentModule(M, DTCallback, PDTCallback))
298 return PreservedAnalyses::all();
299
300 PreservedAnalyses PA = PreservedAnalyses::none();
301 // GlobalsAA is considered stateless and does not get invalidated unless
302 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
303 // make changes that require GlobalsAA to be invalidated.
304 PA.abandon<GlobalsAA>();
305 return PA;
306}
307
308std::pair<Value *, Value *>
309ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
310 Type *Ty) {
311 // Use ExternalWeak so that if all sections are discarded due to section
312 // garbage collection, the linker will not report undefined symbol errors.
313 // Windows defines the start/stop symbols in compiler-rt so no need for
314 // ExternalWeak.
315 GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF()
316 ? GlobalVariable::ExternalLinkage
317 : GlobalVariable::ExternalWeakLinkage;
318 GlobalVariable *SecStart =
319 new GlobalVariable(M, Ty, false, Linkage, nullptr,
320 getSectionStart(Section));
321 SecStart->setVisibility(GlobalValue::HiddenVisibility);
322 GlobalVariable *SecEnd =
323 new GlobalVariable(M, Ty, false, Linkage, nullptr,
324 getSectionEnd(Section));
325 SecEnd->setVisibility(GlobalValue::HiddenVisibility);
326 IRBuilder<> IRB(M.getContext());
327 if (!TargetTriple.isOSBinFormatCOFF())
328 return std::make_pair(x&: SecStart, y&: SecEnd);
329
330 // Account for the fact that on windows-msvc __start_* symbols actually
331 // point to a uint64_t before the start of the array.
332 auto GEP =
333 IRB.CreatePtrAdd(Ptr: SecStart, Offset: ConstantInt::get(Ty: IntptrTy, V: sizeof(uint64_t)));
334 return std::make_pair(x&: GEP, y&: SecEnd);
335}
336
337Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
338 Module &M, const char *CtorName, const char *InitFunctionName, Type *Ty,
339 const char *Section) {
340 auto SecStartEnd = CreateSecStartEnd(M, Section, Ty);
341 auto SecStart = SecStartEnd.first;
342 auto SecEnd = SecStartEnd.second;
343 Function *CtorFunc;
344 std::tie(args&: CtorFunc, args: std::ignore) = createSanitizerCtorAndInitFunctions(
345 M, CtorName, InitName: InitFunctionName, InitArgTypes: {PtrTy, PtrTy}, InitArgs: {SecStart, SecEnd});
346 assert(CtorFunc->getName() == CtorName);
347
348 if (TargetTriple.supportsCOMDAT()) {
349 // Use comdat to dedup CtorFunc.
350 CtorFunc->setComdat(M.getOrInsertComdat(Name: CtorName));
351 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority, Data: CtorFunc);
352 } else {
353 appendToGlobalCtors(M, F: CtorFunc, Priority: SanCtorAndDtorPriority);
354 }
355
356 if (TargetTriple.isOSBinFormatCOFF()) {
357 // In COFF files, if the contructors are set as COMDAT (they are because
358 // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
359 // functions and data) is used, the constructors get stripped. To prevent
360 // this, give the constructors weak ODR linkage and ensure the linker knows
361 // to include the sancov constructor. This way the linker can deduplicate
362 // the constructors but always leave one copy.
363 CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
364 }
365 return CtorFunc;
366}
367
368bool ModuleSanitizerCoverage::instrumentModule(
369 Module &M, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) {
370 if (Options.CoverageType == SanitizerCoverageOptions::SCK_None)
371 return false;
372 if (Allowlist &&
373 !Allowlist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
374 return false;
375 if (Blocklist &&
376 Blocklist->inSection(Section: "coverage", Prefix: "src", Query: M.getSourceFileName()))
377 return false;
378 C = &(M.getContext());
379 DL = &M.getDataLayout();
380 CurModule = &M;
381 CurModuleUniqueId = getUniqueModuleId(M: CurModule);
382 TargetTriple = Triple(M.getTargetTriple());
383 FunctionGuardArray = nullptr;
384 Function8bitCounterArray = nullptr;
385 FunctionBoolArray = nullptr;
386 FunctionPCsArray = nullptr;
387 FunctionCFsArray = nullptr;
388 IntptrTy = Type::getIntNTy(C&: *C, N: DL->getPointerSizeInBits());
389 PtrTy = PointerType::getUnqual(C&: *C);
390 Type *VoidTy = Type::getVoidTy(C&: *C);
391 IRBuilder<> IRB(*C);
392 Int64Ty = IRB.getInt64Ty();
393 Int32Ty = IRB.getInt32Ty();
394 Int16Ty = IRB.getInt16Ty();
395 Int8Ty = IRB.getInt8Ty();
396 Int1Ty = IRB.getInt1Ty();
397
398 SanCovTracePCIndir =
399 M.getOrInsertFunction(Name: SanCovTracePCIndirName, RetTy: VoidTy, Args: IntptrTy);
400 // Make sure smaller parameters are zero-extended to i64 if required by the
401 // target ABI.
402 AttributeList SanCovTraceCmpZeroExtAL;
403 SanCovTraceCmpZeroExtAL =
404 SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);
405 SanCovTraceCmpZeroExtAL =
406 SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);
407
408 SanCovTraceCmpFunction[0] =
409 M.getOrInsertFunction(Name: SanCovTraceCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
410 Args: IRB.getInt8Ty(), Args: IRB.getInt8Ty());
411 SanCovTraceCmpFunction[1] =
412 M.getOrInsertFunction(Name: SanCovTraceCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
413 Args: IRB.getInt16Ty(), Args: IRB.getInt16Ty());
414 SanCovTraceCmpFunction[2] =
415 M.getOrInsertFunction(Name: SanCovTraceCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy,
416 Args: IRB.getInt32Ty(), Args: IRB.getInt32Ty());
417 SanCovTraceCmpFunction[3] =
418 M.getOrInsertFunction(Name: SanCovTraceCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
419
420 SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
421 Name: SanCovTraceConstCmp1, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int8Ty, Args: Int8Ty);
422 SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
423 Name: SanCovTraceConstCmp2, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int16Ty, Args: Int16Ty);
424 SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
425 Name: SanCovTraceConstCmp4, AttributeList: SanCovTraceCmpZeroExtAL, RetTy: VoidTy, Args: Int32Ty, Args: Int32Ty);
426 SanCovTraceConstCmpFunction[3] =
427 M.getOrInsertFunction(Name: SanCovTraceConstCmp8, RetTy: VoidTy, Args: Int64Ty, Args: Int64Ty);
428
429 // Loads.
430 SanCovLoadFunction[0] = M.getOrInsertFunction(Name: SanCovLoad1, RetTy: VoidTy, Args: PtrTy);
431 SanCovLoadFunction[1] =
432 M.getOrInsertFunction(Name: SanCovLoad2, RetTy: VoidTy, Args: PtrTy);
433 SanCovLoadFunction[2] =
434 M.getOrInsertFunction(Name: SanCovLoad4, RetTy: VoidTy, Args: PtrTy);
435 SanCovLoadFunction[3] =
436 M.getOrInsertFunction(Name: SanCovLoad8, RetTy: VoidTy, Args: PtrTy);
437 SanCovLoadFunction[4] =
438 M.getOrInsertFunction(Name: SanCovLoad16, RetTy: VoidTy, Args: PtrTy);
439 // Stores.
440 SanCovStoreFunction[0] =
441 M.getOrInsertFunction(Name: SanCovStore1, RetTy: VoidTy, Args: PtrTy);
442 SanCovStoreFunction[1] =
443 M.getOrInsertFunction(Name: SanCovStore2, RetTy: VoidTy, Args: PtrTy);
444 SanCovStoreFunction[2] =
445 M.getOrInsertFunction(Name: SanCovStore4, RetTy: VoidTy, Args: PtrTy);
446 SanCovStoreFunction[3] =
447 M.getOrInsertFunction(Name: SanCovStore8, RetTy: VoidTy, Args: PtrTy);
448 SanCovStoreFunction[4] =
449 M.getOrInsertFunction(Name: SanCovStore16, RetTy: VoidTy, Args: PtrTy);
450
451 {
452 AttributeList AL;
453 AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);
454 SanCovTraceDivFunction[0] =
455 M.getOrInsertFunction(Name: SanCovTraceDiv4, AttributeList: AL, RetTy: VoidTy, Args: IRB.getInt32Ty());
456 }
457 SanCovTraceDivFunction[1] =
458 M.getOrInsertFunction(Name: SanCovTraceDiv8, RetTy: VoidTy, Args: Int64Ty);
459 SanCovTraceGepFunction =
460 M.getOrInsertFunction(Name: SanCovTraceGep, RetTy: VoidTy, Args: IntptrTy);
461 SanCovTraceSwitchFunction =
462 M.getOrInsertFunction(Name: SanCovTraceSwitchName, RetTy: VoidTy, Args: Int64Ty, Args: PtrTy);
463
464 Constant *SanCovLowestStackConstant =
465 M.getOrInsertGlobal(Name: SanCovLowestStackName, Ty: IntptrTy);
466 SanCovLowestStack = dyn_cast<GlobalVariable>(Val: SanCovLowestStackConstant);
467 if (!SanCovLowestStack || SanCovLowestStack->getValueType() != IntptrTy) {
468 C->emitError(ErrorStr: StringRef("'") + SanCovLowestStackName +
469 "' should not be declared by the user");
470 return true;
471 }
472 SanCovLowestStack->setThreadLocalMode(
473 GlobalValue::ThreadLocalMode::InitialExecTLSModel);
474 if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
475 SanCovLowestStack->setInitializer(Constant::getAllOnesValue(Ty: IntptrTy));
476
477 SanCovTracePC = M.getOrInsertFunction(Name: SanCovTracePCName, RetTy: VoidTy);
478 SanCovTracePCGuard =
479 M.getOrInsertFunction(Name: SanCovTracePCGuardName, RetTy: VoidTy, Args: PtrTy);
480
481 for (auto &F : M)
482 instrumentFunction(F, DTCallback, PDTCallback);
483
484 Function *Ctor = nullptr;
485
486 if (FunctionGuardArray)
487 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorTracePcGuardName,
488 InitFunctionName: SanCovTracePCGuardInitName, Ty: Int32Ty,
489 Section: SanCovGuardsSectionName);
490 if (Function8bitCounterArray)
491 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtor8bitCountersName,
492 InitFunctionName: SanCov8bitCountersInitName, Ty: Int8Ty,
493 Section: SanCovCountersSectionName);
494 if (FunctionBoolArray) {
495 Ctor = CreateInitCallsForSections(M, CtorName: SanCovModuleCtorBoolFlagName,
496 InitFunctionName: SanCovBoolFlagInitName, Ty: Int1Ty,
497 Section: SanCovBoolFlagSectionName);
498 }
499 if (Ctor && Options.PCTable) {
500 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovPCsSectionName, Ty: IntptrTy);
501 FunctionCallee InitFunction = declareSanitizerInitFunction(
502 M, InitName: SanCovPCsInitName, InitArgTypes: {PtrTy, PtrTy});
503 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
504 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
505 }
506
507 if (Ctor && Options.CollectControlFlow) {
508 auto SecStartEnd = CreateSecStartEnd(M, Section: SanCovCFsSectionName, Ty: IntptrTy);
509 FunctionCallee InitFunction = declareSanitizerInitFunction(
510 M, InitName: SanCovCFsInitName, InitArgTypes: {PtrTy, PtrTy});
511 IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
512 IRBCtor.CreateCall(Callee: InitFunction, Args: {SecStartEnd.first, SecStartEnd.second});
513 }
514
515 appendToUsed(M, Values: GlobalsToAppendToUsed);
516 appendToCompilerUsed(M, Values: GlobalsToAppendToCompilerUsed);
517 return true;
518}
519
520// True if block has successors and it dominates all of them.
521static bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) {
522 if (succ_empty(BB))
523 return false;
524
525 return llvm::all_of(Range: successors(BB), P: [&](const BasicBlock *SUCC) {
526 return DT->dominates(A: BB, B: SUCC);
527 });
528}
529
530// True if block has predecessors and it postdominates all of them.
531static bool isFullPostDominator(const BasicBlock *BB,
532 const PostDominatorTree *PDT) {
533 if (pred_empty(BB))
534 return false;
535
536 return llvm::all_of(Range: predecessors(BB), P: [&](const BasicBlock *PRED) {
537 return PDT->dominates(A: BB, B: PRED);
538 });
539}
540
541static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
542 const DominatorTree *DT,
543 const PostDominatorTree *PDT,
544 const SanitizerCoverageOptions &Options) {
545 // Don't insert coverage for blocks containing nothing but unreachable: we
546 // will never call __sanitizer_cov() for them, so counting them in
547 // NumberOfInstrumentedBlocks() might complicate calculation of code coverage
548 // percentage. Also, unreachable instructions frequently have no debug
549 // locations.
550 if (isa<UnreachableInst>(Val: BB->getFirstNonPHIOrDbgOrLifetime()))
551 return false;
552
553 // Don't insert coverage into blocks without a valid insertion point
554 // (catchswitch blocks).
555 if (BB->getFirstInsertionPt() == BB->end())
556 return false;
557
558 if (Options.NoPrune || &F.getEntryBlock() == BB)
559 return true;
560
561 if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function &&
562 &F.getEntryBlock() != BB)
563 return false;
564
565 // Do not instrument full dominators, or full post-dominators with multiple
566 // predecessors.
567 return !isFullDominator(BB, DT)
568 && !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
569}
570
571
572// Returns true iff From->To is a backedge.
573// A twist here is that we treat From->To as a backedge if
574// * To dominates From or
575// * To->UniqueSuccessor dominates From
576static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
577 const DominatorTree *DT) {
578 if (DT->dominates(A: To, B: From))
579 return true;
580 if (auto Next = To->getUniqueSuccessor())
581 if (DT->dominates(A: Next, B: From))
582 return true;
583 return false;
584}
585
586// Prunes uninteresting Cmp instrumentation:
587// * CMP instructions that feed into loop backedge branch.
588//
589// Note that Cmp pruning is controlled by the same flag as the
590// BB pruning.
591static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT,
592 const SanitizerCoverageOptions &Options) {
593 if (!Options.NoPrune)
594 if (CMP->hasOneUse())
595 if (auto BR = dyn_cast<BranchInst>(Val: CMP->user_back()))
596 for (BasicBlock *B : BR->successors())
597 if (IsBackEdge(From: BR->getParent(), To: B, DT))
598 return false;
599 return true;
600}
601
602void ModuleSanitizerCoverage::instrumentFunction(
603 Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) {
604 if (F.empty())
605 return;
606 if (F.getName().contains(Other: ".module_ctor"))
607 return; // Should not instrument sanitizer init functions.
608 if (F.getName().starts_with(Prefix: "__sanitizer_"))
609 return; // Don't instrument __sanitizer_* callbacks.
610 // Don't touch available_externally functions, their actual body is elewhere.
611 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
612 return;
613 // Don't instrument MSVC CRT configuration helpers. They may run before normal
614 // initialization.
615 if (F.getName() == "__local_stdio_printf_options" ||
616 F.getName() == "__local_stdio_scanf_options")
617 return;
618 if (isa<UnreachableInst>(Val: F.getEntryBlock().getTerminator()))
619 return;
620 // Don't instrument functions using SEH for now. Splitting basic blocks like
621 // we do for coverage breaks WinEHPrepare.
622 // FIXME: Remove this when SEH no longer uses landingpad pattern matching.
623 if (F.hasPersonalityFn() &&
624 isAsynchronousEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
625 return;
626 if (Allowlist && !Allowlist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
627 return;
628 if (Blocklist && Blocklist->inSection(Section: "coverage", Prefix: "fun", Query: F.getName()))
629 return;
630 if (F.hasFnAttribute(Attribute::NoSanitizeCoverage))
631 return;
632 if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge)
633 SplitAllCriticalEdges(F, Options: CriticalEdgeSplittingOptions().setIgnoreUnreachableDests());
634 SmallVector<Instruction *, 8> IndirCalls;
635 SmallVector<BasicBlock *, 16> BlocksToInstrument;
636 SmallVector<Instruction *, 8> CmpTraceTargets;
637 SmallVector<Instruction *, 8> SwitchTraceTargets;
638 SmallVector<BinaryOperator *, 8> DivTraceTargets;
639 SmallVector<GetElementPtrInst *, 8> GepTraceTargets;
640 SmallVector<LoadInst *, 8> Loads;
641 SmallVector<StoreInst *, 8> Stores;
642
643 const DominatorTree *DT = DTCallback(F);
644 const PostDominatorTree *PDT = PDTCallback(F);
645 bool IsLeafFunc = true;
646
647 for (auto &BB : F) {
648 if (shouldInstrumentBlock(F, BB: &BB, DT, PDT, Options))
649 BlocksToInstrument.push_back(Elt: &BB);
650 for (auto &Inst : BB) {
651 if (Options.IndirectCalls) {
652 CallBase *CB = dyn_cast<CallBase>(Val: &Inst);
653 if (CB && CB->isIndirectCall())
654 IndirCalls.push_back(Elt: &Inst);
655 }
656 if (Options.TraceCmp) {
657 if (ICmpInst *CMP = dyn_cast<ICmpInst>(Val: &Inst))
658 if (IsInterestingCmp(CMP, DT, Options))
659 CmpTraceTargets.push_back(Elt: &Inst);
660 if (isa<SwitchInst>(Val: &Inst))
661 SwitchTraceTargets.push_back(Elt: &Inst);
662 }
663 if (Options.TraceDiv)
664 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: &Inst))
665 if (BO->getOpcode() == Instruction::SDiv ||
666 BO->getOpcode() == Instruction::UDiv)
667 DivTraceTargets.push_back(Elt: BO);
668 if (Options.TraceGep)
669 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &Inst))
670 GepTraceTargets.push_back(Elt: GEP);
671 if (Options.TraceLoads)
672 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &Inst))
673 Loads.push_back(Elt: LI);
674 if (Options.TraceStores)
675 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &Inst))
676 Stores.push_back(Elt: SI);
677 if (Options.StackDepth)
678 if (isa<InvokeInst>(Val: Inst) ||
679 (isa<CallInst>(Val: Inst) && !isa<IntrinsicInst>(Val: Inst)))
680 IsLeafFunc = false;
681 }
682 }
683
684 if (Options.CollectControlFlow)
685 createFunctionControlFlow(F);
686
687 InjectCoverage(F, AllBlocks: BlocksToInstrument, IsLeafFunc);
688 InjectCoverageForIndirectCalls(F, IndirCalls);
689 InjectTraceForCmp(F, CmpTraceTargets);
690 InjectTraceForSwitch(F, SwitchTraceTargets);
691 InjectTraceForDiv(F, DivTraceTargets);
692 InjectTraceForGep(F, GepTraceTargets);
693 InjectTraceForLoadsAndStores(F, Loads, Stores);
694}
695
696GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection(
697 size_t NumElements, Function &F, Type *Ty, const char *Section) {
698 ArrayType *ArrayTy = ArrayType::get(ElementType: Ty, NumElements);
699 auto Array = new GlobalVariable(
700 *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
701 Constant::getNullValue(Ty: ArrayTy), "__sancov_gen_");
702
703 if (TargetTriple.supportsCOMDAT() &&
704 (TargetTriple.isOSBinFormatELF() || !F.isInterposable()))
705 if (auto Comdat = getOrCreateFunctionComdat(F, T&: TargetTriple))
706 Array->setComdat(Comdat);
707 Array->setSection(getSectionName(Section));
708 Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue()));
709
710 // sancov_pcs parallels the other metadata section(s). Optimizers (e.g.
711 // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other
712 // section(s) as a unit, so we conservatively retain all unconditionally in
713 // the compiler.
714 //
715 // With comdat (COFF/ELF), the linker can guarantee the associated sections
716 // will be retained or discarded as a unit, so llvm.compiler.used is
717 // sufficient. Otherwise, conservatively make all of them retained by the
718 // linker.
719 if (Array->hasComdat())
720 GlobalsToAppendToCompilerUsed.push_back(Elt: Array);
721 else
722 GlobalsToAppendToUsed.push_back(Elt: Array);
723
724 return Array;
725}
726
727GlobalVariable *
728ModuleSanitizerCoverage::CreatePCArray(Function &F,
729 ArrayRef<BasicBlock *> AllBlocks) {
730 size_t N = AllBlocks.size();
731 assert(N);
732 SmallVector<Constant *, 32> PCs;
733 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
734 for (size_t i = 0; i < N; i++) {
735 if (&F.getEntryBlock() == AllBlocks[i]) {
736 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
737 PCs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
738 V: ConstantInt::get(Ty: IntptrTy, V: 1), DestTy: PtrTy));
739 } else {
740 PCs.push_back(Elt: (Constant *)IRB.CreatePointerCast(
741 V: BlockAddress::get(BB: AllBlocks[i]), DestTy: PtrTy));
742 PCs.push_back(Elt: Constant::getNullValue(Ty: PtrTy));
743 }
744 }
745 auto *PCArray = CreateFunctionLocalArrayInSection(NumElements: N * 2, F, Ty: PtrTy,
746 Section: SanCovPCsSectionName);
747 PCArray->setInitializer(
748 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: N * 2), V: PCs));
749 PCArray->setConstant(true);
750
751 return PCArray;
752}
753
754void ModuleSanitizerCoverage::CreateFunctionLocalArrays(
755 Function &F, ArrayRef<BasicBlock *> AllBlocks) {
756 if (Options.TracePCGuard)
757 FunctionGuardArray = CreateFunctionLocalArrayInSection(
758 NumElements: AllBlocks.size(), F, Ty: Int32Ty, Section: SanCovGuardsSectionName);
759
760 if (Options.Inline8bitCounters)
761 Function8bitCounterArray = CreateFunctionLocalArrayInSection(
762 NumElements: AllBlocks.size(), F, Ty: Int8Ty, Section: SanCovCountersSectionName);
763 if (Options.InlineBoolFlag)
764 FunctionBoolArray = CreateFunctionLocalArrayInSection(
765 NumElements: AllBlocks.size(), F, Ty: Int1Ty, Section: SanCovBoolFlagSectionName);
766
767 if (Options.PCTable)
768 FunctionPCsArray = CreatePCArray(F, AllBlocks);
769}
770
771bool ModuleSanitizerCoverage::InjectCoverage(Function &F,
772 ArrayRef<BasicBlock *> AllBlocks,
773 bool IsLeafFunc) {
774 if (AllBlocks.empty()) return false;
775 CreateFunctionLocalArrays(F, AllBlocks);
776 for (size_t i = 0, N = AllBlocks.size(); i < N; i++)
777 InjectCoverageAtBlock(F, BB&: *AllBlocks[i], Idx: i, IsLeafFunc);
778 return true;
779}
780
781// On every indirect call we call a run-time function
782// __sanitizer_cov_indir_call* with two parameters:
783// - callee address,
784// - global cache array that contains CacheSize pointers (zero-initialized).
785// The cache is used to speed up recording the caller-callee pairs.
786// The address of the caller is passed implicitly via caller PC.
787// CacheSize is encoded in the name of the run-time function.
788void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls(
789 Function &F, ArrayRef<Instruction *> IndirCalls) {
790 if (IndirCalls.empty())
791 return;
792 assert(Options.TracePC || Options.TracePCGuard ||
793 Options.Inline8bitCounters || Options.InlineBoolFlag);
794 for (auto *I : IndirCalls) {
795 InstrumentationIRBuilder IRB(I);
796 CallBase &CB = cast<CallBase>(Val&: *I);
797 Value *Callee = CB.getCalledOperand();
798 if (isa<InlineAsm>(Val: Callee))
799 continue;
800 IRB.CreateCall(Callee: SanCovTracePCIndir, Args: IRB.CreatePointerCast(V: Callee, DestTy: IntptrTy));
801 }
802}
803
804// For every switch statement we insert a call:
805// __sanitizer_cov_trace_switch(CondValue,
806// {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... })
807
808void ModuleSanitizerCoverage::InjectTraceForSwitch(
809 Function &, ArrayRef<Instruction *> SwitchTraceTargets) {
810 for (auto *I : SwitchTraceTargets) {
811 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: I)) {
812 InstrumentationIRBuilder IRB(I);
813 SmallVector<Constant *, 16> Initializers;
814 Value *Cond = SI->getCondition();
815 if (Cond->getType()->getScalarSizeInBits() >
816 Int64Ty->getScalarSizeInBits())
817 continue;
818 Initializers.push_back(Elt: ConstantInt::get(Ty: Int64Ty, V: SI->getNumCases()));
819 Initializers.push_back(
820 Elt: ConstantInt::get(Ty: Int64Ty, V: Cond->getType()->getScalarSizeInBits()));
821 if (Cond->getType()->getScalarSizeInBits() <
822 Int64Ty->getScalarSizeInBits())
823 Cond = IRB.CreateIntCast(V: Cond, DestTy: Int64Ty, isSigned: false);
824 for (auto It : SI->cases()) {
825 ConstantInt *C = It.getCaseValue();
826 if (C->getType()->getScalarSizeInBits() < 64)
827 C = ConstantInt::get(Context&: C->getContext(), V: C->getValue().zext(width: 64));
828 Initializers.push_back(Elt: C);
829 }
830 llvm::sort(C: drop_begin(RangeOrContainer&: Initializers, N: 2),
831 Comp: [](const Constant *A, const Constant *B) {
832 return cast<ConstantInt>(Val: A)->getLimitedValue() <
833 cast<ConstantInt>(Val: B)->getLimitedValue();
834 });
835 ArrayType *ArrayOfInt64Ty = ArrayType::get(ElementType: Int64Ty, NumElements: Initializers.size());
836 GlobalVariable *GV = new GlobalVariable(
837 *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage,
838 ConstantArray::get(T: ArrayOfInt64Ty, V: Initializers),
839 "__sancov_gen_cov_switch_values");
840 IRB.CreateCall(Callee: SanCovTraceSwitchFunction, Args: {Cond, GV});
841 }
842 }
843}
844
845void ModuleSanitizerCoverage::InjectTraceForDiv(
846 Function &, ArrayRef<BinaryOperator *> DivTraceTargets) {
847 for (auto *BO : DivTraceTargets) {
848 InstrumentationIRBuilder IRB(BO);
849 Value *A1 = BO->getOperand(i_nocapture: 1);
850 if (isa<ConstantInt>(Val: A1)) continue;
851 if (!A1->getType()->isIntegerTy())
852 continue;
853 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A1->getType());
854 int CallbackIdx = TypeSize == 32 ? 0 :
855 TypeSize == 64 ? 1 : -1;
856 if (CallbackIdx < 0) continue;
857 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
858 IRB.CreateCall(Callee: SanCovTraceDivFunction[CallbackIdx],
859 Args: {IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
860 }
861}
862
863void ModuleSanitizerCoverage::InjectTraceForGep(
864 Function &, ArrayRef<GetElementPtrInst *> GepTraceTargets) {
865 for (auto *GEP : GepTraceTargets) {
866 InstrumentationIRBuilder IRB(GEP);
867 for (Use &Idx : GEP->indices())
868 if (!isa<ConstantInt>(Val: Idx) && Idx->getType()->isIntegerTy())
869 IRB.CreateCall(Callee: SanCovTraceGepFunction,
870 Args: {IRB.CreateIntCast(V: Idx, DestTy: IntptrTy, isSigned: true)});
871 }
872}
873
874void ModuleSanitizerCoverage::InjectTraceForLoadsAndStores(
875 Function &, ArrayRef<LoadInst *> Loads, ArrayRef<StoreInst *> Stores) {
876 auto CallbackIdx = [&](Type *ElementTy) -> int {
877 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: ElementTy);
878 return TypeSize == 8 ? 0
879 : TypeSize == 16 ? 1
880 : TypeSize == 32 ? 2
881 : TypeSize == 64 ? 3
882 : TypeSize == 128 ? 4
883 : -1;
884 };
885 for (auto *LI : Loads) {
886 InstrumentationIRBuilder IRB(LI);
887 auto Ptr = LI->getPointerOperand();
888 int Idx = CallbackIdx(LI->getType());
889 if (Idx < 0)
890 continue;
891 IRB.CreateCall(Callee: SanCovLoadFunction[Idx], Args: Ptr);
892 }
893 for (auto *SI : Stores) {
894 InstrumentationIRBuilder IRB(SI);
895 auto Ptr = SI->getPointerOperand();
896 int Idx = CallbackIdx(SI->getValueOperand()->getType());
897 if (Idx < 0)
898 continue;
899 IRB.CreateCall(Callee: SanCovStoreFunction[Idx], Args: Ptr);
900 }
901}
902
903void ModuleSanitizerCoverage::InjectTraceForCmp(
904 Function &, ArrayRef<Instruction *> CmpTraceTargets) {
905 for (auto *I : CmpTraceTargets) {
906 if (ICmpInst *ICMP = dyn_cast<ICmpInst>(Val: I)) {
907 InstrumentationIRBuilder IRB(ICMP);
908 Value *A0 = ICMP->getOperand(i_nocapture: 0);
909 Value *A1 = ICMP->getOperand(i_nocapture: 1);
910 if (!A0->getType()->isIntegerTy())
911 continue;
912 uint64_t TypeSize = DL->getTypeStoreSizeInBits(Ty: A0->getType());
913 int CallbackIdx = TypeSize == 8 ? 0 :
914 TypeSize == 16 ? 1 :
915 TypeSize == 32 ? 2 :
916 TypeSize == 64 ? 3 : -1;
917 if (CallbackIdx < 0) continue;
918 // __sanitizer_cov_trace_cmp((type_size << 32) | predicate, A0, A1);
919 auto CallbackFunc = SanCovTraceCmpFunction[CallbackIdx];
920 bool FirstIsConst = isa<ConstantInt>(Val: A0);
921 bool SecondIsConst = isa<ConstantInt>(Val: A1);
922 // If both are const, then we don't need such a comparison.
923 if (FirstIsConst && SecondIsConst) continue;
924 // If only one is const, then make it the first callback argument.
925 if (FirstIsConst || SecondIsConst) {
926 CallbackFunc = SanCovTraceConstCmpFunction[CallbackIdx];
927 if (SecondIsConst)
928 std::swap(a&: A0, b&: A1);
929 }
930
931 auto Ty = Type::getIntNTy(C&: *C, N: TypeSize);
932 IRB.CreateCall(Callee: CallbackFunc, Args: {IRB.CreateIntCast(V: A0, DestTy: Ty, isSigned: true),
933 IRB.CreateIntCast(V: A1, DestTy: Ty, isSigned: true)});
934 }
935 }
936}
937
938void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
939 size_t Idx,
940 bool IsLeafFunc) {
941 BasicBlock::iterator IP = BB.getFirstInsertionPt();
942 bool IsEntryBB = &BB == &F.getEntryBlock();
943 DebugLoc EntryLoc;
944 if (IsEntryBB) {
945 if (auto SP = F.getSubprogram())
946 EntryLoc = DILocation::get(Context&: SP->getContext(), Line: SP->getScopeLine(), Column: 0, Scope: SP);
947 // Keep static allocas and llvm.localescape calls in the entry block. Even
948 // if we aren't splitting the block, it's nice for allocas to be before
949 // calls.
950 IP = PrepareToSplitEntryBlock(BB, IP);
951 }
952
953 InstrumentationIRBuilder IRB(&*IP);
954 if (EntryLoc)
955 IRB.SetCurrentDebugLocation(EntryLoc);
956 if (Options.TracePC) {
957 IRB.CreateCall(Callee: SanCovTracePC)
958 ->setCannotMerge(); // gets the PC using GET_CALLER_PC.
959 }
960 if (Options.TracePCGuard) {
961 auto GuardPtr = IRB.CreateIntToPtr(
962 V: IRB.CreateAdd(LHS: IRB.CreatePointerCast(V: FunctionGuardArray, DestTy: IntptrTy),
963 RHS: ConstantInt::get(Ty: IntptrTy, V: Idx * 4)),
964 DestTy: PtrTy);
965 IRB.CreateCall(Callee: SanCovTracePCGuard, Args: GuardPtr)->setCannotMerge();
966 }
967 if (Options.Inline8bitCounters) {
968 auto CounterPtr = IRB.CreateGEP(
969 Ty: Function8bitCounterArray->getValueType(), Ptr: Function8bitCounterArray,
970 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
971 auto Load = IRB.CreateLoad(Ty: Int8Ty, Ptr: CounterPtr);
972 auto Inc = IRB.CreateAdd(LHS: Load, RHS: ConstantInt::get(Ty: Int8Ty, V: 1));
973 auto Store = IRB.CreateStore(Val: Inc, Ptr: CounterPtr);
974 Load->setNoSanitizeMetadata();
975 Store->setNoSanitizeMetadata();
976 }
977 if (Options.InlineBoolFlag) {
978 auto FlagPtr = IRB.CreateGEP(
979 Ty: FunctionBoolArray->getValueType(), Ptr: FunctionBoolArray,
980 IdxList: {ConstantInt::get(Ty: IntptrTy, V: 0), ConstantInt::get(Ty: IntptrTy, V: Idx)});
981 auto Load = IRB.CreateLoad(Ty: Int1Ty, Ptr: FlagPtr);
982 auto ThenTerm =
983 SplitBlockAndInsertIfThen(Cond: IRB.CreateIsNull(Arg: Load), SplitBefore: &*IP, Unreachable: false);
984 IRBuilder<> ThenIRB(ThenTerm);
985 auto Store = ThenIRB.CreateStore(Val: ConstantInt::getTrue(Ty: Int1Ty), Ptr: FlagPtr);
986 Load->setNoSanitizeMetadata();
987 Store->setNoSanitizeMetadata();
988 }
989 if (Options.StackDepth && IsEntryBB && !IsLeafFunc) {
990 // Check stack depth. If it's the deepest so far, record it.
991 Module *M = F.getParent();
992 Function *GetFrameAddr = Intrinsic::getDeclaration(
993 M, Intrinsic::id: frameaddress,
994 Tys: IRB.getPtrTy(AddrSpace: M->getDataLayout().getAllocaAddrSpace()));
995 auto FrameAddrPtr =
996 IRB.CreateCall(Callee: GetFrameAddr, Args: {Constant::getNullValue(Ty: Int32Ty)});
997 auto FrameAddrInt = IRB.CreatePtrToInt(V: FrameAddrPtr, DestTy: IntptrTy);
998 auto LowestStack = IRB.CreateLoad(Ty: IntptrTy, Ptr: SanCovLowestStack);
999 auto IsStackLower = IRB.CreateICmpULT(LHS: FrameAddrInt, RHS: LowestStack);
1000 auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false);
1001 IRBuilder<> ThenIRB(ThenTerm);
1002 auto Store = ThenIRB.CreateStore(Val: FrameAddrInt, Ptr: SanCovLowestStack);
1003 LowestStack->setNoSanitizeMetadata();
1004 Store->setNoSanitizeMetadata();
1005 }
1006}
1007
1008std::string
1009ModuleSanitizerCoverage::getSectionName(const std::string &Section) const {
1010 if (TargetTriple.isOSBinFormatCOFF()) {
1011 if (Section == SanCovCountersSectionName)
1012 return ".SCOV$CM";
1013 if (Section == SanCovBoolFlagSectionName)
1014 return ".SCOV$BM";
1015 if (Section == SanCovPCsSectionName)
1016 return ".SCOVP$M";
1017 return ".SCOV$GM"; // For SanCovGuardsSectionName.
1018 }
1019 if (TargetTriple.isOSBinFormatMachO())
1020 return "__DATA,__" + Section;
1021 return "__" + Section;
1022}
1023
1024std::string
1025ModuleSanitizerCoverage::getSectionStart(const std::string &Section) const {
1026 if (TargetTriple.isOSBinFormatMachO())
1027 return "\1section$start$__DATA$__" + Section;
1028 return "__start___" + Section;
1029}
1030
1031std::string
1032ModuleSanitizerCoverage::getSectionEnd(const std::string &Section) const {
1033 if (TargetTriple.isOSBinFormatMachO())
1034 return "\1section$end$__DATA$__" + Section;
1035 return "__stop___" + Section;
1036}
1037
1038void ModuleSanitizerCoverage::createFunctionControlFlow(Function &F) {
1039 SmallVector<Constant *, 32> CFs;
1040 IRBuilder<> IRB(&*F.getEntryBlock().getFirstInsertionPt());
1041
1042 for (auto &BB : F) {
1043 // blockaddress can not be used on function's entry block.
1044 if (&BB == &F.getEntryBlock())
1045 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: &F, DestTy: PtrTy));
1046 else
1047 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: &BB),
1048 DestTy: PtrTy));
1049
1050 for (auto SuccBB : successors(BB: &BB)) {
1051 assert(SuccBB != &F.getEntryBlock());
1052 CFs.push_back(Elt: (Constant *)IRB.CreatePointerCast(V: BlockAddress::get(BB: SuccBB),
1053 DestTy: PtrTy));
1054 }
1055
1056 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1057
1058 for (auto &Inst : BB) {
1059 if (CallBase *CB = dyn_cast<CallBase>(Val: &Inst)) {
1060 if (CB->isIndirectCall()) {
1061 // TODO(navidem): handle indirect calls, for now mark its existence.
1062 CFs.push_back(Elt: (Constant *)IRB.CreateIntToPtr(
1063 V: ConstantInt::get(Ty: IntptrTy, V: -1), DestTy: PtrTy));
1064 } else {
1065 auto CalledF = CB->getCalledFunction();
1066 if (CalledF && !CalledF->isIntrinsic())
1067 CFs.push_back(
1068 Elt: (Constant *)IRB.CreatePointerCast(V: CalledF, DestTy: PtrTy));
1069 }
1070 }
1071 }
1072
1073 CFs.push_back(Elt: (Constant *)Constant::getNullValue(Ty: PtrTy));
1074 }
1075
1076 FunctionCFsArray = CreateFunctionLocalArrayInSection(
1077 NumElements: CFs.size(), F, Ty: PtrTy, Section: SanCovCFsSectionName);
1078 FunctionCFsArray->setInitializer(
1079 ConstantArray::get(T: ArrayType::get(ElementType: PtrTy, NumElements: CFs.size()), V: CFs));
1080 FunctionCFsArray->setConstant(true);
1081}
1082

source code of llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp