1	//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	/// \file
10	/// This file is a part of MemorySanitizer, a detector of uninitialized
11	/// reads.
12	///
13	/// The algorithm of the tool is similar to Memcheck
14	/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15	/// byte of the application memory, poison the shadow of the malloc-ed
16	/// or alloca-ed memory, load the shadow bits on every memory read,
17	/// propagate the shadow bits through some of the arithmetic
18	/// instruction (including MOV), store the shadow bits on every memory
19	/// write, report a bug on some other instructions (e.g. JMP) if the
20	/// associated shadow is poisoned.
21	///
22	/// But there are differences too. The first and the major one:
23	/// compiler instrumentation instead of binary instrumentation. This
24	/// gives us much better register allocation, possible compiler
25	/// optimizations and a fast start-up. But this brings the major issue
26	/// as well: msan needs to see all program events, including system
27	/// calls and reads/writes in system libraries, so we either need to
28	/// compile *everything* with msan or use a binary translation
29	/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30	/// Another difference from Memcheck is that we use 8 shadow bits per
31	/// byte of application memory and use a direct shadow mapping. This
32	/// greatly simplifies the instrumentation code and avoids races on
33	/// shadow updates (Memcheck is single-threaded so races are not a
34	/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35	/// path storage that uses 8 bits per byte).
36	///
37	/// The default value of shadow is 0, which means "clean" (not poisoned).
38	///
39	/// Every module initializer should call __msan_init to ensure that the
40	/// shadow memory is ready. On error, __msan_warning is called. Since
41	/// parameters and return values may be passed via registers, we have a
42	/// specialized thread-local shadow for return values
43	/// (__msan_retval_tls) and parameters (__msan_param_tls).
44	///
45	/// Origin tracking.
46	///
47	/// MemorySanitizer can track origins (allocation points) of all uninitialized
48	/// values. This behavior is controlled with a flag (msan-track-origins) and is
49	/// disabled by default.
50	///
51	/// Origins are 4-byte values created and interpreted by the runtime library.
52	/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53	/// of application memory. Propagation of origins is basically a bunch of
54	/// "select" instructions that pick the origin of a dirty argument, if an
55	/// instruction has one.
56	///
57	/// Every 4 aligned, consecutive bytes of application memory have one origin
58	/// value associated with them. If these bytes contain uninitialized data
59	/// coming from 2 different allocations, the last store wins. Because of this,
60	/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61	/// practice.
62	///
63	/// Origins are meaningless for fully initialized values, so MemorySanitizer
64	/// avoids storing origin to memory when a fully initialized value is stored.
65	/// This way it avoids needless overwriting origin of the 4-byte region on
66	/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67	///
68	/// Atomic handling.
69	///
70	/// Ideally, every atomic store of application value should update the
71	/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72	/// of two disjoint locations can not be done without severe slowdown.
73	///
74	/// Therefore, we implement an approximation that may err on the safe side.
75	/// In this implementation, every atomically accessed location in the program
76	/// may only change from (partially) uninitialized to fully initialized, but
77	/// not the other way around. We load the shadow _after_ the application load,
78	/// and we store the shadow _before_ the app store. Also, we always store clean
79	/// shadow (if the application store is atomic). This way, if the store-load
80	/// pair constitutes a happens-before arc, shadow store and load are correctly
81	/// ordered such that the load will get either the value that was stored, or
82	/// some later value (which is always clean).
83	///
84	/// This does not work very well with Compare-And-Swap (CAS) and
85	/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86	/// must store the new shadow before the app operation, and load the shadow
87	/// after the app operation. Computers don't work this way. Current
88	/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89	/// value. It implements the store part as a simple atomic store by storing a
90	/// clean shadow.
91	///
92	/// Instrumenting inline assembly.
93	///
94	/// For inline assembly code LLVM has little idea about which memory locations
95	/// become initialized depending on the arguments. It can be possible to figure
96	/// out which arguments are meant to point to inputs and outputs, but the
97	/// actual semantics can be only visible at runtime. In the Linux kernel it's
98	/// also possible that the arguments only indicate the offset for a base taken
99	/// from a segment register, so it's dangerous to treat any asm() arguments as
100	/// pointers. We take a conservative approach generating calls to
101	/// __msan_instrument_asm_store(ptr, size)
102	/// , which defer the memory unpoisoning to the runtime library.
103	/// The latter can perform more complex address checks to figure out whether
104	/// it's safe to touch the shadow memory.
105	/// Like with atomic operations, we call __msan_instrument_asm_store() before
106	/// the assembly call, so that changes to the shadow memory will be seen by
107	/// other threads together with main memory initialization.
108	///
109	/// KernelMemorySanitizer (KMSAN) implementation.
110	///
111	/// The major differences between KMSAN and MSan instrumentation are:
112	/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113	/// - KMSAN allocates shadow and origin memory for each page separately, so
114	/// there are no explicit accesses to shadow and origin in the
115	/// instrumentation.
116	/// Shadow and origin values for a particular X-byte memory location
117	/// (X=1,2,4,8) are accessed through pointers obtained via the
118	/// __msan_metadata_ptr_for_load_X(ptr)
119	/// __msan_metadata_ptr_for_store_X(ptr)
120	/// functions. The corresponding functions check that the X-byte accesses
121	/// are possible and returns the pointers to shadow and origin memory.
122	/// Arbitrary sized accesses are handled with:
123	/// __msan_metadata_ptr_for_load_n(ptr, size)
124	/// __msan_metadata_ptr_for_store_n(ptr, size);
125	/// Note that the sanitizer code has to deal with how shadow/origin pairs
126	/// returned by the these functions are represented in different ABIs. In
127	/// the X86_64 ABI they are returned in RDX:RAX, and in the SystemZ ABI they
128	/// are written to memory pointed to by a hidden parameter.
129	/// - TLS variables are stored in a single per-task struct. A call to a
130	/// function __msan_get_context_state() returning a pointer to that struct
131	/// is inserted into every instrumented function before the entry block;
132	/// - __msan_warning() takes a 32-bit origin parameter;
133	/// - local variables are poisoned with __msan_poison_alloca() upon function
134	/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
135	/// function;
136	/// - the pass doesn't declare any global variables or add global constructors
137	/// to the translation unit.
138	///
139	/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
140	/// calls, making sure we're on the safe side wrt. possible false positives.
141	///
142	/// KernelMemorySanitizer only supports X86_64 and SystemZ at the moment.
143	///
144	//
145	// FIXME: This sanitizer does not yet handle scalable vectors
146	//
147	//===----------------------------------------------------------------------===//
148
149	#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
150	#include "llvm/ADT/APInt.h"
151	#include "llvm/ADT/ArrayRef.h"
152	#include "llvm/ADT/DenseMap.h"
153	#include "llvm/ADT/DepthFirstIterator.h"
154	#include "llvm/ADT/SetVector.h"
155	#include "llvm/ADT/SmallPtrSet.h"
156	#include "llvm/ADT/SmallVector.h"
157	#include "llvm/ADT/StringExtras.h"
158	#include "llvm/ADT/StringRef.h"
159	#include "llvm/Analysis/GlobalsModRef.h"
160	#include "llvm/Analysis/TargetLibraryInfo.h"
161	#include "llvm/Analysis/ValueTracking.h"
162	#include "llvm/IR/Argument.h"
163	#include "llvm/IR/AttributeMask.h"
164	#include "llvm/IR/Attributes.h"
165	#include "llvm/IR/BasicBlock.h"
166	#include "llvm/IR/CallingConv.h"
167	#include "llvm/IR/Constant.h"
168	#include "llvm/IR/Constants.h"
169	#include "llvm/IR/DataLayout.h"
170	#include "llvm/IR/DerivedTypes.h"
171	#include "llvm/IR/Function.h"
172	#include "llvm/IR/GlobalValue.h"
173	#include "llvm/IR/GlobalVariable.h"
174	#include "llvm/IR/IRBuilder.h"
175	#include "llvm/IR/InlineAsm.h"
176	#include "llvm/IR/InstVisitor.h"
177	#include "llvm/IR/InstrTypes.h"
178	#include "llvm/IR/Instruction.h"
179	#include "llvm/IR/Instructions.h"
180	#include "llvm/IR/IntrinsicInst.h"
181	#include "llvm/IR/Intrinsics.h"
182	#include "llvm/IR/IntrinsicsX86.h"
183	#include "llvm/IR/MDBuilder.h"
184	#include "llvm/IR/Module.h"
185	#include "llvm/IR/Type.h"
186	#include "llvm/IR/Value.h"
187	#include "llvm/IR/ValueMap.h"
188	#include "llvm/Support/Alignment.h"
189	#include "llvm/Support/AtomicOrdering.h"
190	#include "llvm/Support/Casting.h"
191	#include "llvm/Support/CommandLine.h"
192	#include "llvm/Support/Debug.h"
193	#include "llvm/Support/DebugCounter.h"
194	#include "llvm/Support/ErrorHandling.h"
195	#include "llvm/Support/MathExtras.h"
196	#include "llvm/Support/raw_ostream.h"
197	#include "llvm/TargetParser/Triple.h"
198	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
199	#include "llvm/Transforms/Utils/Local.h"
200	#include "llvm/Transforms/Utils/ModuleUtils.h"
201	#include <algorithm>
202	#include <cassert>
203	#include <cstddef>
204	#include <cstdint>
205	#include <memory>
206	#include <string>
207	#include <tuple>
208
209	using namespace llvm;
210
211	#define DEBUG_TYPE "msan"
212
213	DEBUG_COUNTER(DebugInsertCheck, "msan-insert-check",
214	"Controls which checks to insert");
215
216	DEBUG_COUNTER(DebugInstrumentInstruction, "msan-instrument-instruction",
217	"Controls which instruction to instrument");
218
219	static const unsigned kOriginSize = 4;
220	static const Align kMinOriginAlignment = Align(4);
221	static const Align kShadowTLSAlignment = Align(8);
222
223	// These constants must be kept in sync with the ones in msan.h.
224	static const unsigned kParamTLSSize = 800;
225	static const unsigned kRetvalTLSSize = 800;
226
227	// Accesses sizes are powers of two: 1, 2, 4, 8.
228	static const size_t kNumberOfAccessSizes = 4;
229
230	/// Track origins of uninitialized values.
231	///
232	/// Adds a section to MemorySanitizer report that points to the allocation
233	/// (stack or heap) the uninitialized bits came from originally.
234	static cl::opt<int> ClTrackOrigins(
235	"msan-track-origins",
236	cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden,
237	cl::init(Val: 0));
238
239	static cl::opt<bool> ClKeepGoing("msan-keep-going",
240	cl::desc("keep going after reporting a UMR"),
241	cl::Hidden, cl::init(Val: false));
242
243	static cl::opt<bool>
244	ClPoisonStack("msan-poison-stack",
245	cl::desc("poison uninitialized stack variables"), cl::Hidden,
246	cl::init(Val: true));
247
248	static cl::opt<bool> ClPoisonStackWithCall(
249	"msan-poison-stack-with-call",
250	cl::desc("poison uninitialized stack variables with a call"), cl::Hidden,
251	cl::init(Val: false));
252
253	static cl::opt<int> ClPoisonStackPattern(
254	"msan-poison-stack-pattern",
255	cl::desc("poison uninitialized stack variables with the given pattern"),
256	cl::Hidden, cl::init(Val: 0xff));
257
258	static cl::opt<bool>
259	ClPrintStackNames("msan-print-stack-names",
260	cl::desc("Print name of local stack variable"),
261	cl::Hidden, cl::init(Val: true));
262
263	static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
264	cl::desc("poison undef temps"), cl::Hidden,
265	cl::init(Val: true));
266
267	static cl::opt<bool>
268	ClHandleICmp("msan-handle-icmp",
269	cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
270	cl::Hidden, cl::init(Val: true));
271
272	static cl::opt<bool>
273	ClHandleICmpExact("msan-handle-icmp-exact",
274	cl::desc("exact handling of relational integer ICmp"),
275	cl::Hidden, cl::init(Val: false));
276
277	static cl::opt<bool> ClHandleLifetimeIntrinsics(
278	"msan-handle-lifetime-intrinsics",
279	cl::desc(
280	"when possible, poison scoped variables at the beginning of the scope "
281	"(slower, but more precise)"),
282	cl::Hidden, cl::init(Val: true));
283
284	// When compiling the Linux kernel, we sometimes see false positives related to
285	// MSan being unable to understand that inline assembly calls may initialize
286	// local variables.
287	// This flag makes the compiler conservatively unpoison every memory location
288	// passed into an assembly call. Note that this may cause false positives.
289	// Because it's impossible to figure out the array sizes, we can only unpoison
290	// the first sizeof(type) bytes for each type* pointer.
291	static cl::opt<bool> ClHandleAsmConservative(
292	"msan-handle-asm-conservative",
293	cl::desc("conservative handling of inline assembly"), cl::Hidden,
294	cl::init(Val: true));
295
296	// This flag controls whether we check the shadow of the address
297	// operand of load or store. Such bugs are very rare, since load from
298	// a garbage address typically results in SEGV, but still happen
299	// (e.g. only lower bits of address are garbage, or the access happens
300	// early at program startup where malloc-ed memory is more likely to
301	// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
302	static cl::opt<bool> ClCheckAccessAddress(
303	"msan-check-access-address",
304	cl::desc("report accesses through a pointer which has poisoned shadow"),
305	cl::Hidden, cl::init(Val: true));
306
307	static cl::opt<bool> ClEagerChecks(
308	"msan-eager-checks",
309	cl::desc("check arguments and return values at function call boundaries"),
310	cl::Hidden, cl::init(Val: false));
311
312	static cl::opt<bool> ClDumpStrictInstructions(
313	"msan-dump-strict-instructions",
314	cl::desc("print out instructions with default strict semantics"),
315	cl::Hidden, cl::init(Val: false));
316
317	static cl::opt<int> ClInstrumentationWithCallThreshold(
318	"msan-instrumentation-with-call-threshold",
319	cl::desc(
320	"If the function being instrumented requires more than "
321	"this number of checks and origin stores, use callbacks instead of "
322	"inline checks (-1 means never use callbacks)."),
323	cl::Hidden, cl::init(Val: 3500));
324
325	static cl::opt<bool>
326	ClEnableKmsan("msan-kernel",
327	cl::desc("Enable KernelMemorySanitizer instrumentation"),
328	cl::Hidden, cl::init(Val: false));
329
330	static cl::opt<bool>
331	ClDisableChecks("msan-disable-checks",
332	cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
333	cl::init(Val: false));
334
335	static cl::opt<bool>
336	ClCheckConstantShadow("msan-check-constant-shadow",
337	cl::desc("Insert checks for constant shadow values"),
338	cl::Hidden, cl::init(Val: true));
339
340	// This is off by default because of a bug in gold:
341	// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
342	static cl::opt<bool>
343	ClWithComdat("msan-with-comdat",
344	cl::desc("Place MSan constructors in comdat sections"),
345	cl::Hidden, cl::init(Val: false));
346
347	// These options allow to specify custom memory map parameters
348	// See MemoryMapParams for details.
349	static cl::opt<uint64_t> ClAndMask("msan-and-mask",
350	cl::desc("Define custom MSan AndMask"),
351	cl::Hidden, cl::init(Val: 0));
352
353	static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
354	cl::desc("Define custom MSan XorMask"),
355	cl::Hidden, cl::init(Val: 0));
356
357	static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
358	cl::desc("Define custom MSan ShadowBase"),
359	cl::Hidden, cl::init(Val: 0));
360
361	static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
362	cl::desc("Define custom MSan OriginBase"),
363	cl::Hidden, cl::init(Val: 0));
364
365	static cl::opt<int>
366	ClDisambiguateWarning("msan-disambiguate-warning-threshold",
367	cl::desc("Define threshold for number of checks per "
368	"debug location to force origin update."),
369	cl::Hidden, cl::init(Val: 3));
370
371	const char kMsanModuleCtorName[] = "msan.module_ctor";
372	const char kMsanInitName[] = "__msan_init";
373
374	namespace {
375
376	// Memory map parameters used in application-to-shadow address calculation.
377	// Offset = (Addr & ~AndMask) ^ XorMask
378	// Shadow = ShadowBase + Offset
379	// Origin = OriginBase + Offset
380	struct MemoryMapParams {
381	uint64_t AndMask;
382	uint64_t XorMask;
383	uint64_t ShadowBase;
384	uint64_t OriginBase;
385	};
386
387	struct PlatformMemoryMapParams {
388	const MemoryMapParams *bits32;
389	const MemoryMapParams *bits64;
390	};
391
392	} // end anonymous namespace
393
394	// i386 Linux
395	static const MemoryMapParams Linux_I386_MemoryMapParams = {
396	.AndMask: 0x000080000000, // AndMask
397	.XorMask: 0, // XorMask (not used)
398	.ShadowBase: 0, // ShadowBase (not used)
399	.OriginBase: 0x000040000000, // OriginBase
400	};
401
402	// x86_64 Linux
403	static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
404	.AndMask: 0, // AndMask (not used)
405	.XorMask: 0x500000000000, // XorMask
406	.ShadowBase: 0, // ShadowBase (not used)
407	.OriginBase: 0x100000000000, // OriginBase
408	};
409
410	// mips64 Linux
411	static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
412	.AndMask: 0, // AndMask (not used)
413	.XorMask: 0x008000000000, // XorMask
414	.ShadowBase: 0, // ShadowBase (not used)
415	.OriginBase: 0x002000000000, // OriginBase
416	};
417
418	// ppc64 Linux
419	static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
420	.AndMask: 0xE00000000000, // AndMask
421	.XorMask: 0x100000000000, // XorMask
422	.ShadowBase: 0x080000000000, // ShadowBase
423	.OriginBase: 0x1C0000000000, // OriginBase
424	};
425
426	// s390x Linux
427	static const MemoryMapParams Linux_S390X_MemoryMapParams = {
428	.AndMask: 0xC00000000000, // AndMask
429	.XorMask: 0, // XorMask (not used)
430	.ShadowBase: 0x080000000000, // ShadowBase
431	.OriginBase: 0x1C0000000000, // OriginBase
432	};
433
434	// aarch64 Linux
435	static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
436	.AndMask: 0, // AndMask (not used)
437	.XorMask: 0x0B00000000000, // XorMask
438	.ShadowBase: 0, // ShadowBase (not used)
439	.OriginBase: 0x0200000000000, // OriginBase
440	};
441
442	// loongarch64 Linux
443	static const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
444	.AndMask: 0, // AndMask (not used)
445	.XorMask: 0x500000000000, // XorMask
446	.ShadowBase: 0, // ShadowBase (not used)
447	.OriginBase: 0x100000000000, // OriginBase
448	};
449
450	// aarch64 FreeBSD
451	static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
452	.AndMask: 0x1800000000000, // AndMask
453	.XorMask: 0x0400000000000, // XorMask
454	.ShadowBase: 0x0200000000000, // ShadowBase
455	.OriginBase: 0x0700000000000, // OriginBase
456	};
457
458	// i386 FreeBSD
459	static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
460	.AndMask: 0x000180000000, // AndMask
461	.XorMask: 0x000040000000, // XorMask
462	.ShadowBase: 0x000020000000, // ShadowBase
463	.OriginBase: 0x000700000000, // OriginBase
464	};
465
466	// x86_64 FreeBSD
467	static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
468	.AndMask: 0xc00000000000, // AndMask
469	.XorMask: 0x200000000000, // XorMask
470	.ShadowBase: 0x100000000000, // ShadowBase
471	.OriginBase: 0x380000000000, // OriginBase
472	};
473
474	// x86_64 NetBSD
475	static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
476	.AndMask: 0, // AndMask
477	.XorMask: 0x500000000000, // XorMask
478	.ShadowBase: 0, // ShadowBase
479	.OriginBase: 0x100000000000, // OriginBase
480	};
481
482	static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
483	.bits32: &Linux_I386_MemoryMapParams,
484	.bits64: &Linux_X86_64_MemoryMapParams,
485	};
486
487	static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
488	.bits32: nullptr,
489	.bits64: &Linux_MIPS64_MemoryMapParams,
490	};
491
492	static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
493	.bits32: nullptr,
494	.bits64: &Linux_PowerPC64_MemoryMapParams,
495	};
496
497	static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
498	.bits32: nullptr,
499	.bits64: &Linux_S390X_MemoryMapParams,
500	};
501
502	static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
503	.bits32: nullptr,
504	.bits64: &Linux_AArch64_MemoryMapParams,
505	};
506
507	static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams = {
508	.bits32: nullptr,
509	.bits64: &Linux_LoongArch64_MemoryMapParams,
510	};
511
512	static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
513	.bits32: nullptr,
514	.bits64: &FreeBSD_AArch64_MemoryMapParams,
515	};
516
517	static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
518	.bits32: &FreeBSD_I386_MemoryMapParams,
519	.bits64: &FreeBSD_X86_64_MemoryMapParams,
520	};
521
522	static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
523	.bits32: nullptr,
524	.bits64: &NetBSD_X86_64_MemoryMapParams,
525	};
526
527	namespace {
528
529	/// Instrument functions of a module to detect uninitialized reads.
530	///
531	/// Instantiating MemorySanitizer inserts the msan runtime library API function
532	/// declarations into the module if they don't exist already. Instantiating
533	/// ensures the __msan_init function is in the list of global constructors for
534	/// the module.
535	class MemorySanitizer {
536	public:
537	MemorySanitizer(Module &M, MemorySanitizerOptions Options)
538	: CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
539	Recover(Options.Recover), EagerChecks(Options.EagerChecks) {
540	initializeModule(M);
541	}
542
543	// MSan cannot be moved or copied because of MapParams.
544	MemorySanitizer(MemorySanitizer &&) = delete;
545	MemorySanitizer &operator=(MemorySanitizer &&) = delete;
546	MemorySanitizer(const MemorySanitizer &) = delete;
547	MemorySanitizer &operator=(const MemorySanitizer &) = delete;
548
549	bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
550
551	private:
552	friend struct MemorySanitizerVisitor;
553	friend struct VarArgHelperBase;
554	friend struct VarArgAMD64Helper;
555	friend struct VarArgMIPS64Helper;
556	friend struct VarArgAArch64Helper;
557	friend struct VarArgPowerPC64Helper;
558	friend struct VarArgSystemZHelper;
559
560	void initializeModule(Module &M);
561	void initializeCallbacks(Module &M, const TargetLibraryInfo &TLI);
562	void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
563	void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
564
565	template <typename... ArgsTy>
566	FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
567	ArgsTy... Args);
568
569	/// True if we're compiling the Linux kernel.
570	bool CompileKernel;
571	/// Track origins (allocation points) of uninitialized values.
572	int TrackOrigins;
573	bool Recover;
574	bool EagerChecks;
575
576	Triple TargetTriple;
577	LLVMContext *C;
578	Type *IntptrTy; ///< Integer type with the size of a ptr in default AS.
579	Type *OriginTy;
580	PointerType *PtrTy; ///< Integer type with the size of a ptr in default AS.
581
582	// XxxTLS variables represent the per-thread state in MSan and per-task state
583	// in KMSAN.
584	// For the userspace these point to thread-local globals. In the kernel land
585	// they point to the members of a per-task struct obtained via a call to
586	// __msan_get_context_state().
587
588	/// Thread-local shadow storage for function parameters.
589	Value *ParamTLS;
590
591	/// Thread-local origin storage for function parameters.
592	Value *ParamOriginTLS;
593
594	/// Thread-local shadow storage for function return value.
595	Value *RetvalTLS;
596
597	/// Thread-local origin storage for function return value.
598	Value *RetvalOriginTLS;
599
600	/// Thread-local shadow storage for in-register va_arg function.
601	Value *VAArgTLS;
602
603	/// Thread-local shadow storage for in-register va_arg function.
604	Value *VAArgOriginTLS;
605
606	/// Thread-local shadow storage for va_arg overflow area.
607	Value *VAArgOverflowSizeTLS;
608
609	/// Are the instrumentation callbacks set up?
610	bool CallbacksInitialized = false;
611
612	/// The run-time callback to print a warning.
613	FunctionCallee WarningFn;
614
615	// These arrays are indexed by log2(AccessSize).
616	FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
617	FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
618
619	/// Run-time helper that generates a new origin value for a stack
620	/// allocation.
621	FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
622	// No description version
623	FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
624
625	/// Run-time helper that poisons stack on function entry.
626	FunctionCallee MsanPoisonStackFn;
627
628	/// Run-time helper that records a store (or any event) of an
629	/// uninitialized value and returns an updated origin id encoding this info.
630	FunctionCallee MsanChainOriginFn;
631
632	/// Run-time helper that paints an origin over a region.
633	FunctionCallee MsanSetOriginFn;
634
635	/// MSan runtime replacements for memmove, memcpy and memset.
636	FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
637
638	/// KMSAN callback for task-local function argument shadow.
639	StructType *MsanContextStateTy;
640	FunctionCallee MsanGetContextStateFn;
641
642	/// Functions for poisoning/unpoisoning local variables
643	FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
644
645	/// Pair of shadow/origin pointers.
646	Type *MsanMetadata;
647
648	/// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata.
649	FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
650	FunctionCallee MsanMetadataPtrForLoad_1_8[4];
651	FunctionCallee MsanMetadataPtrForStore_1_8[4];
652	FunctionCallee MsanInstrumentAsmStoreFn;
653
654	/// Storage for return values of the MsanMetadataPtrXxx functions.
655	Value *MsanMetadataAlloca;
656
657	/// Helper to choose between different MsanMetadataPtrXxx().
658	FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
659
660	/// Memory map parameters used in application-to-shadow calculation.
661	const MemoryMapParams *MapParams;
662
663	/// Custom memory map parameters used when -msan-shadow-base or
664	// -msan-origin-base is provided.
665	MemoryMapParams CustomMapParams;
666
667	MDNode *ColdCallWeights;
668
669	/// Branch weights for origin store.
670	MDNode *OriginStoreWeights;
671	};
672
673	void insertModuleCtor(Module &M) {
674	getOrCreateSanitizerCtorAndInitFunctions(
675	M, CtorName: kMsanModuleCtorName, InitName: kMsanInitName,
676	/*InitArgTypes=*/{},
677	/*InitArgs=*/{},
678	// This callback is invoked when the functions are created the first
679	// time. Hook them into the global ctors list in that case:
680	FunctionsCreatedCallback: [&](Function *Ctor, FunctionCallee) {
681	if (!ClWithComdat) {
682	appendToGlobalCtors(M, F: Ctor, Priority: 0);
683	return;
684	}
685	Comdat *MsanCtorComdat = M.getOrInsertComdat(Name: kMsanModuleCtorName);
686	Ctor->setComdat(MsanCtorComdat);
687	appendToGlobalCtors(M, F: Ctor, Priority: 0, Data: Ctor);
688	});
689	}
690
691	template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
692	return (Opt.getNumOccurrences() > 0) ? Opt : Default;
693	}
694
695	} // end anonymous namespace
696
697	MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K,
698	bool EagerChecks)
699	: Kernel(getOptOrDefault(Opt: ClEnableKmsan, Default: K)),
700	TrackOrigins(getOptOrDefault(Opt: ClTrackOrigins, Default: Kernel ? 2 : TO)),
701	Recover(getOptOrDefault(Opt: ClKeepGoing, Default: Kernel || R)),
702	EagerChecks(getOptOrDefault(Opt: ClEagerChecks, Default: EagerChecks)) {}
703
704	PreservedAnalyses MemorySanitizerPass::run(Module &M,
705	ModuleAnalysisManager &AM) {
706	bool Modified = false;
707	if (!Options.Kernel) {
708	insertModuleCtor(M);
709	Modified = true;
710	}
711
712	auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
713	for (Function &F : M) {
714	if (F.empty())
715	continue;
716	MemorySanitizer Msan(*F.getParent(), Options);
717	Modified |=
718	Msan.sanitizeFunction(F, TLI&: FAM.getResult<TargetLibraryAnalysis>(IR&: F));
719	}
720
721	if (!Modified)
722	return PreservedAnalyses::all();
723
724	PreservedAnalyses PA = PreservedAnalyses::none();
725	// GlobalsAA is considered stateless and does not get invalidated unless
726	// explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
727	// make changes that require GlobalsAA to be invalidated.
728	PA.abandon<GlobalsAA>();
729	return PA;
730	}
731
732	void MemorySanitizerPass::printPipeline(
733	raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
734	static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
735	OS, MapClassName2PassName);
736	OS << '<';
737	if (Options.Recover)
738	OS << "recover;";
739	if (Options.Kernel)
740	OS << "kernel;";
741	if (Options.EagerChecks)
742	OS << "eager-checks;";
743	OS << "track-origins=" << Options.TrackOrigins;
744	OS << '>';
745	}
746
747	/// Create a non-const global initialized with the given string.
748	///
749	/// Creates a writable global for Str so that we can pass it to the
750	/// run-time lib. Runtime uses first 4 bytes of the string to store the
751	/// frame ID, so the string needs to be mutable.
752	static GlobalVariable *createPrivateConstGlobalForString(Module &M,
753	StringRef Str) {
754	Constant *StrConst = ConstantDataArray::getString(Context&: M.getContext(), Initializer: Str);
755	return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
756	GlobalValue::PrivateLinkage, StrConst, "");
757	}
758
759	template <typename... ArgsTy>
760	FunctionCallee
761	MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
762	ArgsTy... Args) {
763	if (TargetTriple.getArch() == Triple::systemz) {
764	// SystemZ ABI: shadow/origin pair is returned via a hidden parameter.
765	return M.getOrInsertFunction(Name, Type::getVoidTy(C&: *C),
766	PointerType::get(ElementType: MsanMetadata, AddressSpace: 0),
767	std::forward<ArgsTy>(Args)...);
768	}
769
770	return M.getOrInsertFunction(Name, MsanMetadata,
771	std::forward<ArgsTy>(Args)...);
772	}
773
774	/// Create KMSAN API callbacks.
775	void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
776	IRBuilder<> IRB(*C);
777
778	// These will be initialized in insertKmsanPrologue().
779	RetvalTLS = nullptr;
780	RetvalOriginTLS = nullptr;
781	ParamTLS = nullptr;
782	ParamOriginTLS = nullptr;
783	VAArgTLS = nullptr;
784	VAArgOriginTLS = nullptr;
785	VAArgOverflowSizeTLS = nullptr;
786
787	WarningFn = M.getOrInsertFunction(Name: "__msan_warning",
788	AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
789	RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
790
791	// Requests the per-task context state (kmsan_context_state*) from the
792	// runtime library.
793	MsanContextStateTy = StructType::get(
794	elt1: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
795	elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8),
796	elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8),
797	elts: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8), /* va_arg_origin */
798	elts: IRB.getInt64Ty(), elts: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4), elts: OriginTy,
799	elts: OriginTy);
800	MsanGetContextStateFn = M.getOrInsertFunction(
801	Name: "__msan_get_context_state", RetTy: PointerType::get(ElementType: MsanContextStateTy, AddressSpace: 0));
802
803	MsanMetadata = StructType::get(elt1: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0),
804	elts: PointerType::get(ElementType: IRB.getInt32Ty(), AddressSpace: 0));
805
806	for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
807	std::string name_load =
808	"__msan_metadata_ptr_for_load_" + std::to_string(val: size);
809	std::string name_store =
810	"__msan_metadata_ptr_for_store_" + std::to_string(val: size);
811	MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
812	M, Name: name_load, Args: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0));
813	MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
814	M, Name: name_store, Args: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0));
815	}
816
817	MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
818	M, Name: "__msan_metadata_ptr_for_load_n", Args: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0),
819	Args: IRB.getInt64Ty());
820	MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
821	M, Name: "__msan_metadata_ptr_for_store_n",
822	Args: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0), Args: IRB.getInt64Ty());
823
824	// Functions for poisoning and unpoisoning memory.
825	MsanPoisonAllocaFn = M.getOrInsertFunction(
826	Name: "__msan_poison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
827	MsanUnpoisonAllocaFn = M.getOrInsertFunction(
828	Name: "__msan_unpoison_alloca", RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
829	}
830
831	static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
832	return M.getOrInsertGlobal(Name, Ty, CreateGlobalCallback: [&] {
833	return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
834	nullptr, Name, nullptr,
835	GlobalVariable::InitialExecTLSModel);
836	});
837	}
838
839	/// Insert declarations for userspace-specific functions and globals.
840	void MemorySanitizer::createUserspaceApi(Module &M, const TargetLibraryInfo &TLI) {
841	IRBuilder<> IRB(*C);
842
843	// Create the callback.
844	// FIXME: this function should have "Cold" calling conv,
845	// which is not yet implemented.
846	if (TrackOrigins) {
847	StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
848	: "__msan_warning_with_origin_noreturn";
849	WarningFn = M.getOrInsertFunction(Name: WarningFnName,
850	AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false),
851	RetTy: IRB.getVoidTy(), Args: IRB.getInt32Ty());
852	} else {
853	StringRef WarningFnName =
854	Recover ? "__msan_warning" : "__msan_warning_noreturn";
855	WarningFn = M.getOrInsertFunction(Name: WarningFnName, RetTy: IRB.getVoidTy());
856	}
857
858	// Create the global TLS variables.
859	RetvalTLS =
860	getOrInsertGlobal(M, Name: "__msan_retval_tls",
861	Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kRetvalTLSSize / 8));
862
863	RetvalOriginTLS = getOrInsertGlobal(M, Name: "__msan_retval_origin_tls", Ty: OriginTy);
864
865	ParamTLS =
866	getOrInsertGlobal(M, Name: "__msan_param_tls",
867	Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
868
869	ParamOriginTLS =
870	getOrInsertGlobal(M, Name: "__msan_param_origin_tls",
871	Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
872
873	VAArgTLS =
874	getOrInsertGlobal(M, Name: "__msan_va_arg_tls",
875	Ty: ArrayType::get(ElementType: IRB.getInt64Ty(), NumElements: kParamTLSSize / 8));
876
877	VAArgOriginTLS =
878	getOrInsertGlobal(M, Name: "__msan_va_arg_origin_tls",
879	Ty: ArrayType::get(ElementType: OriginTy, NumElements: kParamTLSSize / 4));
880
881	VAArgOverflowSizeTLS =
882	getOrInsertGlobal(M, Name: "__msan_va_arg_overflow_size_tls", Ty: IRB.getInt64Ty());
883
884	for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
885	AccessSizeIndex++) {
886	unsigned AccessSize = 1 << AccessSizeIndex;
887	std::string FunctionName = "__msan_maybe_warning_" + itostr(X: AccessSize);
888	MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
889	Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 1}, /*Signed=*/false),
890	RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: IRB.getInt32Ty());
891
892	FunctionName = "__msan_maybe_store_origin_" + itostr(X: AccessSize);
893	MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
894	Name: FunctionName, AttributeList: TLI.getAttrList(C, ArgNos: {0, 2}, /*Signed=*/false),
895	RetTy: IRB.getVoidTy(), Args: IRB.getIntNTy(N: AccessSize * 8), Args: PtrTy,
896	Args: IRB.getInt32Ty());
897	}
898
899	MsanSetAllocaOriginWithDescriptionFn =
900	M.getOrInsertFunction(Name: "__msan_set_alloca_origin_with_descr",
901	RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy, Args: PtrTy);
902	MsanSetAllocaOriginNoDescriptionFn =
903	M.getOrInsertFunction(Name: "__msan_set_alloca_origin_no_descr",
904	RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: PtrTy);
905	MsanPoisonStackFn = M.getOrInsertFunction(Name: "__msan_poison_stack",
906	RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy);
907	}
908
909	/// Insert extern declaration of runtime-provided functions and globals.
910	void MemorySanitizer::initializeCallbacks(Module &M, const TargetLibraryInfo &TLI) {
911	// Only do this once.
912	if (CallbacksInitialized)
913	return;
914
915	IRBuilder<> IRB(*C);
916	// Initialize callbacks that are common for kernel and userspace
917	// instrumentation.
918	MsanChainOriginFn = M.getOrInsertFunction(
919	Name: "__msan_chain_origin",
920	AttributeList: TLI.getAttrList(C, ArgNos: {0}, /*Signed=*/false, /*Ret=*/true), RetTy: IRB.getInt32Ty(),
921	Args: IRB.getInt32Ty());
922	MsanSetOriginFn = M.getOrInsertFunction(
923	Name: "__msan_set_origin", AttributeList: TLI.getAttrList(C, ArgNos: {2}, /*Signed=*/false),
924	RetTy: IRB.getVoidTy(), Args: PtrTy, Args: IntptrTy, Args: IRB.getInt32Ty());
925	MemmoveFn =
926	M.getOrInsertFunction(Name: "__msan_memmove", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
927	MemcpyFn =
928	M.getOrInsertFunction(Name: "__msan_memcpy", RetTy: PtrTy, Args: PtrTy, Args: PtrTy, Args: IntptrTy);
929	MemsetFn = M.getOrInsertFunction(Name: "__msan_memset",
930	AttributeList: TLI.getAttrList(C, ArgNos: {1}, /*Signed=*/true),
931	RetTy: PtrTy, Args: PtrTy, Args: IRB.getInt32Ty(), Args: IntptrTy);
932
933	MsanInstrumentAsmStoreFn =
934	M.getOrInsertFunction(Name: "__msan_instrument_asm_store", RetTy: IRB.getVoidTy(),
935	Args: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0), Args: IntptrTy);
936
937	if (CompileKernel) {
938	createKernelApi(M, TLI);
939	} else {
940	createUserspaceApi(M, TLI);
941	}
942	CallbacksInitialized = true;
943	}
944
945	FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
946	int size) {
947	FunctionCallee *Fns =
948	isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
949	switch (size) {
950	case 1:
951	return Fns[0];
952	case 2:
953	return Fns[1];
954	case 4:
955	return Fns[2];
956	case 8:
957	return Fns[3];
958	default:
959	return nullptr;
960	}
961	}
962
963	/// Module-level initialization.
964	///
965	/// inserts a call to __msan_init to the module's constructor list.
966	void MemorySanitizer::initializeModule(Module &M) {
967	auto &DL = M.getDataLayout();
968
969	TargetTriple = Triple(M.getTargetTriple());
970
971	bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
972	bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
973	// Check the overrides first
974	if (ShadowPassed || OriginPassed) {
975	CustomMapParams.AndMask = ClAndMask;
976	CustomMapParams.XorMask = ClXorMask;
977	CustomMapParams.ShadowBase = ClShadowBase;
978	CustomMapParams.OriginBase = ClOriginBase;
979	MapParams = &CustomMapParams;
980	} else {
981	switch (TargetTriple.getOS()) {
982	case Triple::FreeBSD:
983	switch (TargetTriple.getArch()) {
984	case Triple::aarch64:
985	MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
986	break;
987	case Triple::x86_64:
988	MapParams = FreeBSD_X86_MemoryMapParams.bits64;
989	break;
990	case Triple::x86:
991	MapParams = FreeBSD_X86_MemoryMapParams.bits32;
992	break;
993	default:
994	report_fatal_error(reason: "unsupported architecture");
995	}
996	break;
997	case Triple::NetBSD:
998	switch (TargetTriple.getArch()) {
999	case Triple::x86_64:
1000	MapParams = NetBSD_X86_MemoryMapParams.bits64;
1001	break;
1002	default:
1003	report_fatal_error(reason: "unsupported architecture");
1004	}
1005	break;
1006	case Triple::Linux:
1007	switch (TargetTriple.getArch()) {
1008	case Triple::x86_64:
1009	MapParams = Linux_X86_MemoryMapParams.bits64;
1010	break;
1011	case Triple::x86:
1012	MapParams = Linux_X86_MemoryMapParams.bits32;
1013	break;
1014	case Triple::mips64:
1015	case Triple::mips64el:
1016	MapParams = Linux_MIPS_MemoryMapParams.bits64;
1017	break;
1018	case Triple::ppc64:
1019	case Triple::ppc64le:
1020	MapParams = Linux_PowerPC_MemoryMapParams.bits64;
1021	break;
1022	case Triple::systemz:
1023	MapParams = Linux_S390_MemoryMapParams.bits64;
1024	break;
1025	case Triple::aarch64:
1026	case Triple::aarch64_be:
1027	MapParams = Linux_ARM_MemoryMapParams.bits64;
1028	break;
1029	case Triple::loongarch64:
1030	MapParams = Linux_LoongArch_MemoryMapParams.bits64;
1031	break;
1032	default:
1033	report_fatal_error(reason: "unsupported architecture");
1034	}
1035	break;
1036	default:
1037	report_fatal_error(reason: "unsupported operating system");
1038	}
1039	}
1040
1041	C = &(M.getContext());
1042	IRBuilder<> IRB(*C);
1043	IntptrTy = IRB.getIntPtrTy(DL);
1044	OriginTy = IRB.getInt32Ty();
1045	PtrTy = IRB.getPtrTy();
1046
1047	ColdCallWeights = MDBuilder(*C).createUnlikelyBranchWeights();
1048	OriginStoreWeights = MDBuilder(*C).createUnlikelyBranchWeights();
1049
1050	if (!CompileKernel) {
1051	if (TrackOrigins)
1052	M.getOrInsertGlobal(Name: "__msan_track_origins", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
1053	return new GlobalVariable(
1054	M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1055	IRB.getInt32(C: TrackOrigins), "__msan_track_origins");
1056	});
1057
1058	if (Recover)
1059	M.getOrInsertGlobal(Name: "__msan_keep_going", Ty: IRB.getInt32Ty(), CreateGlobalCallback: [&] {
1060	return new GlobalVariable(M, IRB.getInt32Ty(), true,
1061	GlobalValue::WeakODRLinkage,
1062	IRB.getInt32(C: Recover), "__msan_keep_going");
1063	});
1064	}
1065	}
1066
1067	namespace {
1068
1069	/// A helper class that handles instrumentation of VarArg
1070	/// functions on a particular platform.
1071	///
1072	/// Implementations are expected to insert the instrumentation
1073	/// necessary to propagate argument shadow through VarArg function
1074	/// calls. Visit* methods are called during an InstVisitor pass over
1075	/// the function, and should avoid creating new basic blocks. A new
1076	/// instance of this class is created for each instrumented function.
1077	struct VarArgHelper {
1078	virtual ~VarArgHelper() = default;
1079
1080	/// Visit a CallBase.
1081	virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1082
1083	/// Visit a va_start call.
1084	virtual void visitVAStartInst(VAStartInst &I) = 0;
1085
1086	/// Visit a va_copy call.
1087	virtual void visitVACopyInst(VACopyInst &I) = 0;
1088
1089	/// Finalize function instrumentation.
1090	///
1091	/// This method is called after visiting all interesting (see above)
1092	/// instructions in a function.
1093	virtual void finalizeInstrumentation() = 0;
1094	};
1095
1096	struct MemorySanitizerVisitor;
1097
1098	} // end anonymous namespace
1099
1100	static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1101	MemorySanitizerVisitor &Visitor);
1102
1103	static unsigned TypeSizeToSizeIndex(TypeSize TS) {
1104	if (TS.isScalable())
1105	// Scalable types unconditionally take slowpaths.
1106	return kNumberOfAccessSizes;
1107	unsigned TypeSizeFixed = TS.getFixedValue();
1108	if (TypeSizeFixed <= 8)
1109	return 0;
1110	return Log2_32_Ceil(Value: (TypeSizeFixed + 7) / 8);
1111	}
1112
1113	namespace {
1114
1115	/// Helper class to attach debug information of the given instruction onto new
1116	/// instructions inserted after.
1117	class NextNodeIRBuilder : public IRBuilder<> {
1118	public:
1119	explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1120	SetCurrentDebugLocation(IP->getDebugLoc());
1121	}
1122	};
1123
1124	/// This class does all the work for a given function. Store and Load
1125	/// instructions store and load corresponding shadow and origin
1126	/// values. Most instructions propagate shadow from arguments to their
1127	/// return values. Certain instructions (most importantly, BranchInst)
1128	/// test their argument shadow and print reports (with a runtime call) if it's
1129	/// non-zero.
1130	struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1131	Function &F;
1132	MemorySanitizer &MS;
1133	SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1134	ValueMap<Value *, Value *> ShadowMap, OriginMap;
1135	std::unique_ptr<VarArgHelper> VAHelper;
1136	const TargetLibraryInfo *TLI;
1137	Instruction *FnPrologueEnd;
1138
1139	// The following flags disable parts of MSan instrumentation based on
1140	// exclusion list contents and command-line options.
1141	bool InsertChecks;
1142	bool PropagateShadow;
1143	bool PoisonStack;
1144	bool PoisonUndef;
1145
1146	struct ShadowOriginAndInsertPoint {
1147	Value *Shadow;
1148	Value *Origin;
1149	Instruction *OrigIns;
1150
1151	ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1152	: Shadow(S), Origin(O), OrigIns(I) {}
1153	};
1154	SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1155	DenseMap<const DILocation *, int> LazyWarningDebugLocationCount;
1156	bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1157	SmallSetVector<AllocaInst *, 16> AllocaSet;
1158	SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1159	SmallVector<StoreInst *, 16> StoreList;
1160	int64_t SplittableBlocksCount = 0;
1161
1162	MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1163	const TargetLibraryInfo &TLI)
1164	: F(F), MS(MS), VAHelper(CreateVarArgHelper(Func&: F, Msan&: MS, Visitor&: *this)), TLI(&TLI) {
1165	bool SanitizeFunction =
1166	F.hasFnAttribute(Attribute::SanitizeMemory) && !ClDisableChecks;
1167	InsertChecks = SanitizeFunction;
1168	PropagateShadow = SanitizeFunction;
1169	PoisonStack = SanitizeFunction && ClPoisonStack;
1170	PoisonUndef = SanitizeFunction && ClPoisonUndef;
1171
1172	// In the presence of unreachable blocks, we may see Phi nodes with
1173	// incoming nodes from such blocks. Since InstVisitor skips unreachable
1174	// blocks, such nodes will not have any shadow value associated with them.
1175	// It's easier to remove unreachable blocks than deal with missing shadow.
1176	removeUnreachableBlocks(F);
1177
1178	MS.initializeCallbacks(M&: *F.getParent(), TLI);
1179	FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
1180	.CreateIntrinsic(Intrinsic::donothing, {}, {});
1181
1182	if (MS.CompileKernel) {
1183	IRBuilder<> IRB(FnPrologueEnd);
1184	insertKmsanPrologue(IRB);
1185	}
1186
1187	LLVM_DEBUG(if (!InsertChecks) dbgs()
1188	<< "MemorySanitizer is not inserting checks into '"
1189	<< F.getName() << "'\n");
1190	}
1191
1192	bool instrumentWithCalls(Value *V) {
1193	// Constants likely will be eliminated by follow-up passes.
1194	if (isa<Constant>(Val: V))
1195	return false;
1196
1197	++SplittableBlocksCount;
1198	return ClInstrumentationWithCallThreshold >= 0 &&
1199	SplittableBlocksCount > ClInstrumentationWithCallThreshold;
1200	}
1201
1202	bool isInPrologue(Instruction &I) {
1203	return I.getParent() == FnPrologueEnd->getParent() &&
1204	(&I == FnPrologueEnd || I.comesBefore(Other: FnPrologueEnd));
1205	}
1206
1207	// Creates a new origin and records the stack trace. In general we can call
1208	// this function for any origin manipulation we like. However it will cost
1209	// runtime resources. So use this wisely only if it can provide additional
1210	// information helpful to a user.
1211	Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1212	if (MS.TrackOrigins <= 1)
1213	return V;
1214	return IRB.CreateCall(Callee: MS.MsanChainOriginFn, Args: V);
1215	}
1216
1217	Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1218	const DataLayout &DL = F.getParent()->getDataLayout();
1219	unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1220	if (IntptrSize == kOriginSize)
1221	return Origin;
1222	assert(IntptrSize == kOriginSize * 2);
1223	Origin = IRB.CreateIntCast(V: Origin, DestTy: MS.IntptrTy, /* isSigned */ false);
1224	return IRB.CreateOr(LHS: Origin, RHS: IRB.CreateShl(LHS: Origin, RHS: kOriginSize * 8));
1225	}
1226
1227	/// Fill memory range with the given origin value.
1228	void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1229	TypeSize TS, Align Alignment) {
1230	const DataLayout &DL = F.getParent()->getDataLayout();
1231	const Align IntptrAlignment = DL.getABITypeAlign(Ty: MS.IntptrTy);
1232	unsigned IntptrSize = DL.getTypeStoreSize(Ty: MS.IntptrTy);
1233	assert(IntptrAlignment >= kMinOriginAlignment);
1234	assert(IntptrSize >= kOriginSize);
1235
1236	// Note: The loop based formation works for fixed length vectors too,
1237	// however we prefer to unroll and specialize alignment below.
1238	if (TS.isScalable()) {
1239	Value *Size = IRB.CreateTypeSize(DstType: IRB.getInt32Ty(), Size: TS);
1240	Value *RoundUp = IRB.CreateAdd(LHS: Size, RHS: IRB.getInt32(C: kOriginSize - 1));
1241	Value *End = IRB.CreateUDiv(LHS: RoundUp, RHS: IRB.getInt32(C: kOriginSize));
1242	auto [InsertPt, Index] =
1243	SplitBlockAndInsertSimpleForLoop(End, SplitBefore: &*IRB.GetInsertPoint());
1244	IRB.SetInsertPoint(InsertPt);
1245
1246	Value *GEP = IRB.CreateGEP(Ty: MS.OriginTy, Ptr: OriginPtr, IdxList: Index);
1247	IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: kMinOriginAlignment);
1248	return;
1249	}
1250
1251	unsigned Size = TS.getFixedValue();
1252
1253	unsigned Ofs = 0;
1254	Align CurrentAlignment = Alignment;
1255	if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1256	Value *IntptrOrigin = originToIntptr(IRB, Origin);
1257	Value *IntptrOriginPtr =
1258	IRB.CreatePointerCast(V: OriginPtr, DestTy: PointerType::get(ElementType: MS.IntptrTy, AddressSpace: 0));
1259	for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1260	Value *Ptr = i ? IRB.CreateConstGEP1_32(Ty: MS.IntptrTy, Ptr: IntptrOriginPtr, Idx0: i)
1261	: IntptrOriginPtr;
1262	IRB.CreateAlignedStore(Val: IntptrOrigin, Ptr, Align: CurrentAlignment);
1263	Ofs += IntptrSize / kOriginSize;
1264	CurrentAlignment = IntptrAlignment;
1265	}
1266	}
1267
1268	for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1269	Value *GEP =
1270	i ? IRB.CreateConstGEP1_32(Ty: MS.OriginTy, Ptr: OriginPtr, Idx0: i) : OriginPtr;
1271	IRB.CreateAlignedStore(Val: Origin, Ptr: GEP, Align: CurrentAlignment);
1272	CurrentAlignment = kMinOriginAlignment;
1273	}
1274	}
1275
1276	void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1277	Value *OriginPtr, Align Alignment) {
1278	const DataLayout &DL = F.getParent()->getDataLayout();
1279	const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1280	TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
1281	Value *ConvertedShadow = convertShadowToScalar(V: Shadow, IRB);
1282	if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1283	if (!ClCheckConstantShadow || ConstantShadow->isZeroValue()) {
1284	// Origin is not needed: value is initialized or const shadow is
1285	// ignored.
1286	return;
1287	}
1288	if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1289	// Copy origin as the value is definitely uninitialized.
1290	paintOrigin(IRB, Origin: updateOrigin(V: Origin, IRB), OriginPtr, TS: StoreSize,
1291	Alignment: OriginAlignment);
1292	return;
1293	}
1294	// Fallback to runtime check, which still can be optimized out later.
1295	}
1296
1297	TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1298	unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1299	if (instrumentWithCalls(V: ConvertedShadow) &&
1300	SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1301	FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1302	Value *ConvertedShadow2 =
1303	IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1304	CallBase *CB = IRB.CreateCall(Callee: Fn, Args: {ConvertedShadow2, Addr, Origin});
1305	CB->addParamAttr(0, Attribute::ZExt);
1306	CB->addParamAttr(2, Attribute::ZExt);
1307	} else {
1308	Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1309	Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1310	Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(), Unreachable: false, BranchWeights: MS.OriginStoreWeights);
1311	IRBuilder<> IRBNew(CheckTerm);
1312	paintOrigin(IRB&: IRBNew, Origin: updateOrigin(V: Origin, IRB&: IRBNew), OriginPtr, TS: StoreSize,
1313	Alignment: OriginAlignment);
1314	}
1315	}
1316
1317	void materializeStores() {
1318	for (StoreInst *SI : StoreList) {
1319	IRBuilder<> IRB(SI);
1320	Value *Val = SI->getValueOperand();
1321	Value *Addr = SI->getPointerOperand();
1322	Value *Shadow = SI->isAtomic() ? getCleanShadow(V: Val) : getShadow(V: Val);
1323	Value *ShadowPtr, *OriginPtr;
1324	Type *ShadowTy = Shadow->getType();
1325	const Align Alignment = SI->getAlign();
1326	const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
1327	std::tie(args&: ShadowPtr, args&: OriginPtr) =
1328	getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1329
1330	StoreInst *NewSI = IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Alignment);
1331	LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1332	(void)NewSI;
1333
1334	if (SI->isAtomic())
1335	SI->setOrdering(addReleaseOrdering(a: SI->getOrdering()));
1336
1337	if (MS.TrackOrigins && !SI->isAtomic())
1338	storeOrigin(IRB, Addr, Shadow, Origin: getOrigin(V: Val), OriginPtr,
1339	Alignment: OriginAlignment);
1340	}
1341	}
1342
1343	// Returns true if Debug Location corresponds to multiple warnings.
1344	bool shouldDisambiguateWarningLocation(const DebugLoc &DebugLoc) {
1345	if (MS.TrackOrigins < 2)
1346	return false;
1347
1348	if (LazyWarningDebugLocationCount.empty())
1349	for (const auto &I : InstrumentationList)
1350	++LazyWarningDebugLocationCount[I.OrigIns->getDebugLoc()];
1351
1352	return LazyWarningDebugLocationCount[DebugLoc] >= ClDisambiguateWarning;
1353	}
1354
1355	/// Helper function to insert a warning at IRB's current insert point.
1356	void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1357	if (!Origin)
1358	Origin = (Value *)IRB.getInt32(C: 0);
1359	assert(Origin->getType()->isIntegerTy());
1360
1361	if (shouldDisambiguateWarningLocation(DebugLoc: IRB.getCurrentDebugLocation())) {
1362	// Try to create additional origin with debug info of the last origin
1363	// instruction. It may provide additional information to the user.
1364	if (Instruction *OI = dyn_cast_or_null<Instruction>(Val: Origin)) {
1365	assert(MS.TrackOrigins);
1366	auto NewDebugLoc = OI->getDebugLoc();
1367	// Origin update with missing or the same debug location provides no
1368	// additional value.
1369	if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1370	// Insert update just before the check, so we call runtime only just
1371	// before the report.
1372	IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1373	IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1374	Origin = updateOrigin(V: Origin, IRB&: IRBOrigin);
1375	}
1376	}
1377	}
1378
1379	if (MS.CompileKernel || MS.TrackOrigins)
1380	IRB.CreateCall(Callee: MS.WarningFn, Args: Origin)->setCannotMerge();
1381	else
1382	IRB.CreateCall(Callee: MS.WarningFn)->setCannotMerge();
1383	// FIXME: Insert UnreachableInst if !MS.Recover?
1384	// This may invalidate some of the following checks and needs to be done
1385	// at the very end.
1386	}
1387
1388	void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1389	Value *Origin) {
1390	const DataLayout &DL = F.getParent()->getDataLayout();
1391	TypeSize TypeSizeInBits = DL.getTypeSizeInBits(Ty: ConvertedShadow->getType());
1392	unsigned SizeIndex = TypeSizeToSizeIndex(TS: TypeSizeInBits);
1393	if (instrumentWithCalls(V: ConvertedShadow) &&
1394	SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1395	FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1396	Value *ConvertedShadow2 =
1397	IRB.CreateZExt(V: ConvertedShadow, DestTy: IRB.getIntNTy(N: 8 * (1 << SizeIndex)));
1398	CallBase *CB = IRB.CreateCall(
1399	Callee: Fn, Args: {ConvertedShadow2,
1400	MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(C: 0)});
1401	CB->addParamAttr(0, Attribute::ZExt);
1402	CB->addParamAttr(1, Attribute::ZExt);
1403	} else {
1404	Value *Cmp = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1405	Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1406	Cond: Cmp, SplitBefore: &*IRB.GetInsertPoint(),
1407	/* Unreachable */ !MS.Recover, BranchWeights: MS.ColdCallWeights);
1408
1409	IRB.SetInsertPoint(CheckTerm);
1410	insertWarningFn(IRB, Origin);
1411	LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1412	}
1413	}
1414
1415	void materializeInstructionChecks(
1416	ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1417	const DataLayout &DL = F.getParent()->getDataLayout();
1418	// Disable combining in some cases. TrackOrigins checks each shadow to pick
1419	// correct origin.
1420	bool Combine = !MS.TrackOrigins;
1421	Instruction *Instruction = InstructionChecks.front().OrigIns;
1422	Value *Shadow = nullptr;
1423	for (const auto &ShadowData : InstructionChecks) {
1424	assert(ShadowData.OrigIns == Instruction);
1425	IRBuilder<> IRB(Instruction);
1426
1427	Value *ConvertedShadow = ShadowData.Shadow;
1428
1429	if (auto *ConstantShadow = dyn_cast<Constant>(Val: ConvertedShadow)) {
1430	if (!ClCheckConstantShadow || ConstantShadow->isZeroValue()) {
1431	// Skip, value is initialized or const shadow is ignored.
1432	continue;
1433	}
1434	if (llvm::isKnownNonZero(V: ConvertedShadow, Q: DL)) {
1435	// Report as the value is definitely uninitialized.
1436	insertWarningFn(IRB, Origin: ShadowData.Origin);
1437	if (!MS.Recover)
1438	return; // Always fail and stop here, not need to check the rest.
1439	// Skip entire instruction,
1440	continue;
1441	}
1442	// Fallback to runtime check, which still can be optimized out later.
1443	}
1444
1445	if (!Combine) {
1446	materializeOneCheck(IRB, ConvertedShadow, Origin: ShadowData.Origin);
1447	continue;
1448	}
1449
1450	if (!Shadow) {
1451	Shadow = ConvertedShadow;
1452	continue;
1453	}
1454
1455	Shadow = convertToBool(V: Shadow, IRB, name: "_mscmp");
1456	ConvertedShadow = convertToBool(V: ConvertedShadow, IRB, name: "_mscmp");
1457	Shadow = IRB.CreateOr(LHS: Shadow, RHS: ConvertedShadow, Name: "_msor");
1458	}
1459
1460	if (Shadow) {
1461	assert(Combine);
1462	IRBuilder<> IRB(Instruction);
1463	materializeOneCheck(IRB, ConvertedShadow: Shadow, Origin: nullptr);
1464	}
1465	}
1466
1467	void materializeChecks() {
1468	#ifndef NDEBUG
1469	// For assert below.
1470	SmallPtrSet<Instruction *, 16> Done;
1471	#endif
1472
1473	for (auto I = InstrumentationList.begin();
1474	I != InstrumentationList.end();) {
1475	auto OrigIns = I->OrigIns;
1476	// Checks are grouped by the original instruction. We call all
1477	// `insertShadowCheck` for an instruction at once.
1478	assert(Done.insert(OrigIns).second);
1479	auto J = std::find_if(first: I + 1, last: InstrumentationList.end(),
1480	pred: [OrigIns](const ShadowOriginAndInsertPoint &R) {
1481	return OrigIns != R.OrigIns;
1482	});
1483	// Process all checks of instruction at once.
1484	materializeInstructionChecks(InstructionChecks: ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1485	I = J;
1486	}
1487
1488	LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1489	}
1490
1491	// Returns the last instruction in the new prologue
1492	void insertKmsanPrologue(IRBuilder<> &IRB) {
1493	Value *ContextState = IRB.CreateCall(Callee: MS.MsanGetContextStateFn, Args: {});
1494	Constant *Zero = IRB.getInt32(C: 0);
1495	MS.ParamTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1496	IdxList: {Zero, IRB.getInt32(C: 0)}, Name: "param_shadow");
1497	MS.RetvalTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1498	IdxList: {Zero, IRB.getInt32(C: 1)}, Name: "retval_shadow");
1499	MS.VAArgTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1500	IdxList: {Zero, IRB.getInt32(C: 2)}, Name: "va_arg_shadow");
1501	MS.VAArgOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1502	IdxList: {Zero, IRB.getInt32(C: 3)}, Name: "va_arg_origin");
1503	MS.VAArgOverflowSizeTLS =
1504	IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1505	IdxList: {Zero, IRB.getInt32(C: 4)}, Name: "va_arg_overflow_size");
1506	MS.ParamOriginTLS = IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1507	IdxList: {Zero, IRB.getInt32(C: 5)}, Name: "param_origin");
1508	MS.RetvalOriginTLS =
1509	IRB.CreateGEP(Ty: MS.MsanContextStateTy, Ptr: ContextState,
1510	IdxList: {Zero, IRB.getInt32(C: 6)}, Name: "retval_origin");
1511	if (MS.TargetTriple.getArch() == Triple::systemz)
1512	MS.MsanMetadataAlloca = IRB.CreateAlloca(Ty: MS.MsanMetadata, AddrSpace: 0u);
1513	}
1514
1515	/// Add MemorySanitizer instrumentation to a function.
1516	bool runOnFunction() {
1517	// Iterate all BBs in depth-first order and create shadow instructions
1518	// for all instructions (where applicable).
1519	// For PHI nodes we create dummy shadow PHIs which will be finalized later.
1520	for (BasicBlock *BB : depth_first(G: FnPrologueEnd->getParent()))
1521	visit(BB&: *BB);
1522
1523	// Finalize PHI nodes.
1524	for (PHINode *PN : ShadowPHINodes) {
1525	PHINode *PNS = cast<PHINode>(Val: getShadow(V: PN));
1526	PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(Val: getOrigin(V: PN)) : nullptr;
1527	size_t NumValues = PN->getNumIncomingValues();
1528	for (size_t v = 0; v < NumValues; v++) {
1529	PNS->addIncoming(V: getShadow(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1530	if (PNO)
1531	PNO->addIncoming(V: getOrigin(I: PN, i: v), BB: PN->getIncomingBlock(i: v));
1532	}
1533	}
1534
1535	VAHelper->finalizeInstrumentation();
1536
1537	// Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1538	// instrumenting only allocas.
1539	if (InstrumentLifetimeStart) {
1540	for (auto Item : LifetimeStartList) {
1541	instrumentAlloca(I&: *Item.second, InsPoint: Item.first);
1542	AllocaSet.remove(X: Item.second);
1543	}
1544	}
1545	// Poison the allocas for which we didn't instrument the corresponding
1546	// lifetime intrinsics.
1547	for (AllocaInst *AI : AllocaSet)
1548	instrumentAlloca(I&: *AI);
1549
1550	// Insert shadow value checks.
1551	materializeChecks();
1552
1553	// Delayed instrumentation of StoreInst.
1554	// This may not add new address checks.
1555	materializeStores();
1556
1557	return true;
1558	}
1559
1560	/// Compute the shadow type that corresponds to a given Value.
1561	Type *getShadowTy(Value *V) { return getShadowTy(OrigTy: V->getType()); }
1562
1563	/// Compute the shadow type that corresponds to a given Type.
1564	Type *getShadowTy(Type *OrigTy) {
1565	if (!OrigTy->isSized()) {
1566	return nullptr;
1567	}
1568	// For integer type, shadow is the same as the original type.
1569	// This may return weird-sized types like i1.
1570	if (IntegerType *IT = dyn_cast<IntegerType>(Val: OrigTy))
1571	return IT;
1572	const DataLayout &DL = F.getParent()->getDataLayout();
1573	if (VectorType *VT = dyn_cast<VectorType>(Val: OrigTy)) {
1574	uint32_t EltSize = DL.getTypeSizeInBits(Ty: VT->getElementType());
1575	return VectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSize),
1576	EC: VT->getElementCount());
1577	}
1578	if (ArrayType *AT = dyn_cast<ArrayType>(Val: OrigTy)) {
1579	return ArrayType::get(ElementType: getShadowTy(OrigTy: AT->getElementType()),
1580	NumElements: AT->getNumElements());
1581	}
1582	if (StructType *ST = dyn_cast<StructType>(Val: OrigTy)) {
1583	SmallVector<Type *, 4> Elements;
1584	for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1585	Elements.push_back(Elt: getShadowTy(OrigTy: ST->getElementType(N: i)));
1586	StructType *Res = StructType::get(Context&: *MS.C, Elements, isPacked: ST->isPacked());
1587	LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1588	return Res;
1589	}
1590	uint32_t TypeSize = DL.getTypeSizeInBits(Ty: OrigTy);
1591	return IntegerType::get(C&: *MS.C, NumBits: TypeSize);
1592	}
1593
1594	/// Extract combined shadow of struct elements as a bool
1595	Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1596	IRBuilder<> &IRB) {
1597	Value *FalseVal = IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1598	Value *Aggregator = FalseVal;
1599
1600	for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1601	// Combine by ORing together each element's bool shadow
1602	Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1603	Value *ShadowBool = convertToBool(V: ShadowItem, IRB);
1604
1605	if (Aggregator != FalseVal)
1606	Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowBool);
1607	else
1608	Aggregator = ShadowBool;
1609	}
1610
1611	return Aggregator;
1612	}
1613
1614	// Extract combined shadow of array elements
1615	Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1616	IRBuilder<> &IRB) {
1617	if (!Array->getNumElements())
1618	return IRB.getIntN(/* width */ N: 1, /* value */ C: 0);
1619
1620	Value *FirstItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: 0);
1621	Value *Aggregator = convertShadowToScalar(V: FirstItem, IRB);
1622
1623	for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1624	Value *ShadowItem = IRB.CreateExtractValue(Agg: Shadow, Idxs: Idx);
1625	Value *ShadowInner = convertShadowToScalar(V: ShadowItem, IRB);
1626	Aggregator = IRB.CreateOr(LHS: Aggregator, RHS: ShadowInner);
1627	}
1628	return Aggregator;
1629	}
1630
1631	/// Convert a shadow value to it's flattened variant. The resulting
1632	/// shadow may not necessarily have the same bit width as the input
1633	/// value, but it will always be comparable to zero.
1634	Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1635	if (StructType *Struct = dyn_cast<StructType>(Val: V->getType()))
1636	return collapseStructShadow(Struct, Shadow: V, IRB);
1637	if (ArrayType *Array = dyn_cast<ArrayType>(Val: V->getType()))
1638	return collapseArrayShadow(Array, Shadow: V, IRB);
1639	if (isa<VectorType>(Val: V->getType())) {
1640	if (isa<ScalableVectorType>(Val: V->getType()))
1641	return convertShadowToScalar(V: IRB.CreateOrReduce(Src: V), IRB);
1642	unsigned BitWidth =
1643	V->getType()->getPrimitiveSizeInBits().getFixedValue();
1644	return IRB.CreateBitCast(V, DestTy: IntegerType::get(C&: *MS.C, NumBits: BitWidth));
1645	}
1646	return V;
1647	}
1648
1649	// Convert a scalar value to an i1 by comparing with 0
1650	Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1651	Type *VTy = V->getType();
1652	if (!VTy->isIntegerTy())
1653	return convertToBool(V: convertShadowToScalar(V, IRB), IRB, name);
1654	if (VTy->getIntegerBitWidth() == 1)
1655	// Just converting a bool to a bool, so do nothing.
1656	return V;
1657	return IRB.CreateICmpNE(LHS: V, RHS: ConstantInt::get(Ty: VTy, V: 0), Name: name);
1658	}
1659
1660	Type *ptrToIntPtrType(Type *PtrTy) const {
1661	if (VectorType *VectTy = dyn_cast<VectorType>(Val: PtrTy)) {
1662	return VectorType::get(ElementType: ptrToIntPtrType(PtrTy: VectTy->getElementType()),
1663	EC: VectTy->getElementCount());
1664	}
1665	assert(PtrTy->isIntOrPtrTy());
1666	return MS.IntptrTy;
1667	}
1668
1669	Type *getPtrToShadowPtrType(Type *IntPtrTy, Type *ShadowTy) const {
1670	if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1671	return VectorType::get(
1672	ElementType: getPtrToShadowPtrType(IntPtrTy: VectTy->getElementType(), ShadowTy),
1673	EC: VectTy->getElementCount());
1674	}
1675	assert(IntPtrTy == MS.IntptrTy);
1676	return PointerType::get(C&: *MS.C, AddressSpace: 0);
1677	}
1678
1679	Constant *constToIntPtr(Type *IntPtrTy, uint64_t C) const {
1680	if (VectorType *VectTy = dyn_cast<VectorType>(Val: IntPtrTy)) {
1681	return ConstantVector::getSplat(
1682	EC: VectTy->getElementCount(), Elt: constToIntPtr(IntPtrTy: VectTy->getElementType(), C));
1683	}
1684	assert(IntPtrTy == MS.IntptrTy);
1685	return ConstantInt::get(Ty: MS.IntptrTy, V: C);
1686	}
1687
1688	/// Compute the integer shadow offset that corresponds to a given
1689	/// application address.
1690	///
1691	/// Offset = (Addr & ~AndMask) ^ XorMask
1692	/// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1693	/// a single pointee.
1694	/// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1695	Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1696	Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1697	Value *OffsetLong = IRB.CreatePointerCast(V: Addr, DestTy: IntptrTy);
1698
1699	if (uint64_t AndMask = MS.MapParams->AndMask)
1700	OffsetLong = IRB.CreateAnd(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~AndMask));
1701
1702	if (uint64_t XorMask = MS.MapParams->XorMask)
1703	OffsetLong = IRB.CreateXor(LHS: OffsetLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: XorMask));
1704	return OffsetLong;
1705	}
1706
1707	/// Compute the shadow and origin addresses corresponding to a given
1708	/// application address.
1709	///
1710	/// Shadow = ShadowBase + Offset
1711	/// Origin = (OriginBase + Offset) & ~3ULL
1712	/// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1713	/// a single pointee.
1714	/// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1715	std::pair<Value *, Value *>
1716	getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1717	MaybeAlign Alignment) {
1718	VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1719	if (!VectTy) {
1720	assert(Addr->getType()->isPointerTy());
1721	} else {
1722	assert(VectTy->getElementType()->isPointerTy());
1723	}
1724	Type *IntptrTy = ptrToIntPtrType(PtrTy: Addr->getType());
1725	Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1726	Value *ShadowLong = ShadowOffset;
1727	if (uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1728	ShadowLong =
1729	IRB.CreateAdd(LHS: ShadowLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ShadowBase));
1730	}
1731	Value *ShadowPtr = IRB.CreateIntToPtr(
1732	V: ShadowLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy));
1733
1734	Value *OriginPtr = nullptr;
1735	if (MS.TrackOrigins) {
1736	Value *OriginLong = ShadowOffset;
1737	uint64_t OriginBase = MS.MapParams->OriginBase;
1738	if (OriginBase != 0)
1739	OriginLong =
1740	IRB.CreateAdd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: OriginBase));
1741	if (!Alignment || *Alignment < kMinOriginAlignment) {
1742	uint64_t Mask = kMinOriginAlignment.value() - 1;
1743	OriginLong = IRB.CreateAnd(LHS: OriginLong, RHS: constToIntPtr(IntPtrTy: IntptrTy, C: ~Mask));
1744	}
1745	OriginPtr = IRB.CreateIntToPtr(
1746	V: OriginLong, DestTy: getPtrToShadowPtrType(IntPtrTy: IntptrTy, ShadowTy: MS.OriginTy));
1747	}
1748	return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1749	}
1750
1751	template <typename... ArgsTy>
1752	Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
1753	ArgsTy... Args) {
1754	if (MS.TargetTriple.getArch() == Triple::systemz) {
1755	IRB.CreateCall(Callee,
1756	{MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1757	return IRB.CreateLoad(Ty: MS.MsanMetadata, Ptr: MS.MsanMetadataAlloca);
1758	}
1759
1760	return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1761	}
1762
1763	std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
1764	IRBuilder<> &IRB,
1765	Type *ShadowTy,
1766	bool isStore) {
1767	Value *ShadowOriginPtrs;
1768	const DataLayout &DL = F.getParent()->getDataLayout();
1769	TypeSize Size = DL.getTypeStoreSize(Ty: ShadowTy);
1770
1771	FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, size: Size);
1772	Value *AddrCast =
1773	IRB.CreatePointerCast(V: Addr, DestTy: PointerType::get(ElementType: IRB.getInt8Ty(), AddressSpace: 0));
1774	if (Getter) {
1775	ShadowOriginPtrs = createMetadataCall(IRB, Callee: Getter, Args: AddrCast);
1776	} else {
1777	Value *SizeVal = ConstantInt::get(Ty: MS.IntptrTy, V: Size);
1778	ShadowOriginPtrs = createMetadataCall(
1779	IRB,
1780	Callee: isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1781	Args: AddrCast, Args: SizeVal);
1782	}
1783	Value *ShadowPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 0);
1784	ShadowPtr = IRB.CreatePointerCast(V: ShadowPtr, DestTy: PointerType::get(ElementType: ShadowTy, AddressSpace: 0));
1785	Value *OriginPtr = IRB.CreateExtractValue(Agg: ShadowOriginPtrs, Idxs: 1);
1786
1787	return std::make_pair(x&: ShadowPtr, y&: OriginPtr);
1788	}
1789
1790	/// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1791	/// a single pointee.
1792	/// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
1793	std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1794	IRBuilder<> &IRB,
1795	Type *ShadowTy,
1796	bool isStore) {
1797	VectorType *VectTy = dyn_cast<VectorType>(Val: Addr->getType());
1798	if (!VectTy) {
1799	assert(Addr->getType()->isPointerTy());
1800	return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1801	}
1802
1803	// TODO: Support callbacs with vectors of addresses.
1804	unsigned NumElements = cast<FixedVectorType>(Val: VectTy)->getNumElements();
1805	Value *ShadowPtrs = ConstantInt::getNullValue(
1806	Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1807	Value *OriginPtrs = nullptr;
1808	if (MS.TrackOrigins)
1809	OriginPtrs = ConstantInt::getNullValue(
1810	Ty: FixedVectorType::get(ElementType: IRB.getPtrTy(), NumElts: NumElements));
1811	for (unsigned i = 0; i < NumElements; ++i) {
1812	Value *OneAddr =
1813	IRB.CreateExtractElement(Vec: Addr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1814	auto [ShadowPtr, OriginPtr] =
1815	getShadowOriginPtrKernelNoVec(Addr: OneAddr, IRB, ShadowTy, isStore);
1816
1817	ShadowPtrs = IRB.CreateInsertElement(
1818	Vec: ShadowPtrs, NewElt: ShadowPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1819	if (MS.TrackOrigins)
1820	OriginPtrs = IRB.CreateInsertElement(
1821	Vec: OriginPtrs, NewElt: OriginPtr, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
1822	}
1823	return {ShadowPtrs, OriginPtrs};
1824	}
1825
1826	std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1827	Type *ShadowTy,
1828	MaybeAlign Alignment,
1829	bool isStore) {
1830	if (MS.CompileKernel)
1831	return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1832	return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1833	}
1834
1835	/// Compute the shadow address for a given function argument.
1836	///
1837	/// Shadow = ParamTLS+ArgOffset.
1838	Value *getShadowPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1839	Value *Base = IRB.CreatePointerCast(V: MS.ParamTLS, DestTy: MS.IntptrTy);
1840	if (ArgOffset)
1841	Base = IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
1842	return IRB.CreateIntToPtr(V: Base, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msarg");
1843	}
1844
1845	/// Compute the origin address for a given function argument.
1846	Value *getOriginPtrForArgument(IRBuilder<> &IRB, int ArgOffset) {
1847	if (!MS.TrackOrigins)
1848	return nullptr;
1849	Value *Base = IRB.CreatePointerCast(V: MS.ParamOriginTLS, DestTy: MS.IntptrTy);
1850	if (ArgOffset)
1851	Base = IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
1852	return IRB.CreateIntToPtr(V: Base, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msarg_o");
1853	}
1854
1855	/// Compute the shadow address for a retval.
1856	Value *getShadowPtrForRetval(IRBuilder<> &IRB) {
1857	return IRB.CreatePointerCast(V: MS.RetvalTLS, DestTy: IRB.getPtrTy(AddrSpace: 0), Name: "_msret");
1858	}
1859
1860	/// Compute the origin address for a retval.
1861	Value *getOriginPtrForRetval() {
1862	// We keep a single origin for the entire retval. Might be too optimistic.
1863	return MS.RetvalOriginTLS;
1864	}
1865
1866	/// Set SV to be the shadow value for V.
1867	void setShadow(Value *V, Value *SV) {
1868	assert(!ShadowMap.count(V) && "Values may only have one shadow");
1869	ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1870	}
1871
1872	/// Set Origin to be the origin value for V.
1873	void setOrigin(Value *V, Value *Origin) {
1874	if (!MS.TrackOrigins)
1875	return;
1876	assert(!OriginMap.count(V) && "Values may only have one origin");
1877	LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1878	OriginMap[V] = Origin;
1879	}
1880
1881	Constant *getCleanShadow(Type *OrigTy) {
1882	Type *ShadowTy = getShadowTy(OrigTy);
1883	if (!ShadowTy)
1884	return nullptr;
1885	return Constant::getNullValue(Ty: ShadowTy);
1886	}
1887
1888	/// Create a clean shadow value for a given value.
1889	///
1890	/// Clean shadow (all zeroes) means all bits of the value are defined
1891	/// (initialized).
1892	Constant *getCleanShadow(Value *V) { return getCleanShadow(OrigTy: V->getType()); }
1893
1894	/// Create a dirty shadow of a given shadow type.
1895	Constant *getPoisonedShadow(Type *ShadowTy) {
1896	assert(ShadowTy);
1897	if (isa<IntegerType>(Val: ShadowTy) || isa<VectorType>(Val: ShadowTy))
1898	return Constant::getAllOnesValue(Ty: ShadowTy);
1899	if (ArrayType *AT = dyn_cast<ArrayType>(Val: ShadowTy)) {
1900	SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1901	getPoisonedShadow(ShadowTy: AT->getElementType()));
1902	return ConstantArray::get(T: AT, V: Vals);
1903	}
1904	if (StructType *ST = dyn_cast<StructType>(Val: ShadowTy)) {
1905	SmallVector<Constant *, 4> Vals;
1906	for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1907	Vals.push_back(Elt: getPoisonedShadow(ShadowTy: ST->getElementType(N: i)));
1908	return ConstantStruct::get(T: ST, V: Vals);
1909	}
1910	llvm_unreachable("Unexpected shadow type");
1911	}
1912
1913	/// Create a dirty shadow for a given value.
1914	Constant *getPoisonedShadow(Value *V) {
1915	Type *ShadowTy = getShadowTy(V);
1916	if (!ShadowTy)
1917	return nullptr;
1918	return getPoisonedShadow(ShadowTy);
1919	}
1920
1921	/// Create a clean (zero) origin.
1922	Value *getCleanOrigin() { return Constant::getNullValue(Ty: MS.OriginTy); }
1923
1924	/// Get the shadow value for a given Value.
1925	///
1926	/// This function either returns the value set earlier with setShadow,
1927	/// or extracts if from ParamTLS (for function arguments).
1928	Value *getShadow(Value *V) {
1929	if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
1930	if (!PropagateShadow || I->getMetadata(KindID: LLVMContext::MD_nosanitize))
1931	return getCleanShadow(V);
1932	// For instructions the shadow is already stored in the map.
1933	Value *Shadow = ShadowMap[V];
1934	if (!Shadow) {
1935	LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1936	(void)I;
1937	assert(Shadow && "No shadow for a value");
1938	}
1939	return Shadow;
1940	}
1941	if (UndefValue *U = dyn_cast<UndefValue>(Val: V)) {
1942	Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1943	: getCleanShadow(V);
1944	LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1945	(void)U;
1946	return AllOnes;
1947	}
1948	if (Argument *A = dyn_cast<Argument>(Val: V)) {
1949	// For arguments we compute the shadow on demand and store it in the map.
1950	Value *&ShadowPtr = ShadowMap[V];
1951	if (ShadowPtr)
1952	return ShadowPtr;
1953	Function *F = A->getParent();
1954	IRBuilder<> EntryIRB(FnPrologueEnd);
1955	unsigned ArgOffset = 0;
1956	const DataLayout &DL = F->getParent()->getDataLayout();
1957	for (auto &FArg : F->args()) {
1958	if (!FArg.getType()->isSized()) {
1959	LLVM_DEBUG(dbgs() << "Arg is not sized\n");
1960	continue;
1961	}
1962
1963	unsigned Size = FArg.hasByValAttr()
1964	? DL.getTypeAllocSize(Ty: FArg.getParamByValType())
1965	: DL.getTypeAllocSize(Ty: FArg.getType());
1966
1967	if (A == &FArg) {
1968	bool Overflow = ArgOffset + Size > kParamTLSSize;
1969	if (FArg.hasByValAttr()) {
1970	// ByVal pointer itself has clean shadow. We copy the actual
1971	// argument shadow to the underlying memory.
1972	// Figure out maximal valid memcpy alignment.
1973	const Align ArgAlign = DL.getValueOrABITypeAlignment(
1974	Alignment: FArg.getParamAlign(), Ty: FArg.getParamByValType());
1975	Value *CpShadowPtr, *CpOriginPtr;
1976	std::tie(args&: CpShadowPtr, args&: CpOriginPtr) =
1977	getShadowOriginPtr(Addr: V, IRB&: EntryIRB, ShadowTy: EntryIRB.getInt8Ty(), Alignment: ArgAlign,
1978	/*isStore*/ true);
1979	if (!PropagateShadow || Overflow) {
1980	// ParamTLS overflow.
1981	EntryIRB.CreateMemSet(
1982	Ptr: CpShadowPtr, Val: Constant::getNullValue(Ty: EntryIRB.getInt8Ty()),
1983	Size, Align: ArgAlign);
1984	} else {
1985	Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
1986	const Align CopyAlign = std::min(a: ArgAlign, b: kShadowTLSAlignment);
1987	Value *Cpy = EntryIRB.CreateMemCpy(Dst: CpShadowPtr, DstAlign: CopyAlign, Src: Base,
1988	SrcAlign: CopyAlign, Size);
1989	LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1990	(void)Cpy;
1991
1992	if (MS.TrackOrigins) {
1993	Value *OriginPtr =
1994	getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
1995	// FIXME: OriginSize should be:
1996	// alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
1997	unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
1998	EntryIRB.CreateMemCpy(
1999	Dst: CpOriginPtr,
2000	/* by getShadowOriginPtr */ DstAlign: kMinOriginAlignment, Src: OriginPtr,
2001	/* by origin_tls[ArgOffset] */ SrcAlign: kMinOriginAlignment,
2002	Size: OriginSize);
2003	}
2004	}
2005	}
2006
2007	if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2008	(MS.EagerChecks && FArg.hasAttribute(Attribute::Kind: NoUndef))) {
2009	ShadowPtr = getCleanShadow(V);
2010	setOrigin(V: A, Origin: getCleanOrigin());
2011	} else {
2012	// Shadow over TLS
2013	Value *Base = getShadowPtrForArgument(IRB&: EntryIRB, ArgOffset);
2014	ShadowPtr = EntryIRB.CreateAlignedLoad(Ty: getShadowTy(V: &FArg), Ptr: Base,
2015	Align: kShadowTLSAlignment);
2016	if (MS.TrackOrigins) {
2017	Value *OriginPtr =
2018	getOriginPtrForArgument(IRB&: EntryIRB, ArgOffset);
2019	setOrigin(V: A, Origin: EntryIRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
2020	}
2021	}
2022	LLVM_DEBUG(dbgs()
2023	<< " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
2024	break;
2025	}
2026
2027	ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
2028	}
2029	assert(ShadowPtr && "Could not find shadow for an argument");
2030	return ShadowPtr;
2031	}
2032	// For everything else the shadow is zero.
2033	return getCleanShadow(V);
2034	}
2035
2036	/// Get the shadow for i-th argument of the instruction I.
2037	Value *getShadow(Instruction *I, int i) {
2038	return getShadow(V: I->getOperand(i));
2039	}
2040
2041	/// Get the origin for a value.
2042	Value *getOrigin(Value *V) {
2043	if (!MS.TrackOrigins)
2044	return nullptr;
2045	if (!PropagateShadow || isa<Constant>(Val: V) || isa<InlineAsm>(Val: V))
2046	return getCleanOrigin();
2047	assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2048	"Unexpected value type in getOrigin()");
2049	if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
2050	if (I->getMetadata(KindID: LLVMContext::MD_nosanitize))
2051	return getCleanOrigin();
2052	}
2053	Value *Origin = OriginMap[V];
2054	assert(Origin && "Missing origin");
2055	return Origin;
2056	}
2057
2058	/// Get the origin for i-th argument of the instruction I.
2059	Value *getOrigin(Instruction *I, int i) {
2060	return getOrigin(V: I->getOperand(i));
2061	}
2062
2063	/// Remember the place where a shadow check should be inserted.
2064	///
2065	/// This location will be later instrumented with a check that will print a
2066	/// UMR warning in runtime if the shadow value is not 0.
2067	void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
2068	assert(Shadow);
2069	if (!InsertChecks)
2070	return;
2071
2072	if (!DebugCounter::shouldExecute(CounterName: DebugInsertCheck)) {
2073	LLVM_DEBUG(dbgs() << "Skipping check of " << *Shadow << " before "
2074	<< *OrigIns << "\n");
2075	return;
2076	}
2077	#ifndef NDEBUG
2078	Type *ShadowTy = Shadow->getType();
2079	assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2080	isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2081	"Can only insert checks for integer, vector, and aggregate shadow "
2082	"types");
2083	#endif
2084	InstrumentationList.push_back(
2085	Elt: ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2086	}
2087
2088	/// Remember the place where a shadow check should be inserted.
2089	///
2090	/// This location will be later instrumented with a check that will print a
2091	/// UMR warning in runtime if the value is not fully defined.
2092	void insertShadowCheck(Value *Val, Instruction *OrigIns) {
2093	assert(Val);
2094	Value *Shadow, *Origin;
2095	if (ClCheckConstantShadow) {
2096	Shadow = getShadow(V: Val);
2097	if (!Shadow)
2098	return;
2099	Origin = getOrigin(V: Val);
2100	} else {
2101	Shadow = dyn_cast_or_null<Instruction>(Val: getShadow(V: Val));
2102	if (!Shadow)
2103	return;
2104	Origin = dyn_cast_or_null<Instruction>(Val: getOrigin(V: Val));
2105	}
2106	insertShadowCheck(Shadow, Origin, OrigIns);
2107	}
2108
2109	AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
2110	switch (a) {
2111	case AtomicOrdering::NotAtomic:
2112	return AtomicOrdering::NotAtomic;
2113	case AtomicOrdering::Unordered:
2114	case AtomicOrdering::Monotonic:
2115	case AtomicOrdering::Release:
2116	return AtomicOrdering::Release;
2117	case AtomicOrdering::Acquire:
2118	case AtomicOrdering::AcquireRelease:
2119	return AtomicOrdering::AcquireRelease;
2120	case AtomicOrdering::SequentiallyConsistent:
2121	return AtomicOrdering::SequentiallyConsistent;
2122	}
2123	llvm_unreachable("Unknown ordering");
2124	}
2125
2126	Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
2127	constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2128	uint32_t OrderingTable[NumOrderings] = {};
2129
2130	OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2131	OrderingTable[(int)AtomicOrderingCABI::release] =
2132	(int)AtomicOrderingCABI::release;
2133	OrderingTable[(int)AtomicOrderingCABI::consume] =
2134	OrderingTable[(int)AtomicOrderingCABI::acquire] =
2135	OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2136	(int)AtomicOrderingCABI::acq_rel;
2137	OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2138	(int)AtomicOrderingCABI::seq_cst;
2139
2140	return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2141	}
2142
2143	AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
2144	switch (a) {
2145	case AtomicOrdering::NotAtomic:
2146	return AtomicOrdering::NotAtomic;
2147	case AtomicOrdering::Unordered:
2148	case AtomicOrdering::Monotonic:
2149	case AtomicOrdering::Acquire:
2150	return AtomicOrdering::Acquire;
2151	case AtomicOrdering::Release:
2152	case AtomicOrdering::AcquireRelease:
2153	return AtomicOrdering::AcquireRelease;
2154	case AtomicOrdering::SequentiallyConsistent:
2155	return AtomicOrdering::SequentiallyConsistent;
2156	}
2157	llvm_unreachable("Unknown ordering");
2158	}
2159
2160	Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
2161	constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2162	uint32_t OrderingTable[NumOrderings] = {};
2163
2164	OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2165	OrderingTable[(int)AtomicOrderingCABI::acquire] =
2166	OrderingTable[(int)AtomicOrderingCABI::consume] =
2167	(int)AtomicOrderingCABI::acquire;
2168	OrderingTable[(int)AtomicOrderingCABI::release] =
2169	OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2170	(int)AtomicOrderingCABI::acq_rel;
2171	OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2172	(int)AtomicOrderingCABI::seq_cst;
2173
2174	return ConstantDataVector::get(Context&: IRB.getContext(), Elts: OrderingTable);
2175	}
2176
2177	// ------------------- Visitors.
2178	using InstVisitor<MemorySanitizerVisitor>::visit;
2179	void visit(Instruction &I) {
2180	if (I.getMetadata(KindID: LLVMContext::MD_nosanitize))
2181	return;
2182	// Don't want to visit if we're in the prologue
2183	if (isInPrologue(I))
2184	return;
2185	if (!DebugCounter::shouldExecute(CounterName: DebugInstrumentInstruction)) {
2186	LLVM_DEBUG(dbgs() << "Skipping instruction: " << I << "\n");
2187	// We still need to set the shadow and origin to clean values.
2188	setShadow(V: &I, SV: getCleanShadow(V: &I));
2189	setOrigin(V: &I, Origin: getCleanOrigin());
2190	return;
2191	}
2192	InstVisitor<MemorySanitizerVisitor>::visit(I);
2193	}
2194
2195	/// Instrument LoadInst
2196	///
2197	/// Loads the corresponding shadow and (optionally) origin.
2198	/// Optionally, checks that the load address is fully defined.
2199	void visitLoadInst(LoadInst &I) {
2200	assert(I.getType()->isSized() && "Load type must have size");
2201	assert(!I.getMetadata(LLVMContext::MD_nosanitize));
2202	NextNodeIRBuilder IRB(&I);
2203	Type *ShadowTy = getShadowTy(V: &I);
2204	Value *Addr = I.getPointerOperand();
2205	Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2206	const Align Alignment = I.getAlign();
2207	if (PropagateShadow) {
2208	std::tie(args&: ShadowPtr, args&: OriginPtr) =
2209	getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2210	setShadow(V: &I,
2211	SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
2212	} else {
2213	setShadow(V: &I, SV: getCleanShadow(V: &I));
2214	}
2215
2216	if (ClCheckAccessAddress)
2217	insertShadowCheck(Val: I.getPointerOperand(), OrigIns: &I);
2218
2219	if (I.isAtomic())
2220	I.setOrdering(addAcquireOrdering(a: I.getOrdering()));
2221
2222	if (MS.TrackOrigins) {
2223	if (PropagateShadow) {
2224	const Align OriginAlignment = std::max(a: kMinOriginAlignment, b: Alignment);
2225	setOrigin(
2226	V: &I, Origin: IRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: OriginPtr, Align: OriginAlignment));
2227	} else {
2228	setOrigin(V: &I, Origin: getCleanOrigin());
2229	}
2230	}
2231	}
2232
2233	/// Instrument StoreInst
2234	///
2235	/// Stores the corresponding shadow and (optionally) origin.
2236	/// Optionally, checks that the store address is fully defined.
2237	void visitStoreInst(StoreInst &I) {
2238	StoreList.push_back(Elt: &I);
2239	if (ClCheckAccessAddress)
2240	insertShadowCheck(Val: I.getPointerOperand(), OrigIns: &I);
2241	}
2242
2243	void handleCASOrRMW(Instruction &I) {
2244	assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2245
2246	IRBuilder<> IRB(&I);
2247	Value *Addr = I.getOperand(i: 0);
2248	Value *Val = I.getOperand(i: 1);
2249	Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, ShadowTy: getShadowTy(V: Val), Alignment: Align(1),
2250	/*isStore*/ true)
2251	.first;
2252
2253	if (ClCheckAccessAddress)
2254	insertShadowCheck(Val: Addr, OrigIns: &I);
2255
2256	// Only test the conditional argument of cmpxchg instruction.
2257	// The other argument can potentially be uninitialized, but we can not
2258	// detect this situation reliably without possible false positives.
2259	if (isa<AtomicCmpXchgInst>(Val: I))
2260	insertShadowCheck(Val, OrigIns: &I);
2261
2262	IRB.CreateStore(Val: getCleanShadow(V: Val), Ptr: ShadowPtr);
2263
2264	setShadow(V: &I, SV: getCleanShadow(V: &I));
2265	setOrigin(V: &I, Origin: getCleanOrigin());
2266	}
2267
2268	void visitAtomicRMWInst(AtomicRMWInst &I) {
2269	handleCASOrRMW(I);
2270	I.setOrdering(addReleaseOrdering(a: I.getOrdering()));
2271	}
2272
2273	void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2274	handleCASOrRMW(I);
2275	I.setSuccessOrdering(addReleaseOrdering(a: I.getSuccessOrdering()));
2276	}
2277
2278	// Vector manipulation.
2279	void visitExtractElementInst(ExtractElementInst &I) {
2280	insertShadowCheck(Val: I.getOperand(i_nocapture: 1), OrigIns: &I);
2281	IRBuilder<> IRB(&I);
2282	setShadow(V: &I, SV: IRB.CreateExtractElement(Vec: getShadow(I: &I, i: 0), Idx: I.getOperand(i_nocapture: 1),
2283	Name: "_msprop"));
2284	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2285	}
2286
2287	void visitInsertElementInst(InsertElementInst &I) {
2288	insertShadowCheck(Val: I.getOperand(i_nocapture: 2), OrigIns: &I);
2289	IRBuilder<> IRB(&I);
2290	auto *Shadow0 = getShadow(I: &I, i: 0);
2291	auto *Shadow1 = getShadow(I: &I, i: 1);
2292	setShadow(V: &I, SV: IRB.CreateInsertElement(Vec: Shadow0, NewElt: Shadow1, Idx: I.getOperand(i_nocapture: 2),
2293	Name: "_msprop"));
2294	setOriginForNaryOp(I);
2295	}
2296
2297	void visitShuffleVectorInst(ShuffleVectorInst &I) {
2298	IRBuilder<> IRB(&I);
2299	auto *Shadow0 = getShadow(I: &I, i: 0);
2300	auto *Shadow1 = getShadow(I: &I, i: 1);
2301	setShadow(V: &I, SV: IRB.CreateShuffleVector(V1: Shadow0, V2: Shadow1, Mask: I.getShuffleMask(),
2302	Name: "_msprop"));
2303	setOriginForNaryOp(I);
2304	}
2305
2306	// Casts.
2307	void visitSExtInst(SExtInst &I) {
2308	IRBuilder<> IRB(&I);
2309	setShadow(V: &I, SV: IRB.CreateSExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2310	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2311	}
2312
2313	void visitZExtInst(ZExtInst &I) {
2314	IRBuilder<> IRB(&I);
2315	setShadow(V: &I, SV: IRB.CreateZExt(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2316	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2317	}
2318
2319	void visitTruncInst(TruncInst &I) {
2320	IRBuilder<> IRB(&I);
2321	setShadow(V: &I, SV: IRB.CreateTrunc(V: getShadow(I: &I, i: 0), DestTy: I.getType(), Name: "_msprop"));
2322	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2323	}
2324
2325	void visitBitCastInst(BitCastInst &I) {
2326	// Special case: if this is the bitcast (there is exactly 1 allowed) between
2327	// a musttail call and a ret, don't instrument. New instructions are not
2328	// allowed after a musttail call.
2329	if (auto *CI = dyn_cast<CallInst>(Val: I.getOperand(i_nocapture: 0)))
2330	if (CI->isMustTailCall())
2331	return;
2332	IRBuilder<> IRB(&I);
2333	setShadow(V: &I, SV: IRB.CreateBitCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I)));
2334	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2335	}
2336
2337	void visitPtrToIntInst(PtrToIntInst &I) {
2338	IRBuilder<> IRB(&I);
2339	setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2340	Name: "_msprop_ptrtoint"));
2341	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2342	}
2343
2344	void visitIntToPtrInst(IntToPtrInst &I) {
2345	IRBuilder<> IRB(&I);
2346	setShadow(V: &I, SV: IRB.CreateIntCast(V: getShadow(I: &I, i: 0), DestTy: getShadowTy(V: &I), isSigned: false,
2347	Name: "_msprop_inttoptr"));
2348	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2349	}
2350
2351	void visitFPToSIInst(CastInst &I) { handleShadowOr(I); }
2352	void visitFPToUIInst(CastInst &I) { handleShadowOr(I); }
2353	void visitSIToFPInst(CastInst &I) { handleShadowOr(I); }
2354	void visitUIToFPInst(CastInst &I) { handleShadowOr(I); }
2355	void visitFPExtInst(CastInst &I) { handleShadowOr(I); }
2356	void visitFPTruncInst(CastInst &I) { handleShadowOr(I); }
2357
2358	/// Propagate shadow for bitwise AND.
2359	///
2360	/// This code is exact, i.e. if, for example, a bit in the left argument
2361	/// is defined and 0, then neither the value not definedness of the
2362	/// corresponding bit in B don't affect the resulting shadow.
2363	void visitAnd(BinaryOperator &I) {
2364	IRBuilder<> IRB(&I);
2365	// "And" of 0 and a poisoned value results in unpoisoned value.
2366	// 1&1 => 1; 0&1 => 0; p&1 => p;
2367	// 1&0 => 0; 0&0 => 0; p&0 => 0;
2368	// 1&p => p; 0&p => 0; p&p => p;
2369	// S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2370	Value *S1 = getShadow(I: &I, i: 0);
2371	Value *S2 = getShadow(I: &I, i: 1);
2372	Value *V1 = I.getOperand(i_nocapture: 0);
2373	Value *V2 = I.getOperand(i_nocapture: 1);
2374	if (V1->getType() != S1->getType()) {
2375	V1 = IRB.CreateIntCast(V: V1, DestTy: S1->getType(), isSigned: false);
2376	V2 = IRB.CreateIntCast(V: V2, DestTy: S2->getType(), isSigned: false);
2377	}
2378	Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2379	Value *V1S2 = IRB.CreateAnd(LHS: V1, RHS: S2);
2380	Value *S1V2 = IRB.CreateAnd(LHS: S1, RHS: V2);
2381	setShadow(V: &I, SV: IRB.CreateOr(Ops: {S1S2, V1S2, S1V2}));
2382	setOriginForNaryOp(I);
2383	}
2384
2385	void visitOr(BinaryOperator &I) {
2386	IRBuilder<> IRB(&I);
2387	// "Or" of 1 and a poisoned value results in unpoisoned value.
2388	// 1|1 => 1; 0|1 => 1; p|1 => 1;
2389	// 1|0 => 1; 0|0 => 0; p|0 => p;
2390	// 1|p => 1; 0|p => p; p|p => p;
2391	// S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2392	Value *S1 = getShadow(I: &I, i: 0);
2393	Value *S2 = getShadow(I: &I, i: 1);
2394	Value *V1 = IRB.CreateNot(V: I.getOperand(i_nocapture: 0));
2395	Value *V2 = IRB.CreateNot(V: I.getOperand(i_nocapture: 1));
2396	if (V1->getType() != S1->getType()) {
2397	V1 = IRB.CreateIntCast(V: V1, DestTy: S1->getType(), isSigned: false);
2398	V2 = IRB.CreateIntCast(V: V2, DestTy: S2->getType(), isSigned: false);
2399	}
2400	Value *S1S2 = IRB.CreateAnd(LHS: S1, RHS: S2);
2401	Value *V1S2 = IRB.CreateAnd(LHS: V1, RHS: S2);
2402	Value *S1V2 = IRB.CreateAnd(LHS: S1, RHS: V2);
2403	setShadow(V: &I, SV: IRB.CreateOr(Ops: {S1S2, V1S2, S1V2}));
2404	setOriginForNaryOp(I);
2405	}
2406
2407	/// Default propagation of shadow and/or origin.
2408	///
2409	/// This class implements the general case of shadow propagation, used in all
2410	/// cases where we don't know and/or don't care about what the operation
2411	/// actually does. It converts all input shadow values to a common type
2412	/// (extending or truncating as necessary), and bitwise OR's them.
2413	///
2414	/// This is much cheaper than inserting checks (i.e. requiring inputs to be
2415	/// fully initialized), and less prone to false positives.
2416	///
2417	/// This class also implements the general case of origin propagation. For a
2418	/// Nary operation, result origin is set to the origin of an argument that is
2419	/// not entirely initialized. If there is more than one such arguments, the
2420	/// rightmost of them is picked. It does not matter which one is picked if all
2421	/// arguments are initialized.
2422	template <bool CombineShadow> class Combiner {
2423	Value *Shadow = nullptr;
2424	Value *Origin = nullptr;
2425	IRBuilder<> &IRB;
2426	MemorySanitizerVisitor *MSV;
2427
2428	public:
2429	Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2430	: IRB(IRB), MSV(MSV) {}
2431
2432	/// Add a pair of shadow and origin values to the mix.
2433	Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2434	if (CombineShadow) {
2435	assert(OpShadow);
2436	if (!Shadow)
2437	Shadow = OpShadow;
2438	else {
2439	OpShadow = MSV->CreateShadowCast(IRB, V: OpShadow, dstTy: Shadow->getType());
2440	Shadow = IRB.CreateOr(LHS: Shadow, RHS: OpShadow, Name: "_msprop");
2441	}
2442	}
2443
2444	if (MSV->MS.TrackOrigins) {
2445	assert(OpOrigin);
2446	if (!Origin) {
2447	Origin = OpOrigin;
2448	} else {
2449	Constant *ConstOrigin = dyn_cast<Constant>(Val: OpOrigin);
2450	// No point in adding something that might result in 0 origin value.
2451	if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2452	Value *Cond = MSV->convertToBool(V: OpShadow, IRB);
2453	Origin = IRB.CreateSelect(C: Cond, True: OpOrigin, False: Origin);
2454	}
2455	}
2456	}
2457	return *this;
2458	}
2459
2460	/// Add an application value to the mix.
2461	Combiner &Add(Value *V) {
2462	Value *OpShadow = MSV->getShadow(V);
2463	Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2464	return Add(OpShadow, OpOrigin);
2465	}
2466
2467	/// Set the current combined values as the given instruction's shadow
2468	/// and origin.
2469	void Done(Instruction *I) {
2470	if (CombineShadow) {
2471	assert(Shadow);
2472	Shadow = MSV->CreateShadowCast(IRB, V: Shadow, dstTy: MSV->getShadowTy(V: I));
2473	MSV->setShadow(V: I, SV: Shadow);
2474	}
2475	if (MSV->MS.TrackOrigins) {
2476	assert(Origin);
2477	MSV->setOrigin(V: I, Origin);
2478	}
2479	}
2480	};
2481
2482	using ShadowAndOriginCombiner = Combiner<true>;
2483	using OriginCombiner = Combiner<false>;
2484
2485	/// Propagate origin for arbitrary operation.
2486	void setOriginForNaryOp(Instruction &I) {
2487	if (!MS.TrackOrigins)
2488	return;
2489	IRBuilder<> IRB(&I);
2490	OriginCombiner OC(this, IRB);
2491	for (Use &Op : I.operands())
2492	OC.Add(V: Op.get());
2493	OC.Done(I: &I);
2494	}
2495
2496	size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2497	assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2498	"Vector of pointers is not a valid shadow type");
2499	return Ty->isVectorTy() ? cast<FixedVectorType>(Val: Ty)->getNumElements() *
2500	Ty->getScalarSizeInBits()
2501	: Ty->getPrimitiveSizeInBits();
2502	}
2503
2504	/// Cast between two shadow types, extending or truncating as
2505	/// necessary.
2506	Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2507	bool Signed = false) {
2508	Type *srcTy = V->getType();
2509	size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: srcTy);
2510	size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(Ty: dstTy);
2511	if (srcSizeInBits > 1 && dstSizeInBits == 1)
2512	return IRB.CreateICmpNE(LHS: V, RHS: getCleanShadow(V));
2513
2514	if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2515	return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2516	if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2517	cast<VectorType>(Val: dstTy)->getElementCount() ==
2518	cast<VectorType>(Val: srcTy)->getElementCount())
2519	return IRB.CreateIntCast(V, DestTy: dstTy, isSigned: Signed);
2520	Value *V1 = IRB.CreateBitCast(V, DestTy: Type::getIntNTy(C&: *MS.C, N: srcSizeInBits));
2521	Value *V2 =
2522	IRB.CreateIntCast(V: V1, DestTy: Type::getIntNTy(C&: *MS.C, N: dstSizeInBits), isSigned: Signed);
2523	return IRB.CreateBitCast(V: V2, DestTy: dstTy);
2524	// TODO: handle struct types.
2525	}
2526
2527	/// Cast an application value to the type of its own shadow.
2528	Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2529	Type *ShadowTy = getShadowTy(V);
2530	if (V->getType() == ShadowTy)
2531	return V;
2532	if (V->getType()->isPtrOrPtrVectorTy())
2533	return IRB.CreatePtrToInt(V, DestTy: ShadowTy);
2534	else
2535	return IRB.CreateBitCast(V, DestTy: ShadowTy);
2536	}
2537
2538	/// Propagate shadow for arbitrary operation.
2539	void handleShadowOr(Instruction &I) {
2540	IRBuilder<> IRB(&I);
2541	ShadowAndOriginCombiner SC(this, IRB);
2542	for (Use &Op : I.operands())
2543	SC.Add(V: Op.get());
2544	SC.Done(I: &I);
2545	}
2546
2547	void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2548
2549	// Handle multiplication by constant.
2550	//
2551	// Handle a special case of multiplication by constant that may have one or
2552	// more zeros in the lower bits. This makes corresponding number of lower bits
2553	// of the result zero as well. We model it by shifting the other operand
2554	// shadow left by the required number of bits. Effectively, we transform
2555	// (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2556	// We use multiplication by 2**N instead of shift to cover the case of
2557	// multiplication by 0, which may occur in some elements of a vector operand.
2558	void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2559	Value *OtherArg) {
2560	Constant *ShadowMul;
2561	Type *Ty = ConstArg->getType();
2562	if (auto *VTy = dyn_cast<VectorType>(Val: Ty)) {
2563	unsigned NumElements = cast<FixedVectorType>(Val: VTy)->getNumElements();
2564	Type *EltTy = VTy->getElementType();
2565	SmallVector<Constant *, 16> Elements;
2566	for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2567	if (ConstantInt *Elt =
2568	dyn_cast<ConstantInt>(Val: ConstArg->getAggregateElement(Elt: Idx))) {
2569	const APInt &V = Elt->getValue();
2570	APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
2571	Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: V2));
2572	} else {
2573	Elements.push_back(Elt: ConstantInt::get(Ty: EltTy, V: 1));
2574	}
2575	}
2576	ShadowMul = ConstantVector::get(V: Elements);
2577	} else {
2578	if (ConstantInt *Elt = dyn_cast<ConstantInt>(Val: ConstArg)) {
2579	const APInt &V = Elt->getValue();
2580	APInt V2 = APInt(V.getBitWidth(), 1) << V.countr_zero();
2581	ShadowMul = ConstantInt::get(Ty, V: V2);
2582	} else {
2583	ShadowMul = ConstantInt::get(Ty, V: 1);
2584	}
2585	}
2586
2587	IRBuilder<> IRB(&I);
2588	setShadow(V: &I,
2589	SV: IRB.CreateMul(LHS: getShadow(V: OtherArg), RHS: ShadowMul, Name: "msprop_mul_cst"));
2590	setOrigin(V: &I, Origin: getOrigin(V: OtherArg));
2591	}
2592
2593	void visitMul(BinaryOperator &I) {
2594	Constant *constOp0 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0));
2595	Constant *constOp1 = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1));
2596	if (constOp0 && !constOp1)
2597	handleMulByConstant(I, ConstArg: constOp0, OtherArg: I.getOperand(i_nocapture: 1));
2598	else if (constOp1 && !constOp0)
2599	handleMulByConstant(I, ConstArg: constOp1, OtherArg: I.getOperand(i_nocapture: 0));
2600	else
2601	handleShadowOr(I);
2602	}
2603
2604	void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2605	void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2606	void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2607	void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2608	void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2609	void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2610
2611	void handleIntegerDiv(Instruction &I) {
2612	IRBuilder<> IRB(&I);
2613	// Strict on the second argument.
2614	insertShadowCheck(Val: I.getOperand(i: 1), OrigIns: &I);
2615	setShadow(V: &I, SV: getShadow(I: &I, i: 0));
2616	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
2617	}
2618
2619	void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2620	void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2621	void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2622	void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2623
2624	// Floating point division is side-effect free. We can not require that the
2625	// divisor is fully initialized and must propagate shadow. See PR37523.
2626	void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2627	void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2628
2629	/// Instrument == and != comparisons.
2630	///
2631	/// Sometimes the comparison result is known even if some of the bits of the
2632	/// arguments are not.
2633	void handleEqualityComparison(ICmpInst &I) {
2634	IRBuilder<> IRB(&I);
2635	Value *A = I.getOperand(i_nocapture: 0);
2636	Value *B = I.getOperand(i_nocapture: 1);
2637	Value *Sa = getShadow(V: A);
2638	Value *Sb = getShadow(V: B);
2639
2640	// Get rid of pointers and vectors of pointers.
2641	// For ints (and vectors of ints), types of A and Sa match,
2642	// and this is a no-op.
2643	A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
2644	B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
2645
2646	// A == B <==> (C = A^B) == 0
2647	// A != B <==> (C = A^B) != 0
2648	// Sc = Sa | Sb
2649	Value *C = IRB.CreateXor(LHS: A, RHS: B);
2650	Value *Sc = IRB.CreateOr(LHS: Sa, RHS: Sb);
2651	// Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2652	// Result is defined if one of the following is true
2653	// * there is a defined 1 bit in C
2654	// * C is fully defined
2655	// Si = !(C & ~Sc) && Sc
2656	Value *Zero = Constant::getNullValue(Ty: Sc->getType());
2657	Value *MinusOne = Constant::getAllOnesValue(Ty: Sc->getType());
2658	Value *LHS = IRB.CreateICmpNE(LHS: Sc, RHS: Zero);
2659	Value *RHS =
2660	IRB.CreateICmpEQ(LHS: IRB.CreateAnd(LHS: IRB.CreateXor(LHS: Sc, RHS: MinusOne), RHS: C), RHS: Zero);
2661	Value *Si = IRB.CreateAnd(LHS, RHS);
2662	Si->setName("_msprop_icmp");
2663	setShadow(V: &I, SV: Si);
2664	setOriginForNaryOp(I);
2665	}
2666
2667	/// Build the lowest possible value of V, taking into account V's
2668	/// uninitialized bits.
2669	Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2670	bool isSigned) {
2671	if (isSigned) {
2672	// Split shadow into sign bit and other bits.
2673	Value *SaOtherBits = IRB.CreateLShr(LHS: IRB.CreateShl(LHS: Sa, RHS: 1), RHS: 1);
2674	Value *SaSignBit = IRB.CreateXor(LHS: Sa, RHS: SaOtherBits);
2675	// Maximise the undefined shadow bit, minimize other undefined bits.
2676	return IRB.CreateOr(LHS: IRB.CreateAnd(LHS: A, RHS: IRB.CreateNot(V: SaOtherBits)),
2677	RHS: SaSignBit);
2678	} else {
2679	// Minimize undefined bits.
2680	return IRB.CreateAnd(LHS: A, RHS: IRB.CreateNot(V: Sa));
2681	}
2682	}
2683
2684	/// Build the highest possible value of V, taking into account V's
2685	/// uninitialized bits.
2686	Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2687	bool isSigned) {
2688	if (isSigned) {
2689	// Split shadow into sign bit and other bits.
2690	Value *SaOtherBits = IRB.CreateLShr(LHS: IRB.CreateShl(LHS: Sa, RHS: 1), RHS: 1);
2691	Value *SaSignBit = IRB.CreateXor(LHS: Sa, RHS: SaOtherBits);
2692	// Minimise the undefined shadow bit, maximise other undefined bits.
2693	return IRB.CreateOr(LHS: IRB.CreateAnd(LHS: A, RHS: IRB.CreateNot(V: SaSignBit)),
2694	RHS: SaOtherBits);
2695	} else {
2696	// Maximize undefined bits.
2697	return IRB.CreateOr(LHS: A, RHS: Sa);
2698	}
2699	}
2700
2701	/// Instrument relational comparisons.
2702	///
2703	/// This function does exact shadow propagation for all relational
2704	/// comparisons of integers, pointers and vectors of those.
2705	/// FIXME: output seems suboptimal when one of the operands is a constant
2706	void handleRelationalComparisonExact(ICmpInst &I) {
2707	IRBuilder<> IRB(&I);
2708	Value *A = I.getOperand(i_nocapture: 0);
2709	Value *B = I.getOperand(i_nocapture: 1);
2710	Value *Sa = getShadow(V: A);
2711	Value *Sb = getShadow(V: B);
2712
2713	// Get rid of pointers and vectors of pointers.
2714	// For ints (and vectors of ints), types of A and Sa match,
2715	// and this is a no-op.
2716	A = IRB.CreatePointerCast(V: A, DestTy: Sa->getType());
2717	B = IRB.CreatePointerCast(V: B, DestTy: Sb->getType());
2718
2719	// Let [a0, a1] be the interval of possible values of A, taking into account
2720	// its undefined bits. Let [b0, b1] be the interval of possible values of B.
2721	// Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2722	bool IsSigned = I.isSigned();
2723	Value *S1 = IRB.CreateICmp(P: I.getPredicate(),
2724	LHS: getLowestPossibleValue(IRB, A, Sa, isSigned: IsSigned),
2725	RHS: getHighestPossibleValue(IRB, A: B, Sa: Sb, isSigned: IsSigned));
2726	Value *S2 = IRB.CreateICmp(P: I.getPredicate(),
2727	LHS: getHighestPossibleValue(IRB, A, Sa, isSigned: IsSigned),
2728	RHS: getLowestPossibleValue(IRB, A: B, Sa: Sb, isSigned: IsSigned));
2729	Value *Si = IRB.CreateXor(LHS: S1, RHS: S2);
2730	setShadow(V: &I, SV: Si);
2731	setOriginForNaryOp(I);
2732	}
2733
2734	/// Instrument signed relational comparisons.
2735	///
2736	/// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2737	/// bit of the shadow. Everything else is delegated to handleShadowOr().
2738	void handleSignedRelationalComparison(ICmpInst &I) {
2739	Constant *constOp;
2740	Value *op = nullptr;
2741	CmpInst::Predicate pre;
2742	if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
2743	op = I.getOperand(i_nocapture: 0);
2744	pre = I.getPredicate();
2745	} else if ((constOp = dyn_cast<Constant>(Val: I.getOperand(i_nocapture: 0)))) {
2746	op = I.getOperand(i_nocapture: 1);
2747	pre = I.getSwappedPredicate();
2748	} else {
2749	handleShadowOr(I);
2750	return;
2751	}
2752
2753	if ((constOp->isNullValue() &&
2754	(pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2755	(constOp->isAllOnesValue() &&
2756	(pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2757	IRBuilder<> IRB(&I);
2758	Value *Shadow = IRB.CreateICmpSLT(LHS: getShadow(V: op), RHS: getCleanShadow(V: op),
2759	Name: "_msprop_icmp_s");
2760	setShadow(V: &I, SV: Shadow);
2761	setOrigin(V: &I, Origin: getOrigin(V: op));
2762	} else {
2763	handleShadowOr(I);
2764	}
2765	}
2766
2767	void visitICmpInst(ICmpInst &I) {
2768	if (!ClHandleICmp) {
2769	handleShadowOr(I);
2770	return;
2771	}
2772	if (I.isEquality()) {
2773	handleEqualityComparison(I);
2774	return;
2775	}
2776
2777	assert(I.isRelational());
2778	if (ClHandleICmpExact) {
2779	handleRelationalComparisonExact(I);
2780	return;
2781	}
2782	if (I.isSigned()) {
2783	handleSignedRelationalComparison(I);
2784	return;
2785	}
2786
2787	assert(I.isUnsigned());
2788	if ((isa<Constant>(Val: I.getOperand(i_nocapture: 0)) || isa<Constant>(Val: I.getOperand(i_nocapture: 1)))) {
2789	handleRelationalComparisonExact(I);
2790	return;
2791	}
2792
2793	handleShadowOr(I);
2794	}
2795
2796	void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
2797
2798	void handleShift(BinaryOperator &I) {
2799	IRBuilder<> IRB(&I);
2800	// If any of the S2 bits are poisoned, the whole thing is poisoned.
2801	// Otherwise perform the same shift on S1.
2802	Value *S1 = getShadow(I: &I, i: 0);
2803	Value *S2 = getShadow(I: &I, i: 1);
2804	Value *S2Conv =
2805	IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
2806	Value *V2 = I.getOperand(i_nocapture: 1);
2807	Value *Shift = IRB.CreateBinOp(Opc: I.getOpcode(), LHS: S1, RHS: V2);
2808	setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
2809	setOriginForNaryOp(I);
2810	}
2811
2812	void visitShl(BinaryOperator &I) { handleShift(I); }
2813	void visitAShr(BinaryOperator &I) { handleShift(I); }
2814	void visitLShr(BinaryOperator &I) { handleShift(I); }
2815
2816	void handleFunnelShift(IntrinsicInst &I) {
2817	IRBuilder<> IRB(&I);
2818	// If any of the S2 bits are poisoned, the whole thing is poisoned.
2819	// Otherwise perform the same shift on S0 and S1.
2820	Value *S0 = getShadow(I: &I, i: 0);
2821	Value *S1 = getShadow(I: &I, i: 1);
2822	Value *S2 = getShadow(I: &I, i: 2);
2823	Value *S2Conv =
2824	IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: getCleanShadow(V: S2)), DestTy: S2->getType());
2825	Value *V2 = I.getOperand(i_nocapture: 2);
2826	Function *Intrin = Intrinsic::getDeclaration(
2827	M: I.getModule(), id: I.getIntrinsicID(), Tys: S2Conv->getType());
2828	Value *Shift = IRB.CreateCall(Callee: Intrin, Args: {S0, S1, V2});
2829	setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
2830	setOriginForNaryOp(I);
2831	}
2832
2833	/// Instrument llvm.memmove
2834	///
2835	/// At this point we don't know if llvm.memmove will be inlined or not.
2836	/// If we don't instrument it and it gets inlined,
2837	/// our interceptor will not kick in and we will lose the memmove.
2838	/// If we instrument the call here, but it does not get inlined,
2839	/// we will memove the shadow twice: which is bad in case
2840	/// of overlapping regions. So, we simply lower the intrinsic to a call.
2841	///
2842	/// Similar situation exists for memcpy and memset.
2843	void visitMemMoveInst(MemMoveInst &I) {
2844	getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
2845	IRBuilder<> IRB(&I);
2846	IRB.CreateCall(Callee: MS.MemmoveFn,
2847	Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
2848	IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
2849	I.eraseFromParent();
2850	}
2851
2852	/// Instrument memcpy
2853	///
2854	/// Similar to memmove: avoid copying shadow twice. This is somewhat
2855	/// unfortunate as it may slowdown small constant memcpys.
2856	/// FIXME: consider doing manual inline for small constant sizes and proper
2857	/// alignment.
2858	///
2859	/// Note: This also handles memcpy.inline, which promises no calls to external
2860	/// functions as an optimization. However, with instrumentation enabled this
2861	/// is difficult to promise; additionally, we know that the MSan runtime
2862	/// exists and provides __msan_memcpy(). Therefore, we assume that with
2863	/// instrumentation it's safe to turn memcpy.inline into a call to
2864	/// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
2865	/// itself, instrumentation should be disabled with the no_sanitize attribute.
2866	void visitMemCpyInst(MemCpyInst &I) {
2867	getShadow(V: I.getArgOperand(i: 1)); // Ensure shadow initialized
2868	IRBuilder<> IRB(&I);
2869	IRB.CreateCall(Callee: MS.MemcpyFn,
2870	Args: {I.getArgOperand(i: 0), I.getArgOperand(i: 1),
2871	IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
2872	I.eraseFromParent();
2873	}
2874
2875	// Same as memcpy.
2876	void visitMemSetInst(MemSetInst &I) {
2877	IRBuilder<> IRB(&I);
2878	IRB.CreateCall(
2879	Callee: MS.MemsetFn,
2880	Args: {I.getArgOperand(i: 0),
2881	IRB.CreateIntCast(V: I.getArgOperand(i: 1), DestTy: IRB.getInt32Ty(), isSigned: false),
2882	IRB.CreateIntCast(V: I.getArgOperand(i: 2), DestTy: MS.IntptrTy, isSigned: false)});
2883	I.eraseFromParent();
2884	}
2885
2886	void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
2887
2888	void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
2889
2890	/// Handle vector store-like intrinsics.
2891	///
2892	/// Instrument intrinsics that look like a simple SIMD store: writes memory,
2893	/// has 1 pointer argument and 1 vector argument, returns void.
2894	bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2895	IRBuilder<> IRB(&I);
2896	Value *Addr = I.getArgOperand(i: 0);
2897	Value *Shadow = getShadow(I: &I, i: 1);
2898	Value *ShadowPtr, *OriginPtr;
2899
2900	// We don't know the pointer alignment (could be unaligned SSE store!).
2901	// Have to assume to worst case.
2902	std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
2903	Addr, IRB, ShadowTy: Shadow->getType(), Alignment: Align(1), /*isStore*/ true);
2904	IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: Align(1));
2905
2906	if (ClCheckAccessAddress)
2907	insertShadowCheck(Val: Addr, OrigIns: &I);
2908
2909	// FIXME: factor out common code from materializeStores
2910	if (MS.TrackOrigins)
2911	IRB.CreateStore(Val: getOrigin(I: &I, i: 1), Ptr: OriginPtr);
2912	return true;
2913	}
2914
2915	/// Handle vector load-like intrinsics.
2916	///
2917	/// Instrument intrinsics that look like a simple SIMD load: reads memory,
2918	/// has 1 pointer argument, returns a vector.
2919	bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2920	IRBuilder<> IRB(&I);
2921	Value *Addr = I.getArgOperand(i: 0);
2922
2923	Type *ShadowTy = getShadowTy(V: &I);
2924	Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2925	if (PropagateShadow) {
2926	// We don't know the pointer alignment (could be unaligned SSE load!).
2927	// Have to assume to worst case.
2928	const Align Alignment = Align(1);
2929	std::tie(args&: ShadowPtr, args&: OriginPtr) =
2930	getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2931	setShadow(V: &I,
2932	SV: IRB.CreateAlignedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Align: Alignment, Name: "_msld"));
2933	} else {
2934	setShadow(V: &I, SV: getCleanShadow(V: &I));
2935	}
2936
2937	if (ClCheckAccessAddress)
2938	insertShadowCheck(Val: Addr, OrigIns: &I);
2939
2940	if (MS.TrackOrigins) {
2941	if (PropagateShadow)
2942	setOrigin(V: &I, Origin: IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr));
2943	else
2944	setOrigin(V: &I, Origin: getCleanOrigin());
2945	}
2946	return true;
2947	}
2948
2949	/// Handle (SIMD arithmetic)-like intrinsics.
2950	///
2951	/// Instrument intrinsics with any number of arguments of the same type,
2952	/// equal to the return type. The type should be simple (no aggregates or
2953	/// pointers; vectors are fine).
2954	/// Caller guarantees that this intrinsic does not access memory.
2955	bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2956	Type *RetTy = I.getType();
2957	if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy() ||
2958	RetTy->isX86_MMXTy()))
2959	return false;
2960
2961	unsigned NumArgOperands = I.arg_size();
2962	for (unsigned i = 0; i < NumArgOperands; ++i) {
2963	Type *Ty = I.getArgOperand(i)->getType();
2964	if (Ty != RetTy)
2965	return false;
2966	}
2967
2968	IRBuilder<> IRB(&I);
2969	ShadowAndOriginCombiner SC(this, IRB);
2970	for (unsigned i = 0; i < NumArgOperands; ++i)
2971	SC.Add(V: I.getArgOperand(i));
2972	SC.Done(I: &I);
2973
2974	return true;
2975	}
2976
2977	/// Heuristically instrument unknown intrinsics.
2978	///
2979	/// The main purpose of this code is to do something reasonable with all
2980	/// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2981	/// We recognize several classes of intrinsics by their argument types and
2982	/// ModRefBehaviour and apply special instrumentation when we are reasonably
2983	/// sure that we know what the intrinsic does.
2984	///
2985	/// We special-case intrinsics where this approach fails. See llvm.bswap
2986	/// handling as an example of that.
2987	bool handleUnknownIntrinsic(IntrinsicInst &I) {
2988	unsigned NumArgOperands = I.arg_size();
2989	if (NumArgOperands == 0)
2990	return false;
2991
2992	if (NumArgOperands == 2 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
2993	I.getArgOperand(i: 1)->getType()->isVectorTy() &&
2994	I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
2995	// This looks like a vector store.
2996	return handleVectorStoreIntrinsic(I);
2997	}
2998
2999	if (NumArgOperands == 1 && I.getArgOperand(i: 0)->getType()->isPointerTy() &&
3000	I.getType()->isVectorTy() && I.onlyReadsMemory()) {
3001	// This looks like a vector load.
3002	return handleVectorLoadIntrinsic(I);
3003	}
3004
3005	if (I.doesNotAccessMemory())
3006	if (maybeHandleSimpleNomemIntrinsic(I))
3007	return true;
3008
3009	// FIXME: detect and handle SSE maskstore/maskload
3010	return false;
3011	}
3012
3013	void handleInvariantGroup(IntrinsicInst &I) {
3014	setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3015	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3016	}
3017
3018	void handleLifetimeStart(IntrinsicInst &I) {
3019	if (!PoisonStack)
3020	return;
3021	AllocaInst *AI = llvm::findAllocaForValue(V: I.getArgOperand(i: 1));
3022	if (!AI)
3023	InstrumentLifetimeStart = false;
3024	LifetimeStartList.push_back(Elt: std::make_pair(x: &I, y&: AI));
3025	}
3026
3027	void handleBswap(IntrinsicInst &I) {
3028	IRBuilder<> IRB(&I);
3029	Value *Op = I.getArgOperand(i: 0);
3030	Type *OpType = Op->getType();
3031	Function *BswapFunc = Intrinsic::getDeclaration(
3032	M: F.getParent(), Intrinsic::id: bswap, Tys: ArrayRef(&OpType, 1));
3033	setShadow(V: &I, SV: IRB.CreateCall(Callee: BswapFunc, Args: getShadow(V: Op)));
3034	setOrigin(V: &I, Origin: getOrigin(V: Op));
3035	}
3036
3037	void handleCountZeroes(IntrinsicInst &I) {
3038	IRBuilder<> IRB(&I);
3039	Value *Src = I.getArgOperand(i: 0);
3040
3041	// Set the Output shadow based on input Shadow
3042	Value *BoolShadow = IRB.CreateIsNotNull(Arg: getShadow(V: Src), Name: "_mscz_bs");
3043
3044	// If zero poison is requested, mix in with the shadow
3045	Constant *IsZeroPoison = cast<Constant>(Val: I.getOperand(i_nocapture: 1));
3046	if (!IsZeroPoison->isZeroValue()) {
3047	Value *BoolZeroPoison = IRB.CreateIsNull(Arg: Src, Name: "_mscz_bzp");
3048	BoolShadow = IRB.CreateOr(LHS: BoolShadow, RHS: BoolZeroPoison, Name: "_mscz_bs");
3049	}
3050
3051	Value *OutputShadow =
3052	IRB.CreateSExt(V: BoolShadow, DestTy: getShadowTy(V: Src), Name: "_mscz_os");
3053
3054	setShadow(V: &I, SV: OutputShadow);
3055	setOriginForNaryOp(I);
3056	}
3057
3058	// Instrument vector convert intrinsic.
3059	//
3060	// This function instruments intrinsics like cvtsi2ss:
3061	// %Out = int_xxx_cvtyyy(%ConvertOp)
3062	// or
3063	// %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
3064	// Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
3065	// number \p Out elements, and (if has 2 arguments) copies the rest of the
3066	// elements from \p CopyOp.
3067	// In most cases conversion involves floating-point value which may trigger a
3068	// hardware exception when not fully initialized. For this reason we require
3069	// \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
3070	// We copy the shadow of \p CopyOp[NumUsedElements:] to \p
3071	// Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
3072	// return a fully initialized value.
3073	void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
3074	bool HasRoundingMode = false) {
3075	IRBuilder<> IRB(&I);
3076	Value *CopyOp, *ConvertOp;
3077
3078	assert((!HasRoundingMode ||
3079	isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
3080	"Invalid rounding mode");
3081
3082	switch (I.arg_size() - HasRoundingMode) {
3083	case 2:
3084	CopyOp = I.getArgOperand(i: 0);
3085	ConvertOp = I.getArgOperand(i: 1);
3086	break;
3087	case 1:
3088	ConvertOp = I.getArgOperand(i: 0);
3089	CopyOp = nullptr;
3090	break;
3091	default:
3092	llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
3093	}
3094
3095	// The first *NumUsedElements* elements of ConvertOp are converted to the
3096	// same number of output elements. The rest of the output is copied from
3097	// CopyOp, or (if not available) filled with zeroes.
3098	// Combine shadow for elements of ConvertOp that are used in this operation,
3099	// and insert a check.
3100	// FIXME: consider propagating shadow of ConvertOp, at least in the case of
3101	// int->any conversion.
3102	Value *ConvertShadow = getShadow(V: ConvertOp);
3103	Value *AggShadow = nullptr;
3104	if (ConvertOp->getType()->isVectorTy()) {
3105	AggShadow = IRB.CreateExtractElement(
3106	Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: 0));
3107	for (int i = 1; i < NumUsedElements; ++i) {
3108	Value *MoreShadow = IRB.CreateExtractElement(
3109	Vec: ConvertShadow, Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3110	AggShadow = IRB.CreateOr(LHS: AggShadow, RHS: MoreShadow);
3111	}
3112	} else {
3113	AggShadow = ConvertShadow;
3114	}
3115	assert(AggShadow->getType()->isIntegerTy());
3116	insertShadowCheck(Shadow: AggShadow, Origin: getOrigin(V: ConvertOp), OrigIns: &I);
3117
3118	// Build result shadow by zero-filling parts of CopyOp shadow that come from
3119	// ConvertOp.
3120	if (CopyOp) {
3121	assert(CopyOp->getType() == I.getType());
3122	assert(CopyOp->getType()->isVectorTy());
3123	Value *ResultShadow = getShadow(V: CopyOp);
3124	Type *EltTy = cast<VectorType>(Val: ResultShadow->getType())->getElementType();
3125	for (int i = 0; i < NumUsedElements; ++i) {
3126	ResultShadow = IRB.CreateInsertElement(
3127	Vec: ResultShadow, NewElt: ConstantInt::getNullValue(Ty: EltTy),
3128	Idx: ConstantInt::get(Ty: IRB.getInt32Ty(), V: i));
3129	}
3130	setShadow(V: &I, SV: ResultShadow);
3131	setOrigin(V: &I, Origin: getOrigin(V: CopyOp));
3132	} else {
3133	setShadow(V: &I, SV: getCleanShadow(V: &I));
3134	setOrigin(V: &I, Origin: getCleanOrigin());
3135	}
3136	}
3137
3138	// Given a scalar or vector, extract lower 64 bits (or less), and return all
3139	// zeroes if it is zero, and all ones otherwise.
3140	Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3141	if (S->getType()->isVectorTy())
3142	S = CreateShadowCast(IRB, V: S, dstTy: IRB.getInt64Ty(), /* Signed */ true);
3143	assert(S->getType()->getPrimitiveSizeInBits() <= 64);
3144	Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3145	return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3146	}
3147
3148	// Given a vector, extract its first element, and return all
3149	// zeroes if it is zero, and all ones otherwise.
3150	Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
3151	Value *S1 = IRB.CreateExtractElement(Vec: S, Idx: (uint64_t)0);
3152	Value *S2 = IRB.CreateICmpNE(LHS: S1, RHS: getCleanShadow(V: S1));
3153	return CreateShadowCast(IRB, V: S2, dstTy: T, /* Signed */ true);
3154	}
3155
3156	Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
3157	Type *T = S->getType();
3158	assert(T->isVectorTy());
3159	Value *S2 = IRB.CreateICmpNE(LHS: S, RHS: getCleanShadow(V: S));
3160	return IRB.CreateSExt(V: S2, DestTy: T);
3161	}
3162
3163	// Instrument vector shift intrinsic.
3164	//
3165	// This function instruments intrinsics like int_x86_avx2_psll_w.
3166	// Intrinsic shifts %In by %ShiftSize bits.
3167	// %ShiftSize may be a vector. In that case the lower 64 bits determine shift
3168	// size, and the rest is ignored. Behavior is defined even if shift size is
3169	// greater than register (or field) width.
3170	void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
3171	assert(I.arg_size() == 2);
3172	IRBuilder<> IRB(&I);
3173	// If any of the S2 bits are poisoned, the whole thing is poisoned.
3174	// Otherwise perform the same shift on S1.
3175	Value *S1 = getShadow(I: &I, i: 0);
3176	Value *S2 = getShadow(I: &I, i: 1);
3177	Value *S2Conv = Variable ? VariableShadowExtend(IRB, S: S2)
3178	: Lower64ShadowExtend(IRB, S: S2, T: getShadowTy(V: &I));
3179	Value *V1 = I.getOperand(i_nocapture: 0);
3180	Value *V2 = I.getOperand(i_nocapture: 1);
3181	Value *Shift = IRB.CreateCall(FTy: I.getFunctionType(), Callee: I.getCalledOperand(),
3182	Args: {IRB.CreateBitCast(V: S1, DestTy: V1->getType()), V2});
3183	Shift = IRB.CreateBitCast(V: Shift, DestTy: getShadowTy(V: &I));
3184	setShadow(V: &I, SV: IRB.CreateOr(LHS: Shift, RHS: S2Conv));
3185	setOriginForNaryOp(I);
3186	}
3187
3188	// Get an X86_MMX-sized vector type.
3189	Type *getMMXVectorTy(unsigned EltSizeInBits) {
3190	const unsigned X86_MMXSizeInBits = 64;
3191	assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3192	"Illegal MMX vector element size");
3193	return FixedVectorType::get(ElementType: IntegerType::get(C&: *MS.C, NumBits: EltSizeInBits),
3194	NumElts: X86_MMXSizeInBits / EltSizeInBits);
3195	}
3196
3197	// Returns a signed counterpart for an (un)signed-saturate-and-pack
3198	// intrinsic.
3199	Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
3200	switch (id) {
3201	case Intrinsic::x86_sse2_packsswb_128:
3202	case Intrinsic::x86_sse2_packuswb_128:
3203	return Intrinsic::x86_sse2_packsswb_128;
3204
3205	case Intrinsic::x86_sse2_packssdw_128:
3206	case Intrinsic::x86_sse41_packusdw:
3207	return Intrinsic::x86_sse2_packssdw_128;
3208
3209	case Intrinsic::x86_avx2_packsswb:
3210	case Intrinsic::x86_avx2_packuswb:
3211	return Intrinsic::x86_avx2_packsswb;
3212
3213	case Intrinsic::x86_avx2_packssdw:
3214	case Intrinsic::x86_avx2_packusdw:
3215	return Intrinsic::x86_avx2_packssdw;
3216
3217	case Intrinsic::x86_mmx_packsswb:
3218	case Intrinsic::x86_mmx_packuswb:
3219	return Intrinsic::x86_mmx_packsswb;
3220
3221	case Intrinsic::x86_mmx_packssdw:
3222	return Intrinsic::x86_mmx_packssdw;
3223	default:
3224	llvm_unreachable("unexpected intrinsic id");
3225	}
3226	}
3227
3228	// Instrument vector pack intrinsic.
3229	//
3230	// This function instruments intrinsics like x86_mmx_packsswb, that
3231	// packs elements of 2 input vectors into half as many bits with saturation.
3232	// Shadow is propagated with the signed variant of the same intrinsic applied
3233	// to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
3234	// EltSizeInBits is used only for x86mmx arguments.
3235	void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
3236	assert(I.arg_size() == 2);
3237	bool isX86_MMX = I.getOperand(i_nocapture: 0)->getType()->isX86_MMXTy();
3238	IRBuilder<> IRB(&I);
3239	Value *S1 = getShadow(I: &I, i: 0);
3240	Value *S2 = getShadow(I: &I, i: 1);
3241	assert(isX86_MMX || S1->getType()->isVectorTy());
3242
3243	// SExt and ICmpNE below must apply to individual elements of input vectors.
3244	// In case of x86mmx arguments, cast them to appropriate vector types and
3245	// back.
3246	Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
3247	if (isX86_MMX) {
3248	S1 = IRB.CreateBitCast(V: S1, DestTy: T);
3249	S2 = IRB.CreateBitCast(V: S2, DestTy: T);
3250	}
3251	Value *S1_ext =
3252	IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S1, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3253	Value *S2_ext =
3254	IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S2, RHS: Constant::getNullValue(Ty: T)), DestTy: T);
3255	if (isX86_MMX) {
3256	Type *X86_MMXTy = Type::getX86_MMXTy(C&: *MS.C);
3257	S1_ext = IRB.CreateBitCast(V: S1_ext, DestTy: X86_MMXTy);
3258	S2_ext = IRB.CreateBitCast(V: S2_ext, DestTy: X86_MMXTy);
3259	}
3260
3261	Function *ShadowFn = Intrinsic::getDeclaration(
3262	M: F.getParent(), id: getSignedPackIntrinsic(id: I.getIntrinsicID()));
3263
3264	Value *S =
3265	IRB.CreateCall(Callee: ShadowFn, Args: {S1_ext, S2_ext}, Name: "_msprop_vector_pack");
3266	if (isX86_MMX)
3267	S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3268	setShadow(V: &I, SV: S);
3269	setOriginForNaryOp(I);
3270	}
3271
3272	// Instrument sum-of-absolute-differences intrinsic.
3273	void handleVectorSadIntrinsic(IntrinsicInst &I) {
3274	const unsigned SignificantBitsPerResultElement = 16;
3275	bool isX86_MMX = I.getOperand(i_nocapture: 0)->getType()->isX86_MMXTy();
3276	Type *ResTy = isX86_MMX ? IntegerType::get(C&: *MS.C, NumBits: 64) : I.getType();
3277	unsigned ZeroBitsPerResultElement =
3278	ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
3279
3280	IRBuilder<> IRB(&I);
3281	auto *Shadow0 = getShadow(I: &I, i: 0);
3282	auto *Shadow1 = getShadow(I: &I, i: 1);
3283	Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3284	S = IRB.CreateBitCast(V: S, DestTy: ResTy);
3285	S = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S, RHS: Constant::getNullValue(Ty: ResTy)),
3286	DestTy: ResTy);
3287	S = IRB.CreateLShr(LHS: S, RHS: ZeroBitsPerResultElement);
3288	S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3289	setShadow(V: &I, SV: S);
3290	setOriginForNaryOp(I);
3291	}
3292
3293	// Instrument multiply-add intrinsic.
3294	void handleVectorPmaddIntrinsic(IntrinsicInst &I,
3295	unsigned EltSizeInBits = 0) {
3296	bool isX86_MMX = I.getOperand(i_nocapture: 0)->getType()->isX86_MMXTy();
3297	Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits: EltSizeInBits * 2) : I.getType();
3298	IRBuilder<> IRB(&I);
3299	auto *Shadow0 = getShadow(I: &I, i: 0);
3300	auto *Shadow1 = getShadow(I: &I, i: 1);
3301	Value *S = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3302	S = IRB.CreateBitCast(V: S, DestTy: ResTy);
3303	S = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: S, RHS: Constant::getNullValue(Ty: ResTy)),
3304	DestTy: ResTy);
3305	S = IRB.CreateBitCast(V: S, DestTy: getShadowTy(V: &I));
3306	setShadow(V: &I, SV: S);
3307	setOriginForNaryOp(I);
3308	}
3309
3310	// Instrument compare-packed intrinsic.
3311	// Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
3312	// all-ones shadow.
3313	void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
3314	IRBuilder<> IRB(&I);
3315	Type *ResTy = getShadowTy(V: &I);
3316	auto *Shadow0 = getShadow(I: &I, i: 0);
3317	auto *Shadow1 = getShadow(I: &I, i: 1);
3318	Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3319	Value *S = IRB.CreateSExt(
3320	V: IRB.CreateICmpNE(LHS: S0, RHS: Constant::getNullValue(Ty: ResTy)), DestTy: ResTy);
3321	setShadow(V: &I, SV: S);
3322	setOriginForNaryOp(I);
3323	}
3324
3325	// Instrument compare-scalar intrinsic.
3326	// This handles both cmp* intrinsics which return the result in the first
3327	// element of a vector, and comi* which return the result as i32.
3328	void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
3329	IRBuilder<> IRB(&I);
3330	auto *Shadow0 = getShadow(I: &I, i: 0);
3331	auto *Shadow1 = getShadow(I: &I, i: 1);
3332	Value *S0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3333	Value *S = LowerElementShadowExtend(IRB, S: S0, T: getShadowTy(V: &I));
3334	setShadow(V: &I, SV: S);
3335	setOriginForNaryOp(I);
3336	}
3337
3338	// Instrument generic vector reduction intrinsics
3339	// by ORing together all their fields.
3340	void handleVectorReduceIntrinsic(IntrinsicInst &I) {
3341	IRBuilder<> IRB(&I);
3342	Value *S = IRB.CreateOrReduce(Src: getShadow(I: &I, i: 0));
3343	setShadow(V: &I, SV: S);
3344	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3345	}
3346
3347	// Instrument vector.reduce.or intrinsic.
3348	// Valid (non-poisoned) set bits in the operand pull low the
3349	// corresponding shadow bits.
3350	void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
3351	IRBuilder<> IRB(&I);
3352	Value *OperandShadow = getShadow(I: &I, i: 0);
3353	Value *OperandUnsetBits = IRB.CreateNot(V: I.getOperand(i_nocapture: 0));
3354	Value *OperandUnsetOrPoison = IRB.CreateOr(LHS: OperandUnsetBits, RHS: OperandShadow);
3355	// Bit N is clean if any field's bit N is 1 and unpoison
3356	Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandUnsetOrPoison);
3357	// Otherwise, it is clean if every field's bit N is unpoison
3358	Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
3359	Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
3360
3361	setShadow(V: &I, SV: S);
3362	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3363	}
3364
3365	// Instrument vector.reduce.and intrinsic.
3366	// Valid (non-poisoned) unset bits in the operand pull down the
3367	// corresponding shadow bits.
3368	void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
3369	IRBuilder<> IRB(&I);
3370	Value *OperandShadow = getShadow(I: &I, i: 0);
3371	Value *OperandSetOrPoison = IRB.CreateOr(LHS: I.getOperand(i_nocapture: 0), RHS: OperandShadow);
3372	// Bit N is clean if any field's bit N is 0 and unpoison
3373	Value *OutShadowMask = IRB.CreateAndReduce(Src: OperandSetOrPoison);
3374	// Otherwise, it is clean if every field's bit N is unpoison
3375	Value *OrShadow = IRB.CreateOrReduce(Src: OperandShadow);
3376	Value *S = IRB.CreateAnd(LHS: OutShadowMask, RHS: OrShadow);
3377
3378	setShadow(V: &I, SV: S);
3379	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3380	}
3381
3382	void handleStmxcsr(IntrinsicInst &I) {
3383	IRBuilder<> IRB(&I);
3384	Value *Addr = I.getArgOperand(i: 0);
3385	Type *Ty = IRB.getInt32Ty();
3386	Value *ShadowPtr =
3387	getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment: Align(1), /*isStore*/ true).first;
3388
3389	IRB.CreateStore(Val: getCleanShadow(OrigTy: Ty), Ptr: ShadowPtr);
3390
3391	if (ClCheckAccessAddress)
3392	insertShadowCheck(Val: Addr, OrigIns: &I);
3393	}
3394
3395	void handleLdmxcsr(IntrinsicInst &I) {
3396	if (!InsertChecks)
3397	return;
3398
3399	IRBuilder<> IRB(&I);
3400	Value *Addr = I.getArgOperand(i: 0);
3401	Type *Ty = IRB.getInt32Ty();
3402	const Align Alignment = Align(1);
3403	Value *ShadowPtr, *OriginPtr;
3404	std::tie(args&: ShadowPtr, args&: OriginPtr) =
3405	getShadowOriginPtr(Addr, IRB, ShadowTy: Ty, Alignment, /*isStore*/ false);
3406
3407	if (ClCheckAccessAddress)
3408	insertShadowCheck(Val: Addr, OrigIns: &I);
3409
3410	Value *Shadow = IRB.CreateAlignedLoad(Ty, Ptr: ShadowPtr, Align: Alignment, Name: "_ldmxcsr");
3411	Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr)
3412	: getCleanOrigin();
3413	insertShadowCheck(Shadow, Origin, OrigIns: &I);
3414	}
3415
3416	void handleMaskedExpandLoad(IntrinsicInst &I) {
3417	IRBuilder<> IRB(&I);
3418	Value *Ptr = I.getArgOperand(i: 0);
3419	Value *Mask = I.getArgOperand(i: 1);
3420	Value *PassThru = I.getArgOperand(i: 2);
3421
3422	if (ClCheckAccessAddress) {
3423	insertShadowCheck(Val: Ptr, OrigIns: &I);
3424	insertShadowCheck(Val: Mask, OrigIns: &I);
3425	}
3426
3427	if (!PropagateShadow) {
3428	setShadow(V: &I, SV: getCleanShadow(V: &I));
3429	setOrigin(V: &I, Origin: getCleanOrigin());
3430	return;
3431	}
3432
3433	Type *ShadowTy = getShadowTy(V: &I);
3434	Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
3435	auto [ShadowPtr, OriginPtr] =
3436	getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: {}, /*isStore*/ false);
3437
3438	Value *Shadow = IRB.CreateMaskedExpandLoad(
3439	Ty: ShadowTy, Ptr: ShadowPtr, Mask, PassThru: getShadow(V: PassThru), Name: "_msmaskedexpload");
3440
3441	setShadow(V: &I, SV: Shadow);
3442
3443	// TODO: Store origins.
3444	setOrigin(V: &I, Origin: getCleanOrigin());
3445	}
3446
3447	void handleMaskedCompressStore(IntrinsicInst &I) {
3448	IRBuilder<> IRB(&I);
3449	Value *Values = I.getArgOperand(i: 0);
3450	Value *Ptr = I.getArgOperand(i: 1);
3451	Value *Mask = I.getArgOperand(i: 2);
3452
3453	if (ClCheckAccessAddress) {
3454	insertShadowCheck(Val: Ptr, OrigIns: &I);
3455	insertShadowCheck(Val: Mask, OrigIns: &I);
3456	}
3457
3458	Value *Shadow = getShadow(V: Values);
3459	Type *ElementShadowTy =
3460	getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
3461	auto [ShadowPtr, OriginPtrs] =
3462	getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy: ElementShadowTy, Alignment: {}, /*isStore*/ true);
3463
3464	IRB.CreateMaskedCompressStore(Val: Shadow, Ptr: ShadowPtr, Mask);
3465
3466	// TODO: Store origins.
3467	}
3468
3469	void handleMaskedGather(IntrinsicInst &I) {
3470	IRBuilder<> IRB(&I);
3471	Value *Ptrs = I.getArgOperand(i: 0);
3472	const Align Alignment(
3473	cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
3474	Value *Mask = I.getArgOperand(i: 2);
3475	Value *PassThru = I.getArgOperand(i: 3);
3476
3477	Type *PtrsShadowTy = getShadowTy(V: Ptrs);
3478	if (ClCheckAccessAddress) {
3479	insertShadowCheck(Val: Mask, OrigIns: &I);
3480	Value *MaskedPtrShadow = IRB.CreateSelect(
3481	C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
3482	Name: "_msmaskedptrs");
3483	insertShadowCheck(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
3484	}
3485
3486	if (!PropagateShadow) {
3487	setShadow(V: &I, SV: getCleanShadow(V: &I));
3488	setOrigin(V: &I, Origin: getCleanOrigin());
3489	return;
3490	}
3491
3492	Type *ShadowTy = getShadowTy(V: &I);
3493	Type *ElementShadowTy = cast<VectorType>(Val: ShadowTy)->getElementType();
3494	auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3495	Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ false);
3496
3497	Value *Shadow =
3498	IRB.CreateMaskedGather(Ty: ShadowTy, Ptrs: ShadowPtrs, Alignment, Mask,
3499	PassThru: getShadow(V: PassThru), Name: "_msmaskedgather");
3500
3501	setShadow(V: &I, SV: Shadow);
3502
3503	// TODO: Store origins.
3504	setOrigin(V: &I, Origin: getCleanOrigin());
3505	}
3506
3507	void handleMaskedScatter(IntrinsicInst &I) {
3508	IRBuilder<> IRB(&I);
3509	Value *Values = I.getArgOperand(i: 0);
3510	Value *Ptrs = I.getArgOperand(i: 1);
3511	const Align Alignment(
3512	cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue());
3513	Value *Mask = I.getArgOperand(i: 3);
3514
3515	Type *PtrsShadowTy = getShadowTy(V: Ptrs);
3516	if (ClCheckAccessAddress) {
3517	insertShadowCheck(Val: Mask, OrigIns: &I);
3518	Value *MaskedPtrShadow = IRB.CreateSelect(
3519	C: Mask, True: getShadow(V: Ptrs), False: Constant::getNullValue(Ty: (PtrsShadowTy)),
3520	Name: "_msmaskedptrs");
3521	insertShadowCheck(Shadow: MaskedPtrShadow, Origin: getOrigin(V: Ptrs), OrigIns: &I);
3522	}
3523
3524	Value *Shadow = getShadow(V: Values);
3525	Type *ElementShadowTy =
3526	getShadowTy(OrigTy: cast<VectorType>(Val: Values->getType())->getElementType());
3527	auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3528	Addr: Ptrs, IRB, ShadowTy: ElementShadowTy, Alignment, /*isStore*/ true);
3529
3530	IRB.CreateMaskedScatter(Val: Shadow, Ptrs: ShadowPtrs, Alignment, Mask);
3531
3532	// TODO: Store origin.
3533	}
3534
3535	void handleMaskedStore(IntrinsicInst &I) {
3536	IRBuilder<> IRB(&I);
3537	Value *V = I.getArgOperand(i: 0);
3538	Value *Ptr = I.getArgOperand(i: 1);
3539	const Align Alignment(
3540	cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue());
3541	Value *Mask = I.getArgOperand(i: 3);
3542	Value *Shadow = getShadow(V);
3543
3544	if (ClCheckAccessAddress) {
3545	insertShadowCheck(Val: Ptr, OrigIns: &I);
3546	insertShadowCheck(Val: Mask, OrigIns: &I);
3547	}
3548
3549	Value *ShadowPtr;
3550	Value *OriginPtr;
3551	std::tie(args&: ShadowPtr, args&: OriginPtr) = getShadowOriginPtr(
3552	Addr: Ptr, IRB, ShadowTy: Shadow->getType(), Alignment, /*isStore*/ true);
3553
3554	IRB.CreateMaskedStore(Val: Shadow, Ptr: ShadowPtr, Alignment, Mask);
3555
3556	if (!MS.TrackOrigins)
3557	return;
3558
3559	auto &DL = F.getParent()->getDataLayout();
3560	paintOrigin(IRB, Origin: getOrigin(V), OriginPtr,
3561	TS: DL.getTypeStoreSize(Ty: Shadow->getType()),
3562	Alignment: std::max(a: Alignment, b: kMinOriginAlignment));
3563	}
3564
3565	void handleMaskedLoad(IntrinsicInst &I) {
3566	IRBuilder<> IRB(&I);
3567	Value *Ptr = I.getArgOperand(i: 0);
3568	const Align Alignment(
3569	cast<ConstantInt>(Val: I.getArgOperand(i: 1))->getZExtValue());
3570	Value *Mask = I.getArgOperand(i: 2);
3571	Value *PassThru = I.getArgOperand(i: 3);
3572
3573	if (ClCheckAccessAddress) {
3574	insertShadowCheck(Val: Ptr, OrigIns: &I);
3575	insertShadowCheck(Val: Mask, OrigIns: &I);
3576	}
3577
3578	if (!PropagateShadow) {
3579	setShadow(V: &I, SV: getCleanShadow(V: &I));
3580	setOrigin(V: &I, Origin: getCleanOrigin());
3581	return;
3582	}
3583
3584	Type *ShadowTy = getShadowTy(V: &I);
3585	Value *ShadowPtr, *OriginPtr;
3586	std::tie(args&: ShadowPtr, args&: OriginPtr) =
3587	getShadowOriginPtr(Addr: Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3588	setShadow(V: &I, SV: IRB.CreateMaskedLoad(Ty: ShadowTy, Ptr: ShadowPtr, Alignment, Mask,
3589	PassThru: getShadow(V: PassThru), Name: "_msmaskedld"));
3590
3591	if (!MS.TrackOrigins)
3592	return;
3593
3594	// Choose between PassThru's and the loaded value's origins.
3595	Value *MaskedPassThruShadow = IRB.CreateAnd(
3596	LHS: getShadow(V: PassThru), RHS: IRB.CreateSExt(V: IRB.CreateNeg(V: Mask), DestTy: ShadowTy));
3597
3598	Value *NotNull = convertToBool(V: MaskedPassThruShadow, IRB, name: "_mscmp");
3599
3600	Value *PtrOrigin = IRB.CreateLoad(Ty: MS.OriginTy, Ptr: OriginPtr);
3601	Value *Origin = IRB.CreateSelect(C: NotNull, True: getOrigin(V: PassThru), False: PtrOrigin);
3602
3603	setOrigin(V: &I, Origin);
3604	}
3605
3606	// Instrument BMI / BMI2 intrinsics.
3607	// All of these intrinsics are Z = I(X, Y)
3608	// where the types of all operands and the result match, and are either i32 or
3609	// i64. The following instrumentation happens to work for all of them:
3610	// Sz = I(Sx, Y) | (sext (Sy != 0))
3611	void handleBmiIntrinsic(IntrinsicInst &I) {
3612	IRBuilder<> IRB(&I);
3613	Type *ShadowTy = getShadowTy(V: &I);
3614
3615	// If any bit of the mask operand is poisoned, then the whole thing is.
3616	Value *SMask = getShadow(I: &I, i: 1);
3617	SMask = IRB.CreateSExt(V: IRB.CreateICmpNE(LHS: SMask, RHS: getCleanShadow(OrigTy: ShadowTy)),
3618	DestTy: ShadowTy);
3619	// Apply the same intrinsic to the shadow of the first operand.
3620	Value *S = IRB.CreateCall(Callee: I.getCalledFunction(),
3621	Args: {getShadow(I: &I, i: 0), I.getOperand(i_nocapture: 1)});
3622	S = IRB.CreateOr(LHS: SMask, RHS: S);
3623	setShadow(V: &I, SV: S);
3624	setOriginForNaryOp(I);
3625	}
3626
3627	SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
3628	SmallVector<int, 8> Mask;
3629	for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
3630	Mask.append(NumInputs: 2, Elt: X);
3631	}
3632	return Mask;
3633	}
3634
3635	// Instrument pclmul intrinsics.
3636	// These intrinsics operate either on odd or on even elements of the input
3637	// vectors, depending on the constant in the 3rd argument, ignoring the rest.
3638	// Replace the unused elements with copies of the used ones, ex:
3639	// (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
3640	// or
3641	// (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
3642	// and then apply the usual shadow combining logic.
3643	void handlePclmulIntrinsic(IntrinsicInst &I) {
3644	IRBuilder<> IRB(&I);
3645	unsigned Width =
3646	cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
3647	assert(isa<ConstantInt>(I.getArgOperand(2)) &&
3648	"pclmul 3rd operand must be a constant");
3649	unsigned Imm = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getZExtValue();
3650	Value *Shuf0 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 0),
3651	Mask: getPclmulMask(Width, OddElements: Imm & 0x01));
3652	Value *Shuf1 = IRB.CreateShuffleVector(V: getShadow(I: &I, i: 1),
3653	Mask: getPclmulMask(Width, OddElements: Imm & 0x10));
3654	ShadowAndOriginCombiner SOC(this, IRB);
3655	SOC.Add(OpShadow: Shuf0, OpOrigin: getOrigin(I: &I, i: 0));
3656	SOC.Add(OpShadow: Shuf1, OpOrigin: getOrigin(I: &I, i: 1));
3657	SOC.Done(I: &I);
3658	}
3659
3660	// Instrument _mm_*_sd|ss intrinsics
3661	void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
3662	IRBuilder<> IRB(&I);
3663	unsigned Width =
3664	cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
3665	Value *First = getShadow(I: &I, i: 0);
3666	Value *Second = getShadow(I: &I, i: 1);
3667	// First element of second operand, remaining elements of first operand
3668	SmallVector<int, 16> Mask;
3669	Mask.push_back(Elt: Width);
3670	for (unsigned i = 1; i < Width; i++)
3671	Mask.push_back(Elt: i);
3672	Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: Second, Mask);
3673
3674	setShadow(V: &I, SV: Shadow);
3675	setOriginForNaryOp(I);
3676	}
3677
3678	void handleVtestIntrinsic(IntrinsicInst &I) {
3679	IRBuilder<> IRB(&I);
3680	Value *Shadow0 = getShadow(I: &I, i: 0);
3681	Value *Shadow1 = getShadow(I: &I, i: 1);
3682	Value *Or = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3683	Value *NZ = IRB.CreateICmpNE(LHS: Or, RHS: Constant::getNullValue(Ty: Or->getType()));
3684	Value *Scalar = convertShadowToScalar(V: NZ, IRB);
3685	Value *Shadow = IRB.CreateZExt(V: Scalar, DestTy: getShadowTy(V: &I));
3686
3687	setShadow(V: &I, SV: Shadow);
3688	setOriginForNaryOp(I);
3689	}
3690
3691	void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
3692	IRBuilder<> IRB(&I);
3693	unsigned Width =
3694	cast<FixedVectorType>(Val: I.getArgOperand(i: 0)->getType())->getNumElements();
3695	Value *First = getShadow(I: &I, i: 0);
3696	Value *Second = getShadow(I: &I, i: 1);
3697	Value *OrShadow = IRB.CreateOr(LHS: First, RHS: Second);
3698	// First element of both OR'd together, remaining elements of first operand
3699	SmallVector<int, 16> Mask;
3700	Mask.push_back(Elt: Width);
3701	for (unsigned i = 1; i < Width; i++)
3702	Mask.push_back(Elt: i);
3703	Value *Shadow = IRB.CreateShuffleVector(V1: First, V2: OrShadow, Mask);
3704
3705	setShadow(V: &I, SV: Shadow);
3706	setOriginForNaryOp(I);
3707	}
3708
3709	// Instrument abs intrinsic.
3710	// handleUnknownIntrinsic can't handle it because of the last
3711	// is_int_min_poison argument which does not match the result type.
3712	void handleAbsIntrinsic(IntrinsicInst &I) {
3713	assert(I.getType()->isIntOrIntVectorTy());
3714	assert(I.getArgOperand(0)->getType() == I.getType());
3715
3716	// FIXME: Handle is_int_min_poison.
3717	IRBuilder<> IRB(&I);
3718	setShadow(V: &I, SV: getShadow(I: &I, i: 0));
3719	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3720	}
3721
3722	void handleIsFpClass(IntrinsicInst &I) {
3723	IRBuilder<> IRB(&I);
3724	Value *Shadow = getShadow(I: &I, i: 0);
3725	setShadow(V: &I, SV: IRB.CreateICmpNE(LHS: Shadow, RHS: getCleanShadow(V: Shadow)));
3726	setOrigin(V: &I, Origin: getOrigin(I: &I, i: 0));
3727	}
3728
3729	void handleArithmeticWithOverflow(IntrinsicInst &I) {
3730	IRBuilder<> IRB(&I);
3731	Value *Shadow0 = getShadow(I: &I, i: 0);
3732	Value *Shadow1 = getShadow(I: &I, i: 1);
3733	Value *ShadowElt0 = IRB.CreateOr(LHS: Shadow0, RHS: Shadow1);
3734	Value *ShadowElt1 =
3735	IRB.CreateICmpNE(LHS: ShadowElt0, RHS: getCleanShadow(V: ShadowElt0));
3736
3737	Value *Shadow = PoisonValue::get(T: getShadowTy(V: &I));
3738	Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt0, Idxs: 0);
3739	Shadow = IRB.CreateInsertValue(Agg: Shadow, Val: ShadowElt1, Idxs: 1);
3740
3741	setShadow(V: &I, SV: Shadow);
3742	setOriginForNaryOp(I);
3743	}
3744
3745	void visitIntrinsicInst(IntrinsicInst &I) {
3746	switch (I.getIntrinsicID()) {
3747	case Intrinsic::uadd_with_overflow:
3748	case Intrinsic::sadd_with_overflow:
3749	case Intrinsic::usub_with_overflow:
3750	case Intrinsic::ssub_with_overflow:
3751	case Intrinsic::umul_with_overflow:
3752	case Intrinsic::smul_with_overflow:
3753	handleArithmeticWithOverflow(I);
3754	break;
3755	case Intrinsic::abs:
3756	handleAbsIntrinsic(I);
3757	break;
3758	case Intrinsic::is_fpclass:
3759	handleIsFpClass(I);
3760	break;
3761	case Intrinsic::lifetime_start:
3762	handleLifetimeStart(I);
3763	break;
3764	case Intrinsic::launder_invariant_group:
3765	case Intrinsic::strip_invariant_group:
3766	handleInvariantGroup(I);
3767	break;
3768	case Intrinsic::bswap:
3769	handleBswap(I);
3770	break;
3771	case Intrinsic::ctlz:
3772	case Intrinsic::cttz:
3773	handleCountZeroes(I);
3774	break;
3775	case Intrinsic::masked_compressstore:
3776	handleMaskedCompressStore(I);
3777	break;
3778	case Intrinsic::masked_expandload:
3779	handleMaskedExpandLoad(I);
3780	break;
3781	case Intrinsic::masked_gather:
3782	handleMaskedGather(I);
3783	break;
3784	case Intrinsic::masked_scatter:
3785	handleMaskedScatter(I);
3786	break;
3787	case Intrinsic::masked_store:
3788	handleMaskedStore(I);
3789	break;
3790	case Intrinsic::masked_load:
3791	handleMaskedLoad(I);
3792	break;
3793	case Intrinsic::vector_reduce_and:
3794	handleVectorReduceAndIntrinsic(I);
3795	break;
3796	case Intrinsic::vector_reduce_or:
3797	handleVectorReduceOrIntrinsic(I);
3798	break;
3799	case Intrinsic::vector_reduce_add:
3800	case Intrinsic::vector_reduce_xor:
3801	case Intrinsic::vector_reduce_mul:
3802	handleVectorReduceIntrinsic(I);
3803	break;
3804	case Intrinsic::x86_sse_stmxcsr:
3805	handleStmxcsr(I);
3806	break;
3807	case Intrinsic::x86_sse_ldmxcsr:
3808	handleLdmxcsr(I);
3809	break;
3810	case Intrinsic::x86_avx512_vcvtsd2usi64:
3811	case Intrinsic::x86_avx512_vcvtsd2usi32:
3812	case Intrinsic::x86_avx512_vcvtss2usi64:
3813	case Intrinsic::x86_avx512_vcvtss2usi32:
3814	case Intrinsic::x86_avx512_cvttss2usi64:
3815	case Intrinsic::x86_avx512_cvttss2usi:
3816	case Intrinsic::x86_avx512_cvttsd2usi64:
3817	case Intrinsic::x86_avx512_cvttsd2usi:
3818	case Intrinsic::x86_avx512_cvtusi2ss:
3819	case Intrinsic::x86_avx512_cvtusi642sd:
3820	case Intrinsic::x86_avx512_cvtusi642ss:
3821	handleVectorConvertIntrinsic(I, NumUsedElements: 1, HasRoundingMode: true);
3822	break;
3823	case Intrinsic::x86_sse2_cvtsd2si64:
3824	case Intrinsic::x86_sse2_cvtsd2si:
3825	case Intrinsic::x86_sse2_cvtsd2ss:
3826	case Intrinsic::x86_sse2_cvttsd2si64:
3827	case Intrinsic::x86_sse2_cvttsd2si:
3828	case Intrinsic::x86_sse_cvtss2si64:
3829	case Intrinsic::x86_sse_cvtss2si:
3830	case Intrinsic::x86_sse_cvttss2si64:
3831	case Intrinsic::x86_sse_cvttss2si:
3832	handleVectorConvertIntrinsic(I, NumUsedElements: 1);
3833	break;
3834	case Intrinsic::x86_sse_cvtps2pi:
3835	case Intrinsic::x86_sse_cvttps2pi:
3836	handleVectorConvertIntrinsic(I, NumUsedElements: 2);
3837	break;
3838
3839	case Intrinsic::x86_avx512_psll_w_512:
3840	case Intrinsic::x86_avx512_psll_d_512:
3841	case Intrinsic::x86_avx512_psll_q_512:
3842	case Intrinsic::x86_avx512_pslli_w_512:
3843	case Intrinsic::x86_avx512_pslli_d_512:
3844	case Intrinsic::x86_avx512_pslli_q_512:
3845	case Intrinsic::x86_avx512_psrl_w_512:
3846	case Intrinsic::x86_avx512_psrl_d_512:
3847	case Intrinsic::x86_avx512_psrl_q_512:
3848	case Intrinsic::x86_avx512_psra_w_512:
3849	case Intrinsic::x86_avx512_psra_d_512:
3850	case Intrinsic::x86_avx512_psra_q_512:
3851	case Intrinsic::x86_avx512_psrli_w_512:
3852	case Intrinsic::x86_avx512_psrli_d_512:
3853	case Intrinsic::x86_avx512_psrli_q_512:
3854	case Intrinsic::x86_avx512_psrai_w_512:
3855	case Intrinsic::x86_avx512_psrai_d_512:
3856	case Intrinsic::x86_avx512_psrai_q_512:
3857	case Intrinsic::x86_avx512_psra_q_256:
3858	case Intrinsic::x86_avx512_psra_q_128:
3859	case Intrinsic::x86_avx512_psrai_q_256:
3860	case Intrinsic::x86_avx512_psrai_q_128:
3861	case Intrinsic::x86_avx2_psll_w:
3862	case Intrinsic::x86_avx2_psll_d:
3863	case Intrinsic::x86_avx2_psll_q:
3864	case Intrinsic::x86_avx2_pslli_w:
3865	case Intrinsic::x86_avx2_pslli_d:
3866	case Intrinsic::x86_avx2_pslli_q:
3867	case Intrinsic::x86_avx2_psrl_w:
3868	case Intrinsic::x86_avx2_psrl_d:
3869	case Intrinsic::x86_avx2_psrl_q:
3870	case Intrinsic::x86_avx2_psra_w:
3871	case Intrinsic::x86_avx2_psra_d:
3872	case Intrinsic::x86_avx2_psrli_w:
3873	case Intrinsic::x86_avx2_psrli_d:
3874	case Intrinsic::x86_avx2_psrli_q:
3875	case Intrinsic::x86_avx2_psrai_w:
3876	case Intrinsic::x86_avx2_psrai_d:
3877	case Intrinsic::x86_sse2_psll_w:
3878	case Intrinsic::x86_sse2_psll_d:
3879	case Intrinsic::x86_sse2_psll_q:
3880	case Intrinsic::x86_sse2_pslli_w:
3881	case Intrinsic::x86_sse2_pslli_d:
3882	case Intrinsic::x86_sse2_pslli_q:
3883	case Intrinsic::x86_sse2_psrl_w:
3884	case Intrinsic::x86_sse2_psrl_d:
3885	case Intrinsic::x86_sse2_psrl_q:
3886	case Intrinsic::x86_sse2_psra_w:
3887	case Intrinsic::x86_sse2_psra_d:
3888	case Intrinsic::x86_sse2_psrli_w:
3889	case Intrinsic::x86_sse2_psrli_d:
3890	case Intrinsic::x86_sse2_psrli_q:
3891	case Intrinsic::x86_sse2_psrai_w:
3892	case Intrinsic::x86_sse2_psrai_d:
3893	case Intrinsic::x86_mmx_psll_w:
3894	case Intrinsic::x86_mmx_psll_d:
3895	case Intrinsic::x86_mmx_psll_q:
3896	case Intrinsic::x86_mmx_pslli_w:
3897	case Intrinsic::x86_mmx_pslli_d:
3898	case Intrinsic::x86_mmx_pslli_q:
3899	case Intrinsic::x86_mmx_psrl_w:
3900	case Intrinsic::x86_mmx_psrl_d:
3901	case Intrinsic::x86_mmx_psrl_q:
3902	case Intrinsic::x86_mmx_psra_w:
3903	case Intrinsic::x86_mmx_psra_d:
3904	case Intrinsic::x86_mmx_psrli_w:
3905	case Intrinsic::x86_mmx_psrli_d:
3906	case Intrinsic::x86_mmx_psrli_q:
3907	case Intrinsic::x86_mmx_psrai_w:
3908	case Intrinsic::x86_mmx_psrai_d:
3909	handleVectorShiftIntrinsic(I, /* Variable */ false);
3910	break;
3911	case Intrinsic::x86_avx2_psllv_d:
3912	case Intrinsic::x86_avx2_psllv_d_256:
3913	case Intrinsic::x86_avx512_psllv_d_512:
3914	case Intrinsic::x86_avx2_psllv_q:
3915	case Intrinsic::x86_avx2_psllv_q_256:
3916	case Intrinsic::x86_avx512_psllv_q_512:
3917	case Intrinsic::x86_avx2_psrlv_d:
3918	case Intrinsic::x86_avx2_psrlv_d_256:
3919	case Intrinsic::x86_avx512_psrlv_d_512:
3920	case Intrinsic::x86_avx2_psrlv_q:
3921	case Intrinsic::x86_avx2_psrlv_q_256:
3922	case Intrinsic::x86_avx512_psrlv_q_512:
3923	case Intrinsic::x86_avx2_psrav_d:
3924	case Intrinsic::x86_avx2_psrav_d_256:
3925	case Intrinsic::x86_avx512_psrav_d_512:
3926	case Intrinsic::x86_avx512_psrav_q_128:
3927	case Intrinsic::x86_avx512_psrav_q_256:
3928	case Intrinsic::x86_avx512_psrav_q_512:
3929	handleVectorShiftIntrinsic(I, /* Variable */ true);
3930	break;
3931
3932	case Intrinsic::x86_sse2_packsswb_128:
3933	case Intrinsic::x86_sse2_packssdw_128:
3934	case Intrinsic::x86_sse2_packuswb_128:
3935	case Intrinsic::x86_sse41_packusdw:
3936	case Intrinsic::x86_avx2_packsswb:
3937	case Intrinsic::x86_avx2_packssdw:
3938	case Intrinsic::x86_avx2_packuswb:
3939	case Intrinsic::x86_avx2_packusdw:
3940	handleVectorPackIntrinsic(I);
3941	break;
3942
3943	case Intrinsic::x86_mmx_packsswb:
3944	case Intrinsic::x86_mmx_packuswb:
3945	handleVectorPackIntrinsic(I, EltSizeInBits: 16);
3946	break;
3947
3948	case Intrinsic::x86_mmx_packssdw:
3949	handleVectorPackIntrinsic(I, EltSizeInBits: 32);
3950	break;
3951
3952	case Intrinsic::x86_mmx_psad_bw:
3953	case Intrinsic::x86_sse2_psad_bw:
3954	case Intrinsic::x86_avx2_psad_bw:
3955	handleVectorSadIntrinsic(I);
3956	break;
3957
3958	case Intrinsic::x86_sse2_pmadd_wd:
3959	case Intrinsic::x86_avx2_pmadd_wd:
3960	case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3961	case Intrinsic::x86_avx2_pmadd_ub_sw:
3962	handleVectorPmaddIntrinsic(I);
3963	break;
3964
3965	case Intrinsic::x86_ssse3_pmadd_ub_sw:
3966	handleVectorPmaddIntrinsic(I, EltSizeInBits: 8);
3967	break;
3968
3969	case Intrinsic::x86_mmx_pmadd_wd:
3970	handleVectorPmaddIntrinsic(I, EltSizeInBits: 16);
3971	break;
3972
3973	case Intrinsic::x86_sse_cmp_ss:
3974	case Intrinsic::x86_sse2_cmp_sd:
3975	case Intrinsic::x86_sse_comieq_ss:
3976	case Intrinsic::x86_sse_comilt_ss:
3977	case Intrinsic::x86_sse_comile_ss:
3978	case Intrinsic::x86_sse_comigt_ss:
3979	case Intrinsic::x86_sse_comige_ss:
3980	case Intrinsic::x86_sse_comineq_ss:
3981	case Intrinsic::x86_sse_ucomieq_ss:
3982	case Intrinsic::x86_sse_ucomilt_ss:
3983	case Intrinsic::x86_sse_ucomile_ss:
3984	case Intrinsic::x86_sse_ucomigt_ss:
3985	case Intrinsic::x86_sse_ucomige_ss:
3986	case Intrinsic::x86_sse_ucomineq_ss:
3987	case Intrinsic::x86_sse2_comieq_sd:
3988	case Intrinsic::x86_sse2_comilt_sd:
3989	case Intrinsic::x86_sse2_comile_sd:
3990	case Intrinsic::x86_sse2_comigt_sd:
3991	case Intrinsic::x86_sse2_comige_sd:
3992	case Intrinsic::x86_sse2_comineq_sd:
3993	case Intrinsic::x86_sse2_ucomieq_sd:
3994	case Intrinsic::x86_sse2_ucomilt_sd:
3995	case Intrinsic::x86_sse2_ucomile_sd:
3996	case Intrinsic::x86_sse2_ucomigt_sd:
3997	case Intrinsic::x86_sse2_ucomige_sd:
3998	case Intrinsic::x86_sse2_ucomineq_sd:
3999	handleVectorCompareScalarIntrinsic(I);
4000	break;
4001
4002	case Intrinsic::x86_avx_cmp_pd_256:
4003	case Intrinsic::x86_avx_cmp_ps_256:
4004	case Intrinsic::x86_sse2_cmp_pd:
4005	case Intrinsic::x86_sse_cmp_ps:
4006	handleVectorComparePackedIntrinsic(I);
4007	break;
4008
4009	case Intrinsic::x86_bmi_bextr_32:
4010	case Intrinsic::x86_bmi_bextr_64:
4011	case Intrinsic::x86_bmi_bzhi_32:
4012	case Intrinsic::x86_bmi_bzhi_64:
4013	case Intrinsic::x86_bmi_pdep_32:
4014	case Intrinsic::x86_bmi_pdep_64:
4015	case Intrinsic::x86_bmi_pext_32:
4016	case Intrinsic::x86_bmi_pext_64:
4017	handleBmiIntrinsic(I);
4018	break;
4019
4020	case Intrinsic::x86_pclmulqdq:
4021	case Intrinsic::x86_pclmulqdq_256:
4022	case Intrinsic::x86_pclmulqdq_512:
4023	handlePclmulIntrinsic(I);
4024	break;
4025
4026	case Intrinsic::x86_sse41_round_sd:
4027	case Intrinsic::x86_sse41_round_ss:
4028	handleUnarySdSsIntrinsic(I);
4029	break;
4030	case Intrinsic::x86_sse2_max_sd:
4031	case Intrinsic::x86_sse_max_ss:
4032	case Intrinsic::x86_sse2_min_sd:
4033	case Intrinsic::x86_sse_min_ss:
4034	handleBinarySdSsIntrinsic(I);
4035	break;
4036
4037	case Intrinsic::x86_avx_vtestc_pd:
4038	case Intrinsic::x86_avx_vtestc_pd_256:
4039	case Intrinsic::x86_avx_vtestc_ps:
4040	case Intrinsic::x86_avx_vtestc_ps_256:
4041	case Intrinsic::x86_avx_vtestnzc_pd:
4042	case Intrinsic::x86_avx_vtestnzc_pd_256:
4043	case Intrinsic::x86_avx_vtestnzc_ps:
4044	case Intrinsic::x86_avx_vtestnzc_ps_256:
4045	case Intrinsic::x86_avx_vtestz_pd:
4046	case Intrinsic::x86_avx_vtestz_pd_256:
4047	case Intrinsic::x86_avx_vtestz_ps:
4048	case Intrinsic::x86_avx_vtestz_ps_256:
4049	case Intrinsic::x86_avx_ptestc_256:
4050	case Intrinsic::x86_avx_ptestnzc_256:
4051	case Intrinsic::x86_avx_ptestz_256:
4052	case Intrinsic::x86_sse41_ptestc:
4053	case Intrinsic::x86_sse41_ptestnzc:
4054	case Intrinsic::x86_sse41_ptestz:
4055	handleVtestIntrinsic(I);
4056	break;
4057
4058	case Intrinsic::fshl:
4059	case Intrinsic::fshr:
4060	handleFunnelShift(I);
4061	break;
4062
4063	case Intrinsic::is_constant:
4064	// The result of llvm.is.constant() is always defined.
4065	setShadow(V: &I, SV: getCleanShadow(V: &I));
4066	setOrigin(V: &I, Origin: getCleanOrigin());
4067	break;
4068
4069	default:
4070	if (!handleUnknownIntrinsic(I))
4071	visitInstruction(I);
4072	break;
4073	}
4074	}
4075
4076	void visitLibAtomicLoad(CallBase &CB) {
4077	// Since we use getNextNode here, we can't have CB terminate the BB.
4078	assert(isa<CallInst>(CB));
4079
4080	IRBuilder<> IRB(&CB);
4081	Value *Size = CB.getArgOperand(i: 0);
4082	Value *SrcPtr = CB.getArgOperand(i: 1);
4083	Value *DstPtr = CB.getArgOperand(i: 2);
4084	Value *Ordering = CB.getArgOperand(i: 3);
4085	// Convert the call to have at least Acquire ordering to make sure
4086	// the shadow operations aren't reordered before it.
4087	Value *NewOrdering =
4088	IRB.CreateExtractElement(Vec: makeAddAcquireOrderingTable(IRB), Idx: Ordering);
4089	CB.setArgOperand(i: 3, v: NewOrdering);
4090
4091	NextNodeIRBuilder NextIRB(&CB);
4092	Value *SrcShadowPtr, *SrcOriginPtr;
4093	std::tie(args&: SrcShadowPtr, args&: SrcOriginPtr) =
4094	getShadowOriginPtr(Addr: SrcPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
4095	/*isStore*/ false);
4096	Value *DstShadowPtr =
4097	getShadowOriginPtr(Addr: DstPtr, IRB&: NextIRB, ShadowTy: NextIRB.getInt8Ty(), Alignment: Align(1),
4098	/*isStore*/ true)
4099	.first;
4100
4101	NextIRB.CreateMemCpy(Dst: DstShadowPtr, DstAlign: Align(1), Src: SrcShadowPtr, SrcAlign: Align(1), Size);
4102	if (MS.TrackOrigins) {
4103	Value *SrcOrigin = NextIRB.CreateAlignedLoad(Ty: MS.OriginTy, Ptr: SrcOriginPtr,
4104	Align: kMinOriginAlignment);
4105	Value *NewOrigin = updateOrigin(V: SrcOrigin, IRB&: NextIRB);
4106	NextIRB.CreateCall(Callee: MS.MsanSetOriginFn, Args: {DstPtr, Size, NewOrigin});
4107	}
4108	}
4109
4110	void visitLibAtomicStore(CallBase &CB) {
4111	IRBuilder<> IRB(&CB);
4112	Value *Size = CB.getArgOperand(i: 0);
4113	Value *DstPtr = CB.getArgOperand(i: 2);
4114	Value *Ordering = CB.getArgOperand(i: 3);
4115	// Convert the call to have at least Release ordering to make sure
4116	// the shadow operations aren't reordered after it.
4117	Value *NewOrdering =
4118	IRB.CreateExtractElement(Vec: makeAddReleaseOrderingTable(IRB), Idx: Ordering);
4119	CB.setArgOperand(i: 3, v: NewOrdering);
4120
4121	Value *DstShadowPtr =
4122	getShadowOriginPtr(Addr: DstPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1),
4123	/*isStore*/ true)
4124	.first;
4125
4126	// Atomic store always paints clean shadow/origin. See file header.
4127	IRB.CreateMemSet(Ptr: DstShadowPtr, Val: getCleanShadow(OrigTy: IRB.getInt8Ty()), Size,
4128	Align: Align(1));
4129	}
4130
4131	void visitCallBase(CallBase &CB) {
4132	assert(!CB.getMetadata(LLVMContext::MD_nosanitize));
4133	if (CB.isInlineAsm()) {
4134	// For inline asm (either a call to asm function, or callbr instruction),
4135	// do the usual thing: check argument shadow and mark all outputs as
4136	// clean. Note that any side effects of the inline asm that are not
4137	// immediately visible in its constraints are not handled.
4138	if (ClHandleAsmConservative)
4139	visitAsmInstruction(I&: CB);
4140	else
4141	visitInstruction(I&: CB);
4142	return;
4143	}
4144	LibFunc LF;
4145	if (TLI->getLibFunc(CB, F&: LF)) {
4146	// libatomic.a functions need to have special handling because there isn't
4147	// a good way to intercept them or compile the library with
4148	// instrumentation.
4149	switch (LF) {
4150	case LibFunc_atomic_load:
4151	if (!isa<CallInst>(Val: CB)) {
4152	llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
4153	"Ignoring!\n";
4154	break;
4155	}
4156	visitLibAtomicLoad(CB);
4157	return;
4158	case LibFunc_atomic_store:
4159	visitLibAtomicStore(CB);
4160	return;
4161	default:
4162	break;
4163	}
4164	}
4165
4166	if (auto *Call = dyn_cast<CallInst>(Val: &CB)) {
4167	assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
4168
4169	// We are going to insert code that relies on the fact that the callee
4170	// will become a non-readonly function after it is instrumented by us. To
4171	// prevent this code from being optimized out, mark that function
4172	// non-readonly in advance.
4173	// TODO: We can likely do better than dropping memory() completely here.
4174	AttributeMask B;
4175	B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4176
4177	Call->removeFnAttrs(AttrsToRemove: B);
4178	if (Function *Func = Call->getCalledFunction()) {
4179	Func->removeFnAttrs(Attrs: B);
4180	}
4181
4182	maybeMarkSanitizerLibraryCallNoBuiltin(CI: Call, TLI);
4183	}
4184	IRBuilder<> IRB(&CB);
4185	bool MayCheckCall = MS.EagerChecks;
4186	if (Function *Func = CB.getCalledFunction()) {
4187	// __sanitizer_unaligned_{load,store} functions may be called by users
4188	// and always expects shadows in the TLS. So don't check them.
4189	MayCheckCall &= !Func->getName().starts_with(Prefix: "__sanitizer_unaligned_");
4190	}
4191
4192	unsigned ArgOffset = 0;
4193	LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
4194	for (const auto &[i, A] : llvm::enumerate(First: CB.args())) {
4195	if (!A->getType()->isSized()) {
4196	LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
4197	continue;
4198	}
4199	unsigned Size = 0;
4200	const DataLayout &DL = F.getParent()->getDataLayout();
4201
4202	bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
4203	bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
4204	bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4205
4206	if (EagerCheck) {
4207	insertShadowCheck(Val: A, OrigIns: &CB);
4208	Size = DL.getTypeAllocSize(Ty: A->getType());
4209	} else {
4210	Value *Store = nullptr;
4211	// Compute the Shadow for arg even if it is ByVal, because
4212	// in that case getShadow() will copy the actual arg shadow to
4213	// __msan_param_tls.
4214	Value *ArgShadow = getShadow(V: A);
4215	Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4216	LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
4217	<< " Shadow: " << *ArgShadow << "\n");
4218	if (ByVal) {
4219	// ByVal requires some special handling as it's too big for a single
4220	// load
4221	assert(A->getType()->isPointerTy() &&
4222	"ByVal argument is not a pointer!");
4223	Size = DL.getTypeAllocSize(Ty: CB.getParamByValType(ArgNo: i));
4224	if (ArgOffset + Size > kParamTLSSize)
4225	break;
4226	const MaybeAlign ParamAlignment(CB.getParamAlign(ArgNo: i));
4227	MaybeAlign Alignment = std::nullopt;
4228	if (ParamAlignment)
4229	Alignment = std::min(a: *ParamAlignment, b: kShadowTLSAlignment);
4230	Value *AShadowPtr, *AOriginPtr;
4231	std::tie(args&: AShadowPtr, args&: AOriginPtr) =
4232	getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
4233	/*isStore*/ false);
4234	if (!PropagateShadow) {
4235	Store = IRB.CreateMemSet(Ptr: ArgShadowBase,
4236	Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
4237	Size, Align: Alignment);
4238	} else {
4239	Store = IRB.CreateMemCpy(Dst: ArgShadowBase, DstAlign: Alignment, Src: AShadowPtr,
4240	SrcAlign: Alignment, Size);
4241	if (MS.TrackOrigins) {
4242	Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4243	// FIXME: OriginSize should be:
4244	// alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
4245	unsigned OriginSize = alignTo(Size, A: kMinOriginAlignment);
4246	IRB.CreateMemCpy(
4247	Dst: ArgOriginBase,
4248	/* by origin_tls[ArgOffset] */ DstAlign: kMinOriginAlignment,
4249	Src: AOriginPtr,
4250	/* by getShadowOriginPtr */ SrcAlign: kMinOriginAlignment, Size: OriginSize);
4251	}
4252	}
4253	} else {
4254	// Any other parameters mean we need bit-grained tracking of uninit
4255	// data
4256	Size = DL.getTypeAllocSize(Ty: A->getType());
4257	if (ArgOffset + Size > kParamTLSSize)
4258	break;
4259	Store = IRB.CreateAlignedStore(Val: ArgShadow, Ptr: ArgShadowBase,
4260	Align: kShadowTLSAlignment);
4261	Constant *Cst = dyn_cast<Constant>(Val: ArgShadow);
4262	if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
4263	IRB.CreateStore(Val: getOrigin(V: A),
4264	Ptr: getOriginPtrForArgument(IRB, ArgOffset));
4265	}
4266	}
4267	(void)Store;
4268	assert(Store != nullptr);
4269	LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
4270	}
4271	assert(Size != 0);
4272	ArgOffset += alignTo(Size, A: kShadowTLSAlignment);
4273	}
4274	LLVM_DEBUG(dbgs() << " done with call args\n");
4275
4276	FunctionType *FT = CB.getFunctionType();
4277	if (FT->isVarArg()) {
4278	VAHelper->visitCallBase(CB, IRB);
4279	}
4280
4281	// Now, get the shadow for the RetVal.
4282	if (!CB.getType()->isSized())
4283	return;
4284	// Don't emit the epilogue for musttail call returns.
4285	if (isa<CallInst>(Val: CB) && cast<CallInst>(Val&: CB).isMustTailCall())
4286	return;
4287
4288	if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
4289	setShadow(V: &CB, SV: getCleanShadow(V: &CB));
4290	setOrigin(V: &CB, Origin: getCleanOrigin());
4291	return;
4292	}
4293
4294	IRBuilder<> IRBBefore(&CB);
4295	// Until we have full dynamic coverage, make sure the retval shadow is 0.
4296	Value *Base = getShadowPtrForRetval(IRB&: IRBBefore);
4297	IRBBefore.CreateAlignedStore(Val: getCleanShadow(V: &CB), Ptr: Base,
4298	Align: kShadowTLSAlignment);
4299	BasicBlock::iterator NextInsn;
4300	if (isa<CallInst>(Val: CB)) {
4301	NextInsn = ++CB.getIterator();
4302	assert(NextInsn != CB.getParent()->end());
4303	} else {
4304	BasicBlock *NormalDest = cast<InvokeInst>(Val&: CB).getNormalDest();
4305	if (!NormalDest->getSinglePredecessor()) {
4306	// FIXME: this case is tricky, so we are just conservative here.
4307	// Perhaps we need to split the edge between this BB and NormalDest,
4308	// but a naive attempt to use SplitEdge leads to a crash.
4309	setShadow(V: &CB, SV: getCleanShadow(V: &CB));
4310	setOrigin(V: &CB, Origin: getCleanOrigin());
4311	return;
4312	}
4313	// FIXME: NextInsn is likely in a basic block that has not been visited
4314	// yet. Anything inserted there will be instrumented by MSan later!
4315	NextInsn = NormalDest->getFirstInsertionPt();
4316	assert(NextInsn != NormalDest->end() &&
4317	"Could not find insertion point for retval shadow load");
4318	}
4319	IRBuilder<> IRBAfter(&*NextInsn);
4320	Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4321	Ty: getShadowTy(V: &CB), Ptr: getShadowPtrForRetval(IRB&: IRBAfter),
4322	Align: kShadowTLSAlignment, Name: "_msret");
4323	setShadow(V: &CB, SV: RetvalShadow);
4324	if (MS.TrackOrigins)
4325	setOrigin(V: &CB, Origin: IRBAfter.CreateLoad(Ty: MS.OriginTy,
4326	Ptr: getOriginPtrForRetval()));
4327	}
4328
4329	bool isAMustTailRetVal(Value *RetVal) {
4330	if (auto *I = dyn_cast<BitCastInst>(Val: RetVal)) {
4331	RetVal = I->getOperand(i_nocapture: 0);
4332	}
4333	if (auto *I = dyn_cast<CallInst>(Val: RetVal)) {
4334	return I->isMustTailCall();
4335	}
4336	return false;
4337	}
4338
4339	void visitReturnInst(ReturnInst &I) {
4340	IRBuilder<> IRB(&I);
4341	Value *RetVal = I.getReturnValue();
4342	if (!RetVal)
4343	return;
4344	// Don't emit the epilogue for musttail call returns.
4345	if (isAMustTailRetVal(RetVal))
4346	return;
4347	Value *ShadowPtr = getShadowPtrForRetval(IRB);
4348	bool HasNoUndef = F.hasRetAttribute(Attribute::NoUndef);
4349	bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4350	// FIXME: Consider using SpecialCaseList to specify a list of functions that
4351	// must always return fully initialized values. For now, we hardcode "main".
4352	bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
4353
4354	Value *Shadow = getShadow(V: RetVal);
4355	bool StoreOrigin = true;
4356	if (EagerCheck) {
4357	insertShadowCheck(Val: RetVal, OrigIns: &I);
4358	Shadow = getCleanShadow(V: RetVal);
4359	StoreOrigin = false;
4360	}
4361
4362	// The caller may still expect information passed over TLS if we pass our
4363	// check
4364	if (StoreShadow) {
4365	IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowPtr, Align: kShadowTLSAlignment);
4366	if (MS.TrackOrigins && StoreOrigin)
4367	IRB.CreateStore(Val: getOrigin(V: RetVal), Ptr: getOriginPtrForRetval());
4368	}
4369	}
4370
4371	void visitPHINode(PHINode &I) {
4372	IRBuilder<> IRB(&I);
4373	if (!PropagateShadow) {
4374	setShadow(V: &I, SV: getCleanShadow(V: &I));
4375	setOrigin(V: &I, Origin: getCleanOrigin());
4376	return;
4377	}
4378
4379	ShadowPHINodes.push_back(Elt: &I);
4380	setShadow(V: &I, SV: IRB.CreatePHI(Ty: getShadowTy(V: &I), NumReservedValues: I.getNumIncomingValues(),
4381	Name: "_msphi_s"));
4382	if (MS.TrackOrigins)
4383	setOrigin(
4384	V: &I, Origin: IRB.CreatePHI(Ty: MS.OriginTy, NumReservedValues: I.getNumIncomingValues(), Name: "_msphi_o"));
4385	}
4386
4387	Value *getLocalVarIdptr(AllocaInst &I) {
4388	ConstantInt *IntConst =
4389	ConstantInt::get(Ty: Type::getInt32Ty(C&: (*F.getParent()).getContext()), V: 0);
4390	return new GlobalVariable(*F.getParent(), IntConst->getType(),
4391	/*isConstant=*/false, GlobalValue::PrivateLinkage,
4392	IntConst);
4393	}
4394
4395	Value *getLocalVarDescription(AllocaInst &I) {
4396	return createPrivateConstGlobalForString(M&: *F.getParent(), Str: I.getName());
4397	}
4398
4399	void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
4400	if (PoisonStack && ClPoisonStackWithCall) {
4401	IRB.CreateCall(Callee: MS.MsanPoisonStackFn, Args: {&I, Len});
4402	} else {
4403	Value *ShadowBase, *OriginBase;
4404	std::tie(args&: ShadowBase, args&: OriginBase) = getShadowOriginPtr(
4405	Addr: &I, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1), /*isStore*/ true);
4406
4407	Value *PoisonValue = IRB.getInt8(C: PoisonStack ? ClPoisonStackPattern : 0);
4408	IRB.CreateMemSet(Ptr: ShadowBase, Val: PoisonValue, Size: Len, Align: I.getAlign());
4409	}
4410
4411	if (PoisonStack && MS.TrackOrigins) {
4412	Value *Idptr = getLocalVarIdptr(I);
4413	if (ClPrintStackNames) {
4414	Value *Descr = getLocalVarDescription(I);
4415	IRB.CreateCall(Callee: MS.MsanSetAllocaOriginWithDescriptionFn,
4416	Args: {&I, Len, Idptr, Descr});
4417	} else {
4418	IRB.CreateCall(Callee: MS.MsanSetAllocaOriginNoDescriptionFn, Args: {&I, Len, Idptr});
4419	}
4420	}
4421	}
4422
4423	void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
4424	Value *Descr = getLocalVarDescription(I);
4425	if (PoisonStack) {
4426	IRB.CreateCall(Callee: MS.MsanPoisonAllocaFn, Args: {&I, Len, Descr});
4427	} else {
4428	IRB.CreateCall(Callee: MS.MsanUnpoisonAllocaFn, Args: {&I, Len});
4429	}
4430	}
4431
4432	void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
4433	if (!InsPoint)
4434	InsPoint = &I;
4435	NextNodeIRBuilder IRB(InsPoint);
4436	const DataLayout &DL = F.getParent()->getDataLayout();
4437	uint64_t TypeSize = DL.getTypeAllocSize(Ty: I.getAllocatedType());
4438	Value *Len = ConstantInt::get(Ty: MS.IntptrTy, V: TypeSize);
4439	if (I.isArrayAllocation())
4440	Len = IRB.CreateMul(LHS: Len,
4441	RHS: IRB.CreateZExtOrTrunc(V: I.getArraySize(), DestTy: MS.IntptrTy));
4442
4443	if (MS.CompileKernel)
4444	poisonAllocaKmsan(I, IRB, Len);
4445	else
4446	poisonAllocaUserspace(I, IRB, Len);
4447	}
4448
4449	void visitAllocaInst(AllocaInst &I) {
4450	setShadow(V: &I, SV: getCleanShadow(V: &I));
4451	setOrigin(V: &I, Origin: getCleanOrigin());
4452	// We'll get to this alloca later unless it's poisoned at the corresponding
4453	// llvm.lifetime.start.
4454	AllocaSet.insert(X: &I);
4455	}
4456
4457	void visitSelectInst(SelectInst &I) {
4458	IRBuilder<> IRB(&I);
4459	// a = select b, c, d
4460	Value *B = I.getCondition();
4461	Value *C = I.getTrueValue();
4462	Value *D = I.getFalseValue();
4463	Value *Sb = getShadow(V: B);
4464	Value *Sc = getShadow(V: C);
4465	Value *Sd = getShadow(V: D);
4466
4467	// Result shadow if condition shadow is 0.
4468	Value *Sa0 = IRB.CreateSelect(C: B, True: Sc, False: Sd);
4469	Value *Sa1;
4470	if (I.getType()->isAggregateType()) {
4471	// To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
4472	// an extra "select". This results in much more compact IR.
4473	// Sa = select Sb, poisoned, (select b, Sc, Sd)
4474	Sa1 = getPoisonedShadow(ShadowTy: getShadowTy(OrigTy: I.getType()));
4475	} else {
4476	// Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
4477	// If Sb (condition is poisoned), look for bits in c and d that are equal
4478	// and both unpoisoned.
4479	// If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
4480
4481	// Cast arguments to shadow-compatible type.
4482	C = CreateAppToShadowCast(IRB, V: C);
4483	D = CreateAppToShadowCast(IRB, V: D);
4484
4485	// Result shadow if condition shadow is 1.
4486	Sa1 = IRB.CreateOr(Ops: {IRB.CreateXor(LHS: C, RHS: D), Sc, Sd});
4487	}
4488	Value *Sa = IRB.CreateSelect(C: Sb, True: Sa1, False: Sa0, Name: "_msprop_select");
4489	setShadow(V: &I, SV: Sa);
4490	if (MS.TrackOrigins) {
4491	// Origins are always i32, so any vector conditions must be flattened.
4492	// FIXME: consider tracking vector origins for app vectors?
4493	if (B->getType()->isVectorTy()) {
4494	B = convertToBool(V: B, IRB);
4495	Sb = convertToBool(V: Sb, IRB);
4496	}
4497	// a = select b, c, d
4498	// Oa = Sb ? Ob : (b ? Oc : Od)
4499	setOrigin(
4500	V: &I, Origin: IRB.CreateSelect(C: Sb, True: getOrigin(V: I.getCondition()),
4501	False: IRB.CreateSelect(C: B, True: getOrigin(V: I.getTrueValue()),
4502	False: getOrigin(V: I.getFalseValue()))));
4503	}
4504	}
4505
4506	void visitLandingPadInst(LandingPadInst &I) {
4507	// Do nothing.
4508	// See https://github.com/google/sanitizers/issues/504
4509	setShadow(V: &I, SV: getCleanShadow(V: &I));
4510	setOrigin(V: &I, Origin: getCleanOrigin());
4511	}
4512
4513	void visitCatchSwitchInst(CatchSwitchInst &I) {
4514	setShadow(V: &I, SV: getCleanShadow(V: &I));
4515	setOrigin(V: &I, Origin: getCleanOrigin());
4516	}
4517
4518	void visitFuncletPadInst(FuncletPadInst &I) {
4519	setShadow(V: &I, SV: getCleanShadow(V: &I));
4520	setOrigin(V: &I, Origin: getCleanOrigin());
4521	}
4522
4523	void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
4524
4525	void visitExtractValueInst(ExtractValueInst &I) {
4526	IRBuilder<> IRB(&I);
4527	Value *Agg = I.getAggregateOperand();
4528	LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
4529	Value *AggShadow = getShadow(V: Agg);
4530	LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
4531	Value *ResShadow = IRB.CreateExtractValue(Agg: AggShadow, Idxs: I.getIndices());
4532	LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
4533	setShadow(V: &I, SV: ResShadow);
4534	setOriginForNaryOp(I);
4535	}
4536
4537	void visitInsertValueInst(InsertValueInst &I) {
4538	IRBuilder<> IRB(&I);
4539	LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
4540	Value *AggShadow = getShadow(V: I.getAggregateOperand());
4541	Value *InsShadow = getShadow(V: I.getInsertedValueOperand());
4542	LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
4543	LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
4544	Value *Res = IRB.CreateInsertValue(Agg: AggShadow, Val: InsShadow, Idxs: I.getIndices());
4545	LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
4546	setShadow(V: &I, SV: Res);
4547	setOriginForNaryOp(I);
4548	}
4549
4550	void dumpInst(Instruction &I) {
4551	if (CallInst *CI = dyn_cast<CallInst>(Val: &I)) {
4552	errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
4553	} else {
4554	errs() << "ZZZ " << I.getOpcodeName() << "\n";
4555	}
4556	errs() << "QQQ " << I << "\n";
4557	}
4558
4559	void visitResumeInst(ResumeInst &I) {
4560	LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
4561	// Nothing to do here.
4562	}
4563
4564	void visitCleanupReturnInst(CleanupReturnInst &CRI) {
4565	LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
4566	// Nothing to do here.
4567	}
4568
4569	void visitCatchReturnInst(CatchReturnInst &CRI) {
4570	LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
4571	// Nothing to do here.
4572	}
4573
4574	void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
4575	IRBuilder<> &IRB, const DataLayout &DL,
4576	bool isOutput) {
4577	// For each assembly argument, we check its value for being initialized.
4578	// If the argument is a pointer, we assume it points to a single element
4579	// of the corresponding type (or to a 8-byte word, if the type is unsized).
4580	// Each such pointer is instrumented with a call to the runtime library.
4581	Type *OpType = Operand->getType();
4582	// Check the operand value itself.
4583	insertShadowCheck(Val: Operand, OrigIns: &I);
4584	if (!OpType->isPointerTy() || !isOutput) {
4585	assert(!isOutput);
4586	return;
4587	}
4588	if (!ElemTy->isSized())
4589	return;
4590	auto Size = DL.getTypeStoreSize(Ty: ElemTy);
4591	Value *SizeVal = IRB.CreateTypeSize(DstType: MS.IntptrTy, Size);
4592	if (MS.CompileKernel) {
4593	IRB.CreateCall(Callee: MS.MsanInstrumentAsmStoreFn, Args: {Operand, SizeVal});
4594	} else {
4595	// ElemTy, derived from elementtype(), does not encode the alignment of
4596	// the pointer. Conservatively assume that the shadow memory is unaligned.
4597	// When Size is large, avoid StoreInst as it would expand to many
4598	// instructions.
4599	auto [ShadowPtr, _] =
4600	getShadowOriginPtrUserspace(Addr: Operand, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: Align(1));
4601	if (Size <= 32)
4602	IRB.CreateAlignedStore(Val: getCleanShadow(OrigTy: ElemTy), Ptr: ShadowPtr, Align: Align(1));
4603	else
4604	IRB.CreateMemSet(Ptr: ShadowPtr, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
4605	Size: SizeVal, Align: Align(1));
4606	}
4607	}
4608
4609	/// Get the number of output arguments returned by pointers.
4610	int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
4611	int NumRetOutputs = 0;
4612	int NumOutputs = 0;
4613	Type *RetTy = cast<Value>(Val: CB)->getType();
4614	if (!RetTy->isVoidTy()) {
4615	// Register outputs are returned via the CallInst return value.
4616	auto *ST = dyn_cast<StructType>(Val: RetTy);
4617	if (ST)
4618	NumRetOutputs = ST->getNumElements();
4619	else
4620	NumRetOutputs = 1;
4621	}
4622	InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
4623	for (const InlineAsm::ConstraintInfo &Info : Constraints) {
4624	switch (Info.Type) {
4625	case InlineAsm::isOutput:
4626	NumOutputs++;
4627	break;
4628	default:
4629	break;
4630	}
4631	}
4632	return NumOutputs - NumRetOutputs;
4633	}
4634
4635	void visitAsmInstruction(Instruction &I) {
4636	// Conservative inline assembly handling: check for poisoned shadow of
4637	// asm() arguments, then unpoison the result and all the memory locations
4638	// pointed to by those arguments.
4639	// An inline asm() statement in C++ contains lists of input and output
4640	// arguments used by the assembly code. These are mapped to operands of the
4641	// CallInst as follows:
4642	// - nR register outputs ("=r) are returned by value in a single structure
4643	// (SSA value of the CallInst);
4644	// - nO other outputs ("=m" and others) are returned by pointer as first
4645	// nO operands of the CallInst;
4646	// - nI inputs ("r", "m" and others) are passed to CallInst as the
4647	// remaining nI operands.
4648	// The total number of asm() arguments in the source is nR+nO+nI, and the
4649	// corresponding CallInst has nO+nI+1 operands (the last operand is the
4650	// function to be called).
4651	const DataLayout &DL = F.getParent()->getDataLayout();
4652	CallBase *CB = cast<CallBase>(Val: &I);
4653	IRBuilder<> IRB(&I);
4654	InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
4655	int OutputArgs = getNumOutputArgs(IA, CB);
4656	// The last operand of a CallInst is the function itself.
4657	int NumOperands = CB->getNumOperands() - 1;
4658
4659	// Check input arguments. Doing so before unpoisoning output arguments, so
4660	// that we won't overwrite uninit values before checking them.
4661	for (int i = OutputArgs; i < NumOperands; i++) {
4662	Value *Operand = CB->getOperand(i_nocapture: i);
4663	instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
4664	/*isOutput*/ false);
4665	}
4666	// Unpoison output arguments. This must happen before the actual InlineAsm
4667	// call, so that the shadow for memory published in the asm() statement
4668	// remains valid.
4669	for (int i = 0; i < OutputArgs; i++) {
4670	Value *Operand = CB->getOperand(i_nocapture: i);
4671	instrumentAsmArgument(Operand, ElemTy: CB->getParamElementType(ArgNo: i), I, IRB, DL,
4672	/*isOutput*/ true);
4673	}
4674
4675	setShadow(V: &I, SV: getCleanShadow(V: &I));
4676	setOrigin(V: &I, Origin: getCleanOrigin());
4677	}
4678
4679	void visitFreezeInst(FreezeInst &I) {
4680	// Freeze always returns a fully defined value.
4681	setShadow(V: &I, SV: getCleanShadow(V: &I));
4682	setOrigin(V: &I, Origin: getCleanOrigin());
4683	}
4684
4685	void visitInstruction(Instruction &I) {
4686	// Everything else: stop propagating and check for poisoned shadow.
4687	if (ClDumpStrictInstructions)
4688	dumpInst(I);
4689	LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
4690	for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
4691	Value *Operand = I.getOperand(i);
4692	if (Operand->getType()->isSized())
4693	insertShadowCheck(Val: Operand, OrigIns: &I);
4694	}
4695	setShadow(V: &I, SV: getCleanShadow(V: &I));
4696	setOrigin(V: &I, Origin: getCleanOrigin());
4697	}
4698	};
4699
4700	struct VarArgHelperBase : public VarArgHelper {
4701	Function &F;
4702	MemorySanitizer &MS;
4703	MemorySanitizerVisitor &MSV;
4704	SmallVector<CallInst *, 16> VAStartInstrumentationList;
4705	const unsigned VAListTagSize;
4706
4707	VarArgHelperBase(Function &F, MemorySanitizer &MS,
4708	MemorySanitizerVisitor &MSV, unsigned VAListTagSize)
4709	: F(F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4710
4711	Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
4712	Value *Base = IRB.CreatePointerCast(V: MS.VAArgTLS, DestTy: MS.IntptrTy);
4713	return IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
4714	}
4715
4716	/// Compute the shadow address for a given va_arg.
4717	Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4718	unsigned ArgOffset) {
4719	Value *Base = IRB.CreatePointerCast(V: MS.VAArgTLS, DestTy: MS.IntptrTy);
4720	Base = IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
4721	return IRB.CreateIntToPtr(V: Base, DestTy: PointerType::get(ElementType: MSV.getShadowTy(OrigTy: Ty), AddressSpace: 0),
4722	Name: "_msarg_va_s");
4723	}
4724
4725	/// Compute the shadow address for a given va_arg.
4726	Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4727	unsigned ArgOffset, unsigned ArgSize) {
4728	// Make sure we don't overflow __msan_va_arg_tls.
4729	if (ArgOffset + ArgSize > kParamTLSSize)
4730	return nullptr;
4731	return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4732	}
4733
4734	/// Compute the origin address for a given va_arg.
4735	Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
4736	Value *Base = IRB.CreatePointerCast(V: MS.VAArgOriginTLS, DestTy: MS.IntptrTy);
4737	// getOriginPtrForVAArgument() is always called after
4738	// getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
4739	// overflow.
4740	Base = IRB.CreateAdd(LHS: Base, RHS: ConstantInt::get(Ty: MS.IntptrTy, V: ArgOffset));
4741	return IRB.CreateIntToPtr(V: Base, DestTy: PointerType::get(ElementType: MS.OriginTy, AddressSpace: 0),
4742	Name: "_msarg_va_o");
4743	}
4744
4745	void CleanUnusedTLS(IRBuilder<> &IRB, Value *ShadowBase,
4746	unsigned BaseOffset) {
4747	// The tails of __msan_va_arg_tls is not large enough to fit full
4748	// value shadow, but it will be copied to backup anyway. Make it
4749	// clean.
4750	if (BaseOffset >= kParamTLSSize)
4751	return;
4752	Value *TailSize =
4753	ConstantInt::getSigned(Ty: IRB.getInt32Ty(), V: kParamTLSSize - BaseOffset);
4754	IRB.CreateMemSet(Ptr: ShadowBase, Val: ConstantInt::getNullValue(Ty: IRB.getInt8Ty()),
4755	Size: TailSize, Align: Align(8));
4756	}
4757
4758	void unpoisonVAListTagForInst(IntrinsicInst &I) {
4759	IRBuilder<> IRB(&I);
4760	Value *VAListTag = I.getArgOperand(i: 0);
4761	const Align Alignment = Align(8);
4762	auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4763	Addr: VAListTag, IRB, ShadowTy: IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4764	// Unpoison the whole __va_list_tag.
4765	IRB.CreateMemSet(Ptr: ShadowPtr, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
4766	Size: VAListTagSize, Align: Alignment, isVolatile: false);
4767	}
4768
4769	void visitVAStartInst(VAStartInst &I) override {
4770	if (F.getCallingConv() == CallingConv::Win64)
4771	return;
4772	VAStartInstrumentationList.push_back(Elt: &I);
4773	unpoisonVAListTagForInst(I);
4774	}
4775
4776	void visitVACopyInst(VACopyInst &I) override {
4777	if (F.getCallingConv() == CallingConv::Win64)
4778	return;
4779	unpoisonVAListTagForInst(I);
4780	}
4781	};
4782
4783	/// AMD64-specific implementation of VarArgHelper.
4784	struct VarArgAMD64Helper : public VarArgHelperBase {
4785	// An unfortunate workaround for asymmetric lowering of va_arg stuff.
4786	// See a comment in visitCallBase for more details.
4787	static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
4788	static const unsigned AMD64FpEndOffsetSSE = 176;
4789	// If SSE is disabled, fp_offset in va_list is zero.
4790	static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4791
4792	unsigned AMD64FpEndOffset;
4793	AllocaInst *VAArgTLSCopy = nullptr;
4794	AllocaInst *VAArgTLSOriginCopy = nullptr;
4795	Value *VAArgOverflowSize = nullptr;
4796
4797	enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4798
4799	VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
4800	MemorySanitizerVisitor &MSV)
4801	: VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/24) {
4802	AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4803	for (const auto &Attr : F.getAttributes().getFnAttrs()) {
4804	if (Attr.isStringAttribute() &&
4805	(Attr.getKindAsString() == "target-features")) {
4806	if (Attr.getValueAsString().contains(Other: "-sse"))
4807	AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4808	break;
4809	}
4810	}
4811	}
4812
4813	ArgKind classifyArgument(Value *arg) {
4814	// A very rough approximation of X86_64 argument classification rules.
4815	Type *T = arg->getType();
4816	if (T->isX86_FP80Ty())
4817	return AK_Memory;
4818	if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
4819	return AK_FloatingPoint;
4820	if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4821	return AK_GeneralPurpose;
4822	if (T->isPointerTy())
4823	return AK_GeneralPurpose;
4824	return AK_Memory;
4825	}
4826
4827	// For VarArg functions, store the argument shadow in an ABI-specific format
4828	// that corresponds to va_list layout.
4829	// We do this because Clang lowers va_arg in the frontend, and this pass
4830	// only sees the low level code that deals with va_list internals.
4831	// A much easier alternative (provided that Clang emits va_arg instructions)
4832	// would have been to associate each live instance of va_list with a copy of
4833	// MSanParamTLS, and extract shadow on va_arg() call in the argument list
4834	// order.
4835	void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4836	unsigned GpOffset = 0;
4837	unsigned FpOffset = AMD64GpEndOffset;
4838	unsigned OverflowOffset = AMD64FpEndOffset;
4839	const DataLayout &DL = F.getParent()->getDataLayout();
4840
4841	for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
4842	bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4843	bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4844	if (IsByVal) {
4845	// ByVal arguments always go to the overflow area.
4846	// Fixed arguments passed through the overflow area will be stepped
4847	// over by va_start, so don't count them towards the offset.
4848	if (IsFixed)
4849	continue;
4850	assert(A->getType()->isPointerTy());
4851	Type *RealTy = CB.getParamByValType(ArgNo);
4852	uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
4853	uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
4854	unsigned BaseOffset = OverflowOffset;
4855	Value *ShadowBase =
4856	getShadowPtrForVAArgument(Ty: RealTy, IRB, ArgOffset: OverflowOffset);
4857	Value *OriginBase = nullptr;
4858	if (MS.TrackOrigins)
4859	OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
4860	OverflowOffset += AlignedSize;
4861
4862	if (OverflowOffset > kParamTLSSize) {
4863	CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4864	continue; // We have no space to copy shadow there.
4865	}
4866
4867	Value *ShadowPtr, *OriginPtr;
4868	std::tie(args&: ShadowPtr, args&: OriginPtr) =
4869	MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(), Alignment: kShadowTLSAlignment,
4870	/*isStore*/ false);
4871	IRB.CreateMemCpy(Dst: ShadowBase, DstAlign: kShadowTLSAlignment, Src: ShadowPtr,
4872	SrcAlign: kShadowTLSAlignment, Size: ArgSize);
4873	if (MS.TrackOrigins)
4874	IRB.CreateMemCpy(Dst: OriginBase, DstAlign: kShadowTLSAlignment, Src: OriginPtr,
4875	SrcAlign: kShadowTLSAlignment, Size: ArgSize);
4876	} else {
4877	ArgKind AK = classifyArgument(arg: A);
4878	if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4879	AK = AK_Memory;
4880	if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4881	AK = AK_Memory;
4882	Value *ShadowBase, *OriginBase = nullptr;
4883	switch (AK) {
4884	case AK_GeneralPurpose:
4885	ShadowBase = getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: GpOffset);
4886	if (MS.TrackOrigins)
4887	OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset);
4888	GpOffset += 8;
4889	assert(GpOffset <= kParamTLSSize);
4890	break;
4891	case AK_FloatingPoint:
4892	ShadowBase = getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: FpOffset);
4893	if (MS.TrackOrigins)
4894	OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
4895	FpOffset += 16;
4896	assert(FpOffset <= kParamTLSSize);
4897	break;
4898	case AK_Memory:
4899	if (IsFixed)
4900	continue;
4901	uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
4902	uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
4903	unsigned BaseOffset = OverflowOffset;
4904	ShadowBase =
4905	getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: OverflowOffset);
4906	if (MS.TrackOrigins) {
4907	OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset);
4908	}
4909	OverflowOffset += AlignedSize;
4910	if (OverflowOffset > kParamTLSSize) {
4911	// We have no space to copy shadow there.
4912	CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4913	continue;
4914	}
4915	}
4916	// Take fixed arguments into account for GpOffset and FpOffset,
4917	// but don't actually store shadows for them.
4918	// TODO(glider): don't call get*PtrForVAArgument() for them.
4919	if (IsFixed)
4920	continue;
4921	Value *Shadow = MSV.getShadow(V: A);
4922	IRB.CreateAlignedStore(Val: Shadow, Ptr: ShadowBase, Align: kShadowTLSAlignment);
4923	if (MS.TrackOrigins) {
4924	Value *Origin = MSV.getOrigin(V: A);
4925	TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
4926	MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
4927	Alignment: std::max(a: kShadowTLSAlignment, b: kMinOriginAlignment));
4928	}
4929	}
4930	}
4931	Constant *OverflowSize =
4932	ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AMD64FpEndOffset);
4933	IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
4934	}
4935
4936	void finalizeInstrumentation() override {
4937	assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4938	"finalizeInstrumentation called twice");
4939	if (!VAStartInstrumentationList.empty()) {
4940	// If there is a va_start in this function, make a backup copy of
4941	// va_arg_tls somewhere in the function entry block.
4942	IRBuilder<> IRB(MSV.FnPrologueEnd);
4943	VAArgOverflowSize =
4944	IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
4945	Value *CopySize = IRB.CreateAdd(
4946	LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AMD64FpEndOffset), RHS: VAArgOverflowSize);
4947	VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
4948	VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
4949	IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
4950	Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
4951
4952	Value *SrcSize = IRB.CreateBinaryIntrinsic(
4953	Intrinsic::umin, CopySize,
4954	ConstantInt::get(MS.IntptrTy, kParamTLSSize));
4955	IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
4956	SrcAlign: kShadowTLSAlignment, Size: SrcSize);
4957	if (MS.TrackOrigins) {
4958	VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
4959	VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
4960	IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
4961	Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
4962	}
4963	}
4964
4965	// Instrument va_start.
4966	// Copy va_list shadow from the backup copy of the TLS contents.
4967	for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4968	CallInst *OrigInst = VAStartInstrumentationList[i];
4969	NextNodeIRBuilder IRB(OrigInst);
4970	Value *VAListTag = OrigInst->getArgOperand(i: 0);
4971
4972	Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C); // i64*
4973	Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
4974	V: IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
4975	RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 16)),
4976	DestTy: PointerType::get(ElementType: RegSaveAreaPtrTy, AddressSpace: 0));
4977	Value *RegSaveAreaPtr =
4978	IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
4979	Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4980	const Align Alignment = Align(16);
4981	std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
4982	MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
4983	Alignment, /*isStore*/ true);
4984	IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
4985	Size: AMD64FpEndOffset);
4986	if (MS.TrackOrigins)
4987	IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
4988	SrcAlign: Alignment, Size: AMD64FpEndOffset);
4989	Type *OverflowArgAreaPtrTy = PointerType::getUnqual(C&: *MS.C); // i64*
4990	Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
4991	V: IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
4992	RHS: ConstantInt::get(Ty: MS.IntptrTy, V: 8)),
4993	DestTy: PointerType::get(ElementType: OverflowArgAreaPtrTy, AddressSpace: 0));
4994	Value *OverflowArgAreaPtr =
4995	IRB.CreateLoad(Ty: OverflowArgAreaPtrTy, Ptr: OverflowArgAreaPtrPtr);
4996	Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4997	std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
4998	MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
4999	Alignment, /*isStore*/ true);
5000	Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
5001	Idx0: AMD64FpEndOffset);
5002	IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
5003	Size: VAArgOverflowSize);
5004	if (MS.TrackOrigins) {
5005	SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
5006	Idx0: AMD64FpEndOffset);
5007	IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
5008	Size: VAArgOverflowSize);
5009	}
5010	}
5011	}
5012	};
5013
5014	/// MIPS64-specific implementation of VarArgHelper.
5015	/// NOTE: This is also used for LoongArch64.
5016	struct VarArgMIPS64Helper : public VarArgHelperBase {
5017	AllocaInst *VAArgTLSCopy = nullptr;
5018	Value *VAArgSize = nullptr;
5019
5020	VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
5021	MemorySanitizerVisitor &MSV)
5022	: VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/8) {}
5023
5024	void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5025	unsigned VAArgOffset = 0;
5026	const DataLayout &DL = F.getParent()->getDataLayout();
5027	for (Value *A :
5028	llvm::drop_begin(RangeOrContainer: CB.args(), N: CB.getFunctionType()->getNumParams())) {
5029	Triple TargetTriple(F.getParent()->getTargetTriple());
5030	Value *Base;
5031	uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
5032	if (TargetTriple.getArch() == Triple::mips64) {
5033	// Adjusting the shadow for argument with size < 8 to match the
5034	// placement of bits in big endian system
5035	if (ArgSize < 8)
5036	VAArgOffset += (8 - ArgSize);
5037	}
5038	Base = getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: VAArgOffset, ArgSize);
5039	VAArgOffset += ArgSize;
5040	VAArgOffset = alignTo(Value: VAArgOffset, Align: 8);
5041	if (!Base)
5042	continue;
5043	IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
5044	}
5045
5046	Constant *TotalVAArgSize = ConstantInt::get(Ty: IRB.getInt64Ty(), V: VAArgOffset);
5047	// Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
5048	// a new class member i.e. it is the total size of all VarArgs.
5049	IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
5050	}
5051
5052	void finalizeInstrumentation() override {
5053	assert(!VAArgSize && !VAArgTLSCopy &&
5054	"finalizeInstrumentation called twice");
5055	IRBuilder<> IRB(MSV.FnPrologueEnd);
5056	VAArgSize = IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
5057	Value *CopySize =
5058	IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: 0), RHS: VAArgSize);
5059
5060	if (!VAStartInstrumentationList.empty()) {
5061	// If there is a va_start in this function, make a backup copy of
5062	// va_arg_tls somewhere in the function entry block.
5063	VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
5064	VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
5065	IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
5066	Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
5067
5068	Value *SrcSize = IRB.CreateBinaryIntrinsic(
5069	Intrinsic::umin, CopySize,
5070	ConstantInt::get(MS.IntptrTy, kParamTLSSize));
5071	IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
5072	SrcAlign: kShadowTLSAlignment, Size: SrcSize);
5073	}
5074
5075	// Instrument va_start.
5076	// Copy va_list shadow from the backup copy of the TLS contents.
5077	for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
5078	CallInst *OrigInst = VAStartInstrumentationList[i];
5079	NextNodeIRBuilder IRB(OrigInst);
5080	Value *VAListTag = OrigInst->getArgOperand(i: 0);
5081	Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C); // i64*
5082	Value *RegSaveAreaPtrPtr =
5083	IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
5084	DestTy: PointerType::get(ElementType: RegSaveAreaPtrTy, AddressSpace: 0));
5085	Value *RegSaveAreaPtr =
5086	IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
5087	Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5088	const Align Alignment = Align(8);
5089	std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
5090	MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
5091	Alignment, /*isStore*/ true);
5092	IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
5093	Size: CopySize);
5094	}
5095	}
5096	};
5097
5098	/// AArch64-specific implementation of VarArgHelper.
5099	struct VarArgAArch64Helper : public VarArgHelperBase {
5100	static const unsigned kAArch64GrArgSize = 64;
5101	static const unsigned kAArch64VrArgSize = 128;
5102
5103	static const unsigned AArch64GrBegOffset = 0;
5104	static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5105	// Make VR space aligned to 16 bytes.
5106	static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5107	static const unsigned AArch64VrEndOffset =
5108	AArch64VrBegOffset + kAArch64VrArgSize;
5109	static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5110
5111	AllocaInst *VAArgTLSCopy = nullptr;
5112	Value *VAArgOverflowSize = nullptr;
5113
5114	enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5115
5116	VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
5117	MemorySanitizerVisitor &MSV)
5118	: VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/32) {}
5119
5120	// A very rough approximation of aarch64 argument classification rules.
5121	std::pair<ArgKind, uint64_t> classifyArgument(Type *T) {
5122	if (T->isIntOrPtrTy() && T->getPrimitiveSizeInBits() <= 64)
5123	return {AK_GeneralPurpose, 1};
5124	if (T->isFloatingPointTy() && T->getPrimitiveSizeInBits() <= 128)
5125	return {AK_FloatingPoint, 1};
5126
5127	if (T->isArrayTy()) {
5128	auto R = classifyArgument(T: T->getArrayElementType());
5129	R.second *= T->getScalarType()->getArrayNumElements();
5130	return R;
5131	}
5132
5133	if (const FixedVectorType *FV = dyn_cast<FixedVectorType>(Val: T)) {
5134	auto R = classifyArgument(T: FV->getScalarType());
5135	R.second *= FV->getNumElements();
5136	return R;
5137	}
5138
5139	LLVM_DEBUG(errs() << "Unknown vararg type: " << *T << "\n");
5140	return {AK_Memory, 0};
5141	}
5142
5143	// The instrumentation stores the argument shadow in a non ABI-specific
5144	// format because it does not know which argument is named (since Clang,
5145	// like x86_64 case, lowers the va_args in the frontend and this pass only
5146	// sees the low level code that deals with va_list internals).
5147	// The first seven GR registers are saved in the first 56 bytes of the
5148	// va_arg tls arra, followed by the first 8 FP/SIMD registers, and then
5149	// the remaining arguments.
5150	// Using constant offset within the va_arg TLS array allows fast copy
5151	// in the finalize instrumentation.
5152	void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5153	unsigned GrOffset = AArch64GrBegOffset;
5154	unsigned VrOffset = AArch64VrBegOffset;
5155	unsigned OverflowOffset = AArch64VAEndOffset;
5156
5157	const DataLayout &DL = F.getParent()->getDataLayout();
5158	for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
5159	bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5160	auto [AK, RegNum] = classifyArgument(T: A->getType());
5161	if (AK == AK_GeneralPurpose &&
5162	(GrOffset + RegNum * 8) > AArch64GrEndOffset)
5163	AK = AK_Memory;
5164	if (AK == AK_FloatingPoint &&
5165	(VrOffset + RegNum * 16) > AArch64VrEndOffset)
5166	AK = AK_Memory;
5167	Value *Base;
5168	switch (AK) {
5169	case AK_GeneralPurpose:
5170	Base = getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: GrOffset);
5171	GrOffset += 8 * RegNum;
5172	break;
5173	case AK_FloatingPoint:
5174	Base = getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: VrOffset);
5175	VrOffset += 16 * RegNum;
5176	break;
5177	case AK_Memory:
5178	// Don't count fixed arguments in the overflow area - va_start will
5179	// skip right over them.
5180	if (IsFixed)
5181	continue;
5182	uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
5183	uint64_t AlignedSize = alignTo(Value: ArgSize, Align: 8);
5184	unsigned BaseOffset = OverflowOffset;
5185	Base = getShadowPtrForVAArgument(Ty: A->getType(), IRB, ArgOffset: BaseOffset);
5186	OverflowOffset += AlignedSize;
5187	if (OverflowOffset > kParamTLSSize) {
5188	// We have no space to copy shadow there.
5189	CleanUnusedTLS(IRB, ShadowBase: Base, BaseOffset);
5190	continue;
5191	}
5192	break;
5193	}
5194	// Count Gp/Vr fixed arguments to their respective offsets, but don't
5195	// bother to actually store a shadow.
5196	if (IsFixed)
5197	continue;
5198	IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
5199	}
5200	Constant *OverflowSize =
5201	ConstantInt::get(Ty: IRB.getInt64Ty(), V: OverflowOffset - AArch64VAEndOffset);
5202	IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
5203	}
5204
5205	// Retrieve a va_list field of 'void*' size.
5206	Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5207	Value *SaveAreaPtrPtr = IRB.CreateIntToPtr(
5208	V: IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
5209	RHS: ConstantInt::get(Ty: MS.IntptrTy, V: offset)),
5210	DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
5211	return IRB.CreateLoad(Ty: Type::getInt64Ty(C&: *MS.C), Ptr: SaveAreaPtrPtr);
5212	}
5213
5214	// Retrieve a va_list field of 'int' size.
5215	Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5216	Value *SaveAreaPtr = IRB.CreateIntToPtr(
5217	V: IRB.CreateAdd(LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
5218	RHS: ConstantInt::get(Ty: MS.IntptrTy, V: offset)),
5219	DestTy: PointerType::get(C&: *MS.C, AddressSpace: 0));
5220	Value *SaveArea32 = IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: SaveAreaPtr);
5221	return IRB.CreateSExt(V: SaveArea32, DestTy: MS.IntptrTy);
5222	}
5223
5224	void finalizeInstrumentation() override {
5225	assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5226	"finalizeInstrumentation called twice");
5227	if (!VAStartInstrumentationList.empty()) {
5228	// If there is a va_start in this function, make a backup copy of
5229	// va_arg_tls somewhere in the function entry block.
5230	IRBuilder<> IRB(MSV.FnPrologueEnd);
5231	VAArgOverflowSize =
5232	IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
5233	Value *CopySize = IRB.CreateAdd(
5234	LHS: ConstantInt::get(Ty: MS.IntptrTy, V: AArch64VAEndOffset), RHS: VAArgOverflowSize);
5235	VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
5236	VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
5237	IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
5238	Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
5239
5240	Value *SrcSize = IRB.CreateBinaryIntrinsic(
5241	Intrinsic::umin, CopySize,
5242	ConstantInt::get(MS.IntptrTy, kParamTLSSize));
5243	IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
5244	SrcAlign: kShadowTLSAlignment, Size: SrcSize);
5245	}
5246
5247	Value *GrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64GrArgSize);
5248	Value *VrArgSize = ConstantInt::get(Ty: MS.IntptrTy, V: kAArch64VrArgSize);
5249
5250	// Instrument va_start, copy va_list shadow from the backup copy of
5251	// the TLS contents.
5252	for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
5253	CallInst *OrigInst = VAStartInstrumentationList[i];
5254	NextNodeIRBuilder IRB(OrigInst);
5255
5256	Value *VAListTag = OrigInst->getArgOperand(i: 0);
5257
5258	// The variadic ABI for AArch64 creates two areas to save the incoming
5259	// argument registers (one for 64-bit general register xn-x7 and another
5260	// for 128-bit FP/SIMD vn-v7).
5261	// We need then to propagate the shadow arguments on both regions
5262	// 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
5263	// The remaining arguments are saved on shadow for 'va::stack'.
5264	// One caveat is it requires only to propagate the non-named arguments,
5265	// however on the call site instrumentation 'all' the arguments are
5266	// saved. So to copy the shadow values from the va_arg TLS array
5267	// we need to adjust the offset for both GR and VR fields based on
5268	// the __{gr,vr}_offs value (since they are stores based on incoming
5269	// named arguments).
5270	Type *RegSaveAreaPtrTy = IRB.getPtrTy();
5271
5272	// Read the stack pointer from the va_list.
5273	Value *StackSaveAreaPtr =
5274	IRB.CreateIntToPtr(V: getVAField64(IRB, VAListTag, offset: 0), DestTy: RegSaveAreaPtrTy);
5275
5276	// Read both the __gr_top and __gr_off and add them up.
5277	Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 8);
5278	Value *GrOffSaveArea = getVAField32(IRB, VAListTag, offset: 24);
5279
5280	Value *GrRegSaveAreaPtr = IRB.CreateIntToPtr(
5281	V: IRB.CreateAdd(LHS: GrTopSaveAreaPtr, RHS: GrOffSaveArea), DestTy: RegSaveAreaPtrTy);
5282
5283	// Read both the __vr_top and __vr_off and add them up.
5284	Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, offset: 16);
5285	Value *VrOffSaveArea = getVAField32(IRB, VAListTag, offset: 28);
5286
5287	Value *VrRegSaveAreaPtr = IRB.CreateIntToPtr(
5288	V: IRB.CreateAdd(LHS: VrTopSaveAreaPtr, RHS: VrOffSaveArea), DestTy: RegSaveAreaPtrTy);
5289
5290	// It does not know how many named arguments is being used and, on the
5291	// callsite all the arguments were saved. Since __gr_off is defined as
5292	// '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
5293	// argument by ignoring the bytes of shadow from named arguments.
5294	Value *GrRegSaveAreaShadowPtrOff =
5295	IRB.CreateAdd(LHS: GrArgSize, RHS: GrOffSaveArea);
5296
5297	Value *GrRegSaveAreaShadowPtr =
5298	MSV.getShadowOriginPtr(Addr: GrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
5299	Alignment: Align(8), /*isStore*/ true)
5300	.first;
5301
5302	Value *GrSrcPtr =
5303	IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy, Offset: GrRegSaveAreaShadowPtrOff);
5304	Value *GrCopySize = IRB.CreateSub(LHS: GrArgSize, RHS: GrRegSaveAreaShadowPtrOff);
5305
5306	IRB.CreateMemCpy(Dst: GrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: GrSrcPtr, SrcAlign: Align(8),
5307	Size: GrCopySize);
5308
5309	// Again, but for FP/SIMD values.
5310	Value *VrRegSaveAreaShadowPtrOff =
5311	IRB.CreateAdd(LHS: VrArgSize, RHS: VrOffSaveArea);
5312
5313	Value *VrRegSaveAreaShadowPtr =
5314	MSV.getShadowOriginPtr(Addr: VrRegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
5315	Alignment: Align(8), /*isStore*/ true)
5316	.first;
5317
5318	Value *VrSrcPtr = IRB.CreateInBoundsPtrAdd(
5319	Ptr: IRB.CreateInBoundsPtrAdd(Ptr: VAArgTLSCopy,
5320	Offset: IRB.getInt32(C: AArch64VrBegOffset)),
5321	Offset: VrRegSaveAreaShadowPtrOff);
5322	Value *VrCopySize = IRB.CreateSub(LHS: VrArgSize, RHS: VrRegSaveAreaShadowPtrOff);
5323
5324	IRB.CreateMemCpy(Dst: VrRegSaveAreaShadowPtr, DstAlign: Align(8), Src: VrSrcPtr, SrcAlign: Align(8),
5325	Size: VrCopySize);
5326
5327	// And finally for remaining arguments.
5328	Value *StackSaveAreaShadowPtr =
5329	MSV.getShadowOriginPtr(Addr: StackSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
5330	Alignment: Align(16), /*isStore*/ true)
5331	.first;
5332
5333	Value *StackSrcPtr = IRB.CreateInBoundsPtrAdd(
5334	Ptr: VAArgTLSCopy, Offset: IRB.getInt32(C: AArch64VAEndOffset));
5335
5336	IRB.CreateMemCpy(Dst: StackSaveAreaShadowPtr, DstAlign: Align(16), Src: StackSrcPtr,
5337	SrcAlign: Align(16), Size: VAArgOverflowSize);
5338	}
5339	}
5340	};
5341
5342	/// PowerPC64-specific implementation of VarArgHelper.
5343	struct VarArgPowerPC64Helper : public VarArgHelperBase {
5344	AllocaInst *VAArgTLSCopy = nullptr;
5345	Value *VAArgSize = nullptr;
5346
5347	VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
5348	MemorySanitizerVisitor &MSV)
5349	: VarArgHelperBase(F, MS, MSV, /*VAListTagSize=*/8) {}
5350
5351	void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5352	// For PowerPC, we need to deal with alignment of stack arguments -
5353	// they are mostly aligned to 8 bytes, but vectors and i128 arrays
5354	// are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
5355	// For that reason, we compute current offset from stack pointer (which is
5356	// always properly aligned), and offset for the first vararg, then subtract
5357	// them.
5358	unsigned VAArgBase;
5359	Triple TargetTriple(F.getParent()->getTargetTriple());
5360	// Parameter save area starts at 48 bytes from frame pointer for ABIv1,
5361	// and 32 bytes for ABIv2. This is usually determined by target
5362	// endianness, but in theory could be overridden by function attribute.
5363	if (TargetTriple.getArch() == Triple::ppc64)
5364	VAArgBase = 48;
5365	else
5366	VAArgBase = 32;
5367	unsigned VAArgOffset = VAArgBase;
5368	const DataLayout &DL = F.getParent()->getDataLayout();
5369	for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
5370	bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5371	bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
5372	if (IsByVal) {
5373	assert(A->getType()->isPointerTy());
5374	Type *RealTy = CB.getParamByValType(ArgNo);
5375	uint64_t ArgSize = DL.getTypeAllocSize(Ty: RealTy);
5376	Align ArgAlign = CB.getParamAlign(ArgNo).value_or(u: Align(8));
5377	if (ArgAlign < 8)
5378	ArgAlign = Align(8);
5379	VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
5380	if (!IsFixed) {
5381	Value *Base = getShadowPtrForVAArgument(
5382	Ty: RealTy, IRB, ArgOffset: VAArgOffset - VAArgBase, ArgSize);
5383	if (Base) {
5384	Value *AShadowPtr, *AOriginPtr;
5385	std::tie(args&: AShadowPtr, args&: AOriginPtr) =
5386	MSV.getShadowOriginPtr(Addr: A, IRB, ShadowTy: IRB.getInt8Ty(),
5387	Alignment: kShadowTLSAlignment, /*isStore*/ false);
5388
5389	IRB.CreateMemCpy(Dst: Base, DstAlign: kShadowTLSAlignment, Src: AShadowPtr,
5390	SrcAlign: kShadowTLSAlignment, Size: ArgSize);
5391	}
5392	}
5393	VAArgOffset += alignTo(Size: ArgSize, A: Align(8));
5394	} else {
5395	Value *Base;
5396	uint64_t ArgSize = DL.getTypeAllocSize(Ty: A->getType());
5397	Align ArgAlign = Align(8);
5398	if (A->getType()->isArrayTy()) {
5399	// Arrays are aligned to element size, except for long double
5400	// arrays, which are aligned to 8 bytes.
5401	Type *ElementTy = A->getType()->getArrayElementType();
5402	if (!ElementTy->isPPC_FP128Ty())
5403	ArgAlign = Align(DL.getTypeAllocSize(Ty: ElementTy));
5404	} else if (A->getType()->isVectorTy()) {
5405	// Vectors are naturally aligned.
5406	ArgAlign = Align(ArgSize);
5407	}
5408	if (ArgAlign < 8)
5409	ArgAlign = Align(8);
5410	VAArgOffset = alignTo(Size: VAArgOffset, A: ArgAlign);
5411	if (DL.isBigEndian()) {
5412	// Adjusting the shadow for argument with size < 8 to match the
5413	// placement of bits in big endian system
5414	if (ArgSize < 8)
5415	VAArgOffset += (8 - ArgSize);
5416	}
5417	if (!IsFixed) {
5418	Base = getShadowPtrForVAArgument(Ty: A->getType(), IRB,
5419	ArgOffset: VAArgOffset - VAArgBase, ArgSize);
5420	if (Base)
5421	IRB.CreateAlignedStore(Val: MSV.getShadow(V: A), Ptr: Base, Align: kShadowTLSAlignment);
5422	}
5423	VAArgOffset += ArgSize;
5424	VAArgOffset = alignTo(Size: VAArgOffset, A: Align(8));
5425	}
5426	if (IsFixed)
5427	VAArgBase = VAArgOffset;
5428	}
5429
5430	Constant *TotalVAArgSize =
5431	ConstantInt::get(Ty: IRB.getInt64Ty(), V: VAArgOffset - VAArgBase);
5432	// Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
5433	// a new class member i.e. it is the total size of all VarArgs.
5434	IRB.CreateStore(Val: TotalVAArgSize, Ptr: MS.VAArgOverflowSizeTLS);
5435	}
5436
5437	void finalizeInstrumentation() override {
5438	assert(!VAArgSize && !VAArgTLSCopy &&
5439	"finalizeInstrumentation called twice");
5440	IRBuilder<> IRB(MSV.FnPrologueEnd);
5441	VAArgSize = IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
5442	Value *CopySize =
5443	IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: 0), RHS: VAArgSize);
5444
5445	if (!VAStartInstrumentationList.empty()) {
5446	// If there is a va_start in this function, make a backup copy of
5447	// va_arg_tls somewhere in the function entry block.
5448
5449	VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
5450	VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
5451	IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
5452	Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
5453
5454	Value *SrcSize = IRB.CreateBinaryIntrinsic(
5455	Intrinsic::umin, CopySize,
5456	ConstantInt::get(MS.IntptrTy, kParamTLSSize));
5457	IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
5458	SrcAlign: kShadowTLSAlignment, Size: SrcSize);
5459	}
5460
5461	// Instrument va_start.
5462	// Copy va_list shadow from the backup copy of the TLS contents.
5463	for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
5464	CallInst *OrigInst = VAStartInstrumentationList[i];
5465	NextNodeIRBuilder IRB(OrigInst);
5466	Value *VAListTag = OrigInst->getArgOperand(i: 0);
5467	Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C); // i64*
5468	Value *RegSaveAreaPtrPtr =
5469	IRB.CreateIntToPtr(V: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
5470	DestTy: PointerType::get(ElementType: RegSaveAreaPtrTy, AddressSpace: 0));
5471	Value *RegSaveAreaPtr =
5472	IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
5473	Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5474	const Align Alignment = Align(8);
5475	std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
5476	MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
5477	Alignment, /*isStore*/ true);
5478	IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
5479	Size: CopySize);
5480	}
5481	}
5482	};
5483
5484	/// SystemZ-specific implementation of VarArgHelper.
5485	struct VarArgSystemZHelper : public VarArgHelperBase {
5486	static const unsigned SystemZGpOffset = 16;
5487	static const unsigned SystemZGpEndOffset = 56;
5488	static const unsigned SystemZFpOffset = 128;
5489	static const unsigned SystemZFpEndOffset = 160;
5490	static const unsigned SystemZMaxVrArgs = 8;
5491	static const unsigned SystemZRegSaveAreaSize = 160;
5492	static const unsigned SystemZOverflowOffset = 160;
5493	static const unsigned SystemZVAListTagSize = 32;
5494	static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5495	static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5496
5497	bool IsSoftFloatABI;
5498	AllocaInst *VAArgTLSCopy = nullptr;
5499	AllocaInst *VAArgTLSOriginCopy = nullptr;
5500	Value *VAArgOverflowSize = nullptr;
5501
5502	enum class ArgKind {
5503	GeneralPurpose,
5504	FloatingPoint,
5505	Vector,
5506	Memory,
5507	Indirect,
5508	};
5509
5510	enum class ShadowExtension { None, Zero, Sign };
5511
5512	VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
5513	MemorySanitizerVisitor &MSV)
5514	: VarArgHelperBase(F, MS, MSV, SystemZVAListTagSize),
5515	IsSoftFloatABI(F.getFnAttribute(Kind: "use-soft-float").getValueAsBool()) {}
5516
5517	ArgKind classifyArgument(Type *T) {
5518	// T is a SystemZABIInfo::classifyArgumentType() output, and there are
5519	// only a few possibilities of what it can be. In particular, enums, single
5520	// element structs and large types have already been taken care of.
5521
5522	// Some i128 and fp128 arguments are converted to pointers only in the
5523	// back end.
5524	if (T->isIntegerTy(Bitwidth: 128) || T->isFP128Ty())
5525	return ArgKind::Indirect;
5526	if (T->isFloatingPointTy())
5527	return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5528	if (T->isIntegerTy() || T->isPointerTy())
5529	return ArgKind::GeneralPurpose;
5530	if (T->isVectorTy())
5531	return ArgKind::Vector;
5532	return ArgKind::Memory;
5533	}
5534
5535	ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
5536	// ABI says: "One of the simple integer types no more than 64 bits wide.
5537	// ... If such an argument is shorter than 64 bits, replace it by a full
5538	// 64-bit integer representing the same number, using sign or zero
5539	// extension". Shadow for an integer argument has the same type as the
5540	// argument itself, so it can be sign or zero extended as well.
5541	bool ZExt = CB.paramHasAttr(ArgNo, Attribute::ZExt);
5542	bool SExt = CB.paramHasAttr(ArgNo, Attribute::SExt);
5543	if (ZExt) {
5544	assert(!SExt);
5545	return ShadowExtension::Zero;
5546	}
5547	if (SExt) {
5548	assert(!ZExt);
5549	return ShadowExtension::Sign;
5550	}
5551	return ShadowExtension::None;
5552	}
5553
5554	void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5555	unsigned GpOffset = SystemZGpOffset;
5556	unsigned FpOffset = SystemZFpOffset;
5557	unsigned VrIndex = 0;
5558	unsigned OverflowOffset = SystemZOverflowOffset;
5559	const DataLayout &DL = F.getParent()->getDataLayout();
5560	for (const auto &[ArgNo, A] : llvm::enumerate(First: CB.args())) {
5561	bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5562	// SystemZABIInfo does not produce ByVal parameters.
5563	assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal));
5564	Type *T = A->getType();
5565	ArgKind AK = classifyArgument(T);
5566	if (AK == ArgKind::Indirect) {
5567	T = PointerType::get(ElementType: T, AddressSpace: 0);
5568	AK = ArgKind::GeneralPurpose;
5569	}
5570	if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5571	AK = ArgKind::Memory;
5572	if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5573	AK = ArgKind::Memory;
5574	if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5575	AK = ArgKind::Memory;
5576	Value *ShadowBase = nullptr;
5577	Value *OriginBase = nullptr;
5578	ShadowExtension SE = ShadowExtension::None;
5579	switch (AK) {
5580	case ArgKind::GeneralPurpose: {
5581	// Always keep track of GpOffset, but store shadow only for varargs.
5582	uint64_t ArgSize = 8;
5583	if (GpOffset + ArgSize <= kParamTLSSize) {
5584	if (!IsFixed) {
5585	SE = getShadowExtension(CB, ArgNo);
5586	uint64_t GapSize = 0;
5587	if (SE == ShadowExtension::None) {
5588	uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
5589	assert(ArgAllocSize <= ArgSize);
5590	GapSize = ArgSize - ArgAllocSize;
5591	}
5592	ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
5593	if (MS.TrackOrigins)
5594	OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: GpOffset + GapSize);
5595	}
5596	GpOffset += ArgSize;
5597	} else {
5598	GpOffset = kParamTLSSize;
5599	}
5600	break;
5601	}
5602	case ArgKind::FloatingPoint: {
5603	// Always keep track of FpOffset, but store shadow only for varargs.
5604	uint64_t ArgSize = 8;
5605	if (FpOffset + ArgSize <= kParamTLSSize) {
5606	if (!IsFixed) {
5607	// PoP says: "A short floating-point datum requires only the
5608	// left-most 32 bit positions of a floating-point register".
5609	// Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
5610	// don't extend shadow and don't mind the gap.
5611	ShadowBase = getShadowAddrForVAArgument(IRB, ArgOffset: FpOffset);
5612	if (MS.TrackOrigins)
5613	OriginBase = getOriginPtrForVAArgument(IRB, ArgOffset: FpOffset);
5614	}
5615	FpOffset += ArgSize;
5616	} else {
5617	FpOffset = kParamTLSSize;
5618	}
5619	break;
5620	}
5621	case ArgKind::Vector: {
5622	// Keep track of VrIndex. No need to store shadow, since vector varargs
5623	// go through AK_Memory.
5624	assert(IsFixed);
5625	VrIndex++;
5626	break;
5627	}
5628	case ArgKind::Memory: {
5629	// Keep track of OverflowOffset and store shadow only for varargs.
5630	// Ignore fixed args, since we need to copy only the vararg portion of
5631	// the overflow area shadow.
5632	if (!IsFixed) {
5633	uint64_t ArgAllocSize = DL.getTypeAllocSize(Ty: T);
5634	uint64_t ArgSize = alignTo(Value: ArgAllocSize, Align: 8);
5635	if (OverflowOffset + ArgSize <= kParamTLSSize) {
5636	SE = getShadowExtension(CB, ArgNo);
5637	uint64_t GapSize =
5638	SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5639	ShadowBase =
5640	getShadowAddrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
5641	if (MS.TrackOrigins)
5642	OriginBase =
5643	getOriginPtrForVAArgument(IRB, ArgOffset: OverflowOffset + GapSize);
5644	OverflowOffset += ArgSize;
5645	} else {
5646	OverflowOffset = kParamTLSSize;
5647	}
5648	}
5649	break;
5650	}
5651	case ArgKind::Indirect:
5652	llvm_unreachable("Indirect must be converted to GeneralPurpose");
5653	}
5654	if (ShadowBase == nullptr)
5655	continue;
5656	Value *Shadow = MSV.getShadow(V: A);
5657	if (SE != ShadowExtension::None)
5658	Shadow = MSV.CreateShadowCast(IRB, V: Shadow, dstTy: IRB.getInt64Ty(),
5659	/*Signed*/ SE == ShadowExtension::Sign);
5660	ShadowBase = IRB.CreateIntToPtr(
5661	V: ShadowBase, DestTy: PointerType::get(ElementType: Shadow->getType(), AddressSpace: 0), Name: "_msarg_va_s");
5662	IRB.CreateStore(Val: Shadow, Ptr: ShadowBase);
5663	if (MS.TrackOrigins) {
5664	Value *Origin = MSV.getOrigin(V: A);
5665	TypeSize StoreSize = DL.getTypeStoreSize(Ty: Shadow->getType());
5666	MSV.paintOrigin(IRB, Origin, OriginPtr: OriginBase, TS: StoreSize,
5667	Alignment: kMinOriginAlignment);
5668	}
5669	}
5670	Constant *OverflowSize = ConstantInt::get(
5671	Ty: IRB.getInt64Ty(), V: OverflowOffset - SystemZOverflowOffset);
5672	IRB.CreateStore(Val: OverflowSize, Ptr: MS.VAArgOverflowSizeTLS);
5673	}
5674
5675	void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
5676	Type *RegSaveAreaPtrTy = PointerType::getUnqual(C&: *MS.C); // i64*
5677	Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5678	V: IRB.CreateAdd(
5679	LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
5680	RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZRegSaveAreaPtrOffset)),
5681	DestTy: PointerType::get(ElementType: RegSaveAreaPtrTy, AddressSpace: 0));
5682	Value *RegSaveAreaPtr = IRB.CreateLoad(Ty: RegSaveAreaPtrTy, Ptr: RegSaveAreaPtrPtr);
5683	Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5684	const Align Alignment = Align(8);
5685	std::tie(args&: RegSaveAreaShadowPtr, args&: RegSaveAreaOriginPtr) =
5686	MSV.getShadowOriginPtr(Addr: RegSaveAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(), Alignment,
5687	/*isStore*/ true);
5688	// TODO(iii): copy only fragments filled by visitCallBase()
5689	// TODO(iii): support packed-stack && !use-soft-float
5690	// For use-soft-float functions, it is enough to copy just the GPRs.
5691	unsigned RegSaveAreaSize =
5692	IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5693	IRB.CreateMemCpy(Dst: RegSaveAreaShadowPtr, DstAlign: Alignment, Src: VAArgTLSCopy, SrcAlign: Alignment,
5694	Size: RegSaveAreaSize);
5695	if (MS.TrackOrigins)
5696	IRB.CreateMemCpy(Dst: RegSaveAreaOriginPtr, DstAlign: Alignment, Src: VAArgTLSOriginCopy,
5697	SrcAlign: Alignment, Size: RegSaveAreaSize);
5698	}
5699
5700	// FIXME: This implementation limits OverflowOffset to kParamTLSSize, so we
5701	// don't know real overflow size and can't clear shadow beyond kParamTLSSize.
5702	void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
5703	Type *OverflowArgAreaPtrTy = PointerType::getUnqual(C&: *MS.C); // i64*
5704	Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5705	V: IRB.CreateAdd(
5706	LHS: IRB.CreatePtrToInt(V: VAListTag, DestTy: MS.IntptrTy),
5707	RHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowArgAreaPtrOffset)),
5708	DestTy: PointerType::get(ElementType: OverflowArgAreaPtrTy, AddressSpace: 0));
5709	Value *OverflowArgAreaPtr =
5710	IRB.CreateLoad(Ty: OverflowArgAreaPtrTy, Ptr: OverflowArgAreaPtrPtr);
5711	Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5712	const Align Alignment = Align(8);
5713	std::tie(args&: OverflowArgAreaShadowPtr, args&: OverflowArgAreaOriginPtr) =
5714	MSV.getShadowOriginPtr(Addr: OverflowArgAreaPtr, IRB, ShadowTy: IRB.getInt8Ty(),
5715	Alignment, /*isStore*/ true);
5716	Value *SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSCopy,
5717	Idx0: SystemZOverflowOffset);
5718	IRB.CreateMemCpy(Dst: OverflowArgAreaShadowPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
5719	Size: VAArgOverflowSize);
5720	if (MS.TrackOrigins) {
5721	SrcPtr = IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(), Ptr: VAArgTLSOriginCopy,
5722	Idx0: SystemZOverflowOffset);
5723	IRB.CreateMemCpy(Dst: OverflowArgAreaOriginPtr, DstAlign: Alignment, Src: SrcPtr, SrcAlign: Alignment,
5724	Size: VAArgOverflowSize);
5725	}
5726	}
5727
5728	void finalizeInstrumentation() override {
5729	assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5730	"finalizeInstrumentation called twice");
5731	if (!VAStartInstrumentationList.empty()) {
5732	// If there is a va_start in this function, make a backup copy of
5733	// va_arg_tls somewhere in the function entry block.
5734	IRBuilder<> IRB(MSV.FnPrologueEnd);
5735	VAArgOverflowSize =
5736	IRB.CreateLoad(Ty: IRB.getInt64Ty(), Ptr: MS.VAArgOverflowSizeTLS);
5737	Value *CopySize =
5738	IRB.CreateAdd(LHS: ConstantInt::get(Ty: MS.IntptrTy, V: SystemZOverflowOffset),
5739	RHS: VAArgOverflowSize);
5740	VAArgTLSCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
5741	VAArgTLSCopy->setAlignment(kShadowTLSAlignment);
5742	IRB.CreateMemSet(Ptr: VAArgTLSCopy, Val: Constant::getNullValue(Ty: IRB.getInt8Ty()),
5743	Size: CopySize, Align: kShadowTLSAlignment, isVolatile: false);
5744
5745	Value *SrcSize = IRB.CreateBinaryIntrinsic(
5746	Intrinsic::umin, CopySize,
5747	ConstantInt::get(MS.IntptrTy, kParamTLSSize));
5748	IRB.CreateMemCpy(Dst: VAArgTLSCopy, DstAlign: kShadowTLSAlignment, Src: MS.VAArgTLS,
5749	SrcAlign: kShadowTLSAlignment, Size: SrcSize);
5750	if (MS.TrackOrigins) {
5751	VAArgTLSOriginCopy = IRB.CreateAlloca(Ty: Type::getInt8Ty(C&: *MS.C), ArraySize: CopySize);
5752	VAArgTLSOriginCopy->setAlignment(kShadowTLSAlignment);
5753	IRB.CreateMemCpy(Dst: VAArgTLSOriginCopy, DstAlign: kShadowTLSAlignment,
5754	Src: MS.VAArgOriginTLS, SrcAlign: kShadowTLSAlignment, Size: SrcSize);
5755	}
5756	}
5757
5758	// Instrument va_start.
5759	// Copy va_list shadow from the backup copy of the TLS contents.
5760	for (size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.size();
5761	VaStartNo < VaStartNum; VaStartNo++) {
5762	CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5763	NextNodeIRBuilder IRB(OrigInst);
5764	Value *VAListTag = OrigInst->getArgOperand(i: 0);
5765	copyRegSaveArea(IRB, VAListTag);
5766	copyOverflowArea(IRB, VAListTag);
5767	}
5768	}
5769	};
5770
5771	// Loongarch64 is not a MIPS, but the current vargs calling convention matches
5772	// the MIPS.
5773	using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5774
5775	/// A no-op implementation of VarArgHelper.
5776	struct VarArgNoOpHelper : public VarArgHelper {
5777	VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
5778	MemorySanitizerVisitor &MSV) {}
5779
5780	void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
5781
5782	void visitVAStartInst(VAStartInst &I) override {}
5783
5784	void visitVACopyInst(VACopyInst &I) override {}
5785
5786	void finalizeInstrumentation() override {}
5787	};
5788
5789	} // end anonymous namespace
5790
5791	static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
5792	MemorySanitizerVisitor &Visitor) {
5793	// VarArg handling is only implemented on AMD64. False positives are possible
5794	// on other platforms.
5795	Triple TargetTriple(Func.getParent()->getTargetTriple());
5796	if (TargetTriple.getArch() == Triple::x86_64)
5797	return new VarArgAMD64Helper(Func, Msan, Visitor);
5798	else if (TargetTriple.isMIPS64())
5799	return new VarArgMIPS64Helper(Func, Msan, Visitor);
5800	else if (TargetTriple.getArch() == Triple::aarch64)
5801	return new VarArgAArch64Helper(Func, Msan, Visitor);
5802	else if (TargetTriple.getArch() == Triple::ppc64 ||
5803	TargetTriple.getArch() == Triple::ppc64le)
5804	return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5805	else if (TargetTriple.getArch() == Triple::systemz)
5806	return new VarArgSystemZHelper(Func, Msan, Visitor);
5807	else if (TargetTriple.isLoongArch64())
5808	return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5809	else
5810	return new VarArgNoOpHelper(Func, Msan, Visitor);
5811	}
5812
5813	bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
5814	if (!CompileKernel && F.getName() == kMsanModuleCtorName)
5815	return false;
5816
5817	if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5818	return false;
5819
5820	MemorySanitizerVisitor Visitor(F, *this, TLI);
5821
5822	// Clear out memory attributes.
5823	AttributeMask B;
5824	B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5825	F.removeFnAttrs(Attrs: B);
5826
5827	return Visitor.runOnFunction();
5828	}
5829