1	/* AddressSanitizer, a fast memory error detector.
2	Copyright (C) 2012-2023 Free Software Foundation, Inc.
3	Contributed by Kostya Serebryany <kcc@google.com>
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify it under
8	the terms of the GNU General Public License as published by the Free
9	Software Foundation; either version 3, or (at your option) any later
10	version.
11
12	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13	WARRANTY; without even the implied warranty of MERCHANTABILITY or
14	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15	for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21
22	#include "config.h"
23	#include "system.h"
24	#include "coretypes.h"
25	#include "backend.h"
26	#include "target.h"
27	#include "rtl.h"
28	#include "tree.h"
29	#include "gimple.h"
30	#include "cfghooks.h"
31	#include "alloc-pool.h"
32	#include "tree-pass.h"
33	#include "memmodel.h"
34	#include "tm_p.h"
35	#include "ssa.h"
36	#include "stringpool.h"
37	#include "tree-ssanames.h"
38	#include "optabs.h"
39	#include "emit-rtl.h"
40	#include "cgraph.h"
41	#include "gimple-pretty-print.h"
42	#include "alias.h"
43	#include "fold-const.h"
44	#include "cfganal.h"
45	#include "gimplify.h"
46	#include "gimple-iterator.h"
47	#include "varasm.h"
48	#include "stor-layout.h"
49	#include "tree-iterator.h"
50	#include "stringpool.h"
51	#include "attribs.h"
52	#include "asan.h"
53	#include "dojump.h"
54	#include "explow.h"
55	#include "expr.h"
56	#include "output.h"
57	#include "langhooks.h"
58	#include "cfgloop.h"
59	#include "gimple-builder.h"
60	#include "gimple-fold.h"
61	#include "ubsan.h"
62	#include "builtins.h"
63	#include "fnmatch.h"
64	#include "tree-inline.h"
65	#include "tree-ssa.h"
66	#include "tree-eh.h"
67	#include "diagnostic-core.h"
68
69	/* AddressSanitizer finds out-of-bounds and use-after-free bugs
70	with <2x slowdown on average.
71
72	The tool consists of two parts:
73	instrumentation module (this file) and a run-time library.
74	The instrumentation module adds a run-time check before every memory insn.
75	For a 8- or 16- byte load accessing address X:
76	ShadowAddr = (X >> 3) + Offset
77	ShadowValue = *(char*)ShadowAddr; // *(short*) for 16-byte access.
78	if (ShadowValue)
79	__asan_report_load8(X);
80	For a load of N bytes (N=1, 2 or 4) from address X:
81	ShadowAddr = (X >> 3) + Offset
82	ShadowValue = *(char*)ShadowAddr;
83	if (ShadowValue)
84	if ((X & 7) + N - 1 > ShadowValue)
85	__asan_report_loadN(X);
86	Stores are instrumented similarly, but using __asan_report_storeN functions.
87	A call too __asan_init_vN() is inserted to the list of module CTORs.
88	N is the version number of the AddressSanitizer API. The changes between the
89	API versions are listed in libsanitizer/asan/asan_interface_internal.h.
90
91	The run-time library redefines malloc (so that redzone are inserted around
92	the allocated memory) and free (so that reuse of free-ed memory is delayed),
93	provides __asan_report* and __asan_init_vN functions.
94
95	Read more:
96	http://code.google.com/p/address-sanitizer/wiki/AddressSanitizerAlgorithm
97
98	The current implementation supports detection of out-of-bounds and
99	use-after-free in the heap, on the stack and for global variables.
100
101	[Protection of stack variables]
102
103	To understand how detection of out-of-bounds and use-after-free works
104	for stack variables, lets look at this example on x86_64 where the
105	stack grows downward:
106
107	int
108	foo ()
109	{
110	char a[24] = {0};
111	int b[2] = {0};
112
113	a[5] = 1;
114	b[1] = 2;
115
116	return a[5] + b[1];
117	}
118
119	For this function, the stack protected by asan will be organized as
120	follows, from the top of the stack to the bottom:
121
122	Slot 1/ [red zone of 32 bytes called 'RIGHT RedZone']
123
124	Slot 2/ [8 bytes of red zone, that adds up to the space of 'a' to make
125	the next slot be 32 bytes aligned; this one is called Partial
126	Redzone; this 32 bytes alignment is an asan constraint]
127
128	Slot 3/ [24 bytes for variable 'a']
129
130	Slot 4/ [red zone of 32 bytes called 'Middle RedZone']
131
132	Slot 5/ [24 bytes of Partial Red Zone (similar to slot 2]
133
134	Slot 6/ [8 bytes for variable 'b']
135
136	Slot 7/ [32 bytes of Red Zone at the bottom of the stack, called
137	'LEFT RedZone']
138
139	The 32 bytes of LEFT red zone at the bottom of the stack can be
140	decomposed as such:
141
142	1/ The first 8 bytes contain a magical asan number that is always
143	0x41B58AB3.
144
145	2/ The following 8 bytes contains a pointer to a string (to be
146	parsed at runtime by the runtime asan library), which format is
147	the following:
148
149	"<function-name> <space> <num-of-variables-on-the-stack>
150	(<32-bytes-aligned-offset-in-bytes-of-variable> <space>
151	<length-of-var-in-bytes> ){n} "
152
153	where '(...){n}' means the content inside the parenthesis occurs 'n'
154	times, with 'n' being the number of variables on the stack.
155
156	3/ The following 8 bytes contain the PC of the current function which
157	will be used by the run-time library to print an error message.
158
159	4/ The following 8 bytes are reserved for internal use by the run-time.
160
161	The shadow memory for that stack layout is going to look like this:
162
163	- content of shadow memory 8 bytes for slot 7: 0xF1F1F1F1.
164	The F1 byte pattern is a magic number called
165	ASAN_STACK_MAGIC_LEFT and is a way for the runtime to know that
166	the memory for that shadow byte is part of a the LEFT red zone
167	intended to seat at the bottom of the variables on the stack.
168
169	- content of shadow memory 8 bytes for slots 6 and 5:
170	0xF4F4F400. The F4 byte pattern is a magic number
171	called ASAN_STACK_MAGIC_PARTIAL. It flags the fact that the
172	memory region for this shadow byte is a PARTIAL red zone
173	intended to pad a variable A, so that the slot following
174	{A,padding} is 32 bytes aligned.
175
176	Note that the fact that the least significant byte of this
177	shadow memory content is 00 means that 8 bytes of its
178	corresponding memory (which corresponds to the memory of
179	variable 'b') is addressable.
180
181	- content of shadow memory 8 bytes for slot 4: 0xF2F2F2F2.
182	The F2 byte pattern is a magic number called
183	ASAN_STACK_MAGIC_MIDDLE. It flags the fact that the memory
184	region for this shadow byte is a MIDDLE red zone intended to
185	seat between two 32 aligned slots of {variable,padding}.
186
187	- content of shadow memory 8 bytes for slot 3 and 2:
188	0xF4000000. This represents is the concatenation of
189	variable 'a' and the partial red zone following it, like what we
190	had for variable 'b'. The least significant 3 bytes being 00
191	means that the 3 bytes of variable 'a' are addressable.
192
193	- content of shadow memory 8 bytes for slot 1: 0xF3F3F3F3.
194	The F3 byte pattern is a magic number called
195	ASAN_STACK_MAGIC_RIGHT. It flags the fact that the memory
196	region for this shadow byte is a RIGHT red zone intended to seat
197	at the top of the variables of the stack.
198
199	Note that the real variable layout is done in expand_used_vars in
200	cfgexpand.cc. As far as Address Sanitizer is concerned, it lays out
201	stack variables as well as the different red zones, emits some
202	prologue code to populate the shadow memory as to poison (mark as
203	non-accessible) the regions of the red zones and mark the regions of
204	stack variables as accessible, and emit some epilogue code to
205	un-poison (mark as accessible) the regions of red zones right before
206	the function exits.
207
208	[Protection of global variables]
209
210	The basic idea is to insert a red zone between two global variables
211	and install a constructor function that calls the asan runtime to do
212	the populating of the relevant shadow memory regions at load time.
213
214	So the global variables are laid out as to insert a red zone between
215	them. The size of the red zones is so that each variable starts on a
216	32 bytes boundary.
217
218	Then a constructor function is installed so that, for each global
219	variable, it calls the runtime asan library function
220	__asan_register_globals_with an instance of this type:
221
222	struct __asan_global
223	{
224	// Address of the beginning of the global variable.
225	const void *__beg;
226
227	// Initial size of the global variable.
228	uptr __size;
229
230	// Size of the global variable + size of the red zone. This
231	// size is 32 bytes aligned.
232	uptr __size_with_redzone;
233
234	// Name of the global variable.
235	const void *__name;
236
237	// Name of the module where the global variable is declared.
238	const void *__module_name;
239
240	// 1 if it has dynamic initialization, 0 otherwise.
241	uptr __has_dynamic_init;
242
243	// A pointer to struct that contains source location, could be NULL.
244	__asan_global_source_location *__location;
245	}
246
247	A destructor function that calls the runtime asan library function
248	_asan_unregister_globals is also installed. */
249
250	static unsigned HOST_WIDE_INT asan_shadow_offset_value;
251	static bool asan_shadow_offset_computed;
252	static vec<char *> sanitized_sections;
253	static tree last_alloca_addr;
254
255	/* Set of variable declarations that are going to be guarded by
256	use-after-scope sanitizer. */
257
258	hash_set<tree> *asan_handled_variables = NULL;
259
260	hash_set <tree> *asan_used_labels = NULL;
261
262	/* Global variables for HWASAN stack tagging. */
263	/* hwasan_frame_tag_offset records the offset from the frame base tag that the
264	next object should have. */
265	static uint8_t hwasan_frame_tag_offset = 0;
266	/* hwasan_frame_base_ptr is a pointer with the same address as
267	`virtual_stack_vars_rtx` for the current frame, and with the frame base tag
268	stored in it. N.b. this global RTX does not need to be marked GTY, but is
269	done so anyway. The need is not there since all uses are in just one pass
270	(cfgexpand) and there are no calls to ggc_collect between the uses. We mark
271	it GTY(()) anyway to allow the use of the variable later on if needed by
272	future features. */
273	static GTY(()) rtx hwasan_frame_base_ptr = NULL_RTX;
274	/* hwasan_frame_base_init_seq is the sequence of RTL insns that will initialize
275	the hwasan_frame_base_ptr. When the hwasan_frame_base_ptr is requested, we
276	generate this sequence but do not emit it. If the sequence was created it
277	is emitted once the function body has been expanded.
278
279	This delay is because the frame base pointer may be needed anywhere in the
280	function body, or needed by the expand_used_vars function. Emitting once in
281	a known place is simpler than requiring the emission of the instructions to
282	be know where it should go depending on the first place the hwasan frame
283	base is needed. */
284	static GTY(()) rtx_insn *hwasan_frame_base_init_seq = NULL;
285
286	/* Structure defining the extent of one object on the stack that HWASAN needs
287	to tag in the corresponding shadow stack space.
288
289	The range this object spans on the stack is between `untagged_base +
290	nearest_offset` and `untagged_base + farthest_offset`.
291	`tagged_base` is an rtx containing the same value as `untagged_base` but
292	with a random tag stored in the top byte. We record both `untagged_base`
293	and `tagged_base` so that `hwasan_emit_prologue` can use both without having
294	to emit RTL into the instruction stream to re-calculate one from the other.
295	(`hwasan_emit_prologue` needs to use both bases since the
296	__hwasan_tag_memory call it emits uses an untagged value, and it calculates
297	the tag to store in shadow memory based on the tag_offset plus the tag in
298	tagged_base). */
299	struct hwasan_stack_var
300	{
301	rtx untagged_base;
302	rtx tagged_base;
303	poly_int64 nearest_offset;
304	poly_int64 farthest_offset;
305	uint8_t tag_offset;
306	};
307
308	/* Variable recording all stack variables that HWASAN needs to tag.
309	Does not need to be marked as GTY(()) since every use is in the cfgexpand
310	pass and gcc_collect is not called in the middle of that pass. */
311	static vec<hwasan_stack_var> hwasan_tagged_stack_vars;
312
313
314	/* Sets shadow offset to value in string VAL. */
315
316	bool
317	set_asan_shadow_offset (const char *val)
318	{
319	char *endp;
320
321	errno = 0;
322	#ifdef HAVE_LONG_LONG
323	asan_shadow_offset_value = strtoull (nptr: val, endptr: &endp, base: 0);
324	#else
325	asan_shadow_offset_value = strtoul (val, &endp, 0);
326	#endif
327	if (!(*val != '\0' && *endp == '\0' && errno == 0))
328	return false;
329
330	asan_shadow_offset_computed = true;
331
332	return true;
333	}
334
335	/* Set list of user-defined sections that need to be sanitized. */
336
337	void
338	set_sanitized_sections (const char *sections)
339	{
340	char *pat;
341	unsigned i;
342	FOR_EACH_VEC_ELT (sanitized_sections, i, pat)
343	free (ptr: pat);
344	sanitized_sections.truncate (size: 0);
345
346	for (const char *s = sections; *s; )
347	{
348	const char *end;
349	for (end = s; *end && *end != ','; ++end);
350	size_t len = end - s;
351	sanitized_sections.safe_push (obj: xstrndup (s, len));
352	s = *end ? end + 1 : end;
353	}
354	}
355
356	bool
357	asan_mark_p (gimple *stmt, enum asan_mark_flags flag)
358	{
359	return (gimple_call_internal_p (gs: stmt, fn: IFN_ASAN_MARK)
360	&& tree_to_uhwi (gimple_call_arg (gs: stmt, index: 0)) == flag);
361	}
362
363	bool
364	asan_sanitize_stack_p (void)
365	{
366	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_stack);
367	}
368
369	bool
370	asan_sanitize_allocas_p (void)
371	{
372	return (asan_sanitize_stack_p () && param_asan_protect_allocas);
373	}
374
375	bool
376	asan_instrument_reads (void)
377	{
378	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_instrument_reads);
379	}
380
381	bool
382	asan_instrument_writes (void)
383	{
384	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_instrument_writes);
385	}
386
387	bool
388	asan_memintrin (void)
389	{
390	return (sanitize_flags_p (flag: SANITIZE_ADDRESS) && param_asan_memintrin);
391	}
392
393
394	/* Support for --param asan-kernel-mem-intrinsic-prefix=1. */
395	static GTY(()) rtx asan_memfn_rtls[3];
396
397	rtx
398	asan_memfn_rtl (tree fndecl)
399	{
400	int i;
401	const char *f, *p;
402	char buf[sizeof ("__hwasan_memmove")];
403
404	switch (DECL_FUNCTION_CODE (decl: fndecl))
405	{
406	case BUILT_IN_MEMCPY: i = 0; f = "memcpy"; break;
407	case BUILT_IN_MEMSET: i = 1; f = "memset"; break;
408	case BUILT_IN_MEMMOVE: i = 2; f = "memmove"; break;
409	default: gcc_unreachable ();
410	}
411	if (asan_memfn_rtls[i] == NULL_RTX)
412	{
413	tree save_name = DECL_NAME (fndecl);
414	tree save_assembler_name = DECL_ASSEMBLER_NAME (fndecl);
415	rtx save_rtl = DECL_RTL (fndecl);
416	if (flag_sanitize & SANITIZE_KERNEL_HWADDRESS)
417	p = "__hwasan_";
418	else
419	p = "__asan_";
420	strcpy (dest: buf, src: p);
421	strcat (dest: buf, src: f);
422	DECL_NAME (fndecl) = get_identifier (buf);
423	DECL_ASSEMBLER_NAME_RAW (fndecl) = NULL_TREE;
424	SET_DECL_RTL (fndecl, NULL_RTX);
425	asan_memfn_rtls[i] = DECL_RTL (fndecl);
426	DECL_NAME (fndecl) = save_name;
427	DECL_ASSEMBLER_NAME_RAW (fndecl) = save_assembler_name;
428	SET_DECL_RTL (fndecl, save_rtl);
429	}
430	return asan_memfn_rtls[i];
431	}
432
433
434	/* Checks whether section SEC should be sanitized. */
435
436	static bool
437	section_sanitized_p (const char *sec)
438	{
439	char *pat;
440	unsigned i;
441	FOR_EACH_VEC_ELT (sanitized_sections, i, pat)
442	if (fnmatch (pattern: pat, string: sec, FNM_PERIOD) == 0)
443	return true;
444	return false;
445	}
446
447	/* Returns Asan shadow offset. */
448
449	static unsigned HOST_WIDE_INT
450	asan_shadow_offset ()
451	{
452	if (!asan_shadow_offset_computed)
453	{
454	asan_shadow_offset_computed = true;
455	asan_shadow_offset_value = targetm.asan_shadow_offset ();
456	}
457	return asan_shadow_offset_value;
458	}
459
460	/* Returns Asan shadow offset has been set. */
461	bool
462	asan_shadow_offset_set_p ()
463	{
464	return asan_shadow_offset_computed;
465	}
466
467	alias_set_type asan_shadow_set = -1;
468
469	/* Pointer types to 1, 2 or 4 byte integers in shadow memory. A separate
470	alias set is used for all shadow memory accesses. */
471	static GTY(()) tree shadow_ptr_types[3];
472
473	/* Decl for __asan_option_detect_stack_use_after_return. */
474	static GTY(()) tree asan_detect_stack_use_after_return;
475
476	/* Hashtable support for memory references used by gimple
477	statements. */
478
479	/* This type represents a reference to a memory region. */
480	struct asan_mem_ref
481	{
482	/* The expression of the beginning of the memory region. */
483	tree start;
484
485	/* The size of the access. */
486	HOST_WIDE_INT access_size;
487	};
488
489	object_allocator <asan_mem_ref> asan_mem_ref_pool ("asan_mem_ref");
490
491	/* Initializes an instance of asan_mem_ref. */
492
493	static void
494	asan_mem_ref_init (asan_mem_ref *ref, tree start, HOST_WIDE_INT access_size)
495	{
496	ref->start = start;
497	ref->access_size = access_size;
498	}
499
500	/* Allocates memory for an instance of asan_mem_ref into the memory
501	pool returned by asan_mem_ref_get_alloc_pool and initialize it.
502	START is the address of (or the expression pointing to) the
503	beginning of memory reference. ACCESS_SIZE is the size of the
504	access to the referenced memory. */
505
506	static asan_mem_ref*
507	asan_mem_ref_new (tree start, HOST_WIDE_INT access_size)
508	{
509	asan_mem_ref *ref = asan_mem_ref_pool.allocate ();
510
511	asan_mem_ref_init (ref, start, access_size);
512	return ref;
513	}
514
515	/* This builds and returns a pointer to the end of the memory region
516	that starts at START and of length LEN. */
517
518	tree
519	asan_mem_ref_get_end (tree start, tree len)
520	{
521	if (len == NULL_TREE || integer_zerop (len))
522	return start;
523
524	if (!ptrofftype_p (type: len))
525	len = convert_to_ptrofftype (len);
526
527	return fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (start), start, len);
528	}
529
530	/* Return a tree expression that represents the end of the referenced
531	memory region. Beware that this function can actually build a new
532	tree expression. */
533
534	tree
535	asan_mem_ref_get_end (const asan_mem_ref *ref, tree len)
536	{
537	return asan_mem_ref_get_end (start: ref->start, len);
538	}
539
540	struct asan_mem_ref_hasher : nofree_ptr_hash <asan_mem_ref>
541	{
542	static inline hashval_t hash (const asan_mem_ref *);
543	static inline bool equal (const asan_mem_ref *, const asan_mem_ref *);
544	};
545
546	/* Hash a memory reference. */
547
548	inline hashval_t
549	asan_mem_ref_hasher::hash (const asan_mem_ref *mem_ref)
550	{
551	return iterative_hash_expr (tree: mem_ref->start, seed: 0);
552	}
553
554	/* Compare two memory references. We accept the length of either
555	memory references to be NULL_TREE. */
556
557	inline bool
558	asan_mem_ref_hasher::equal (const asan_mem_ref *m1,
559	const asan_mem_ref *m2)
560	{
561	return operand_equal_p (m1->start, m2->start, flags: 0);
562	}
563
564	static hash_table<asan_mem_ref_hasher> *asan_mem_ref_ht;
565
566	/* Returns a reference to the hash table containing memory references.
567	This function ensures that the hash table is created. Note that
568	this hash table is updated by the function
569	update_mem_ref_hash_table. */
570
571	static hash_table<asan_mem_ref_hasher> *
572	get_mem_ref_hash_table ()
573	{
574	if (!asan_mem_ref_ht)
575	asan_mem_ref_ht = new hash_table<asan_mem_ref_hasher> (10);
576
577	return asan_mem_ref_ht;
578	}
579
580	/* Clear all entries from the memory references hash table. */
581
582	static void
583	empty_mem_ref_hash_table ()
584	{
585	if (asan_mem_ref_ht)
586	asan_mem_ref_ht->empty ();
587	}
588
589	/* Free the memory references hash table. */
590
591	static void
592	free_mem_ref_resources ()
593	{
594	delete asan_mem_ref_ht;
595	asan_mem_ref_ht = NULL;
596
597	asan_mem_ref_pool.release ();
598	}
599
600	/* Return true iff the memory reference REF has been instrumented. */
601
602	static bool
603	has_mem_ref_been_instrumented (tree ref, HOST_WIDE_INT access_size)
604	{
605	asan_mem_ref r;
606	asan_mem_ref_init (ref: &r, start: ref, access_size);
607
608	asan_mem_ref *saved_ref = get_mem_ref_hash_table ()->find (value: &r);
609	return saved_ref && saved_ref->access_size >= access_size;
610	}
611
612	/* Return true iff the memory reference REF has been instrumented. */
613
614	static bool
615	has_mem_ref_been_instrumented (const asan_mem_ref *ref)
616	{
617	return has_mem_ref_been_instrumented (ref: ref->start, access_size: ref->access_size);
618	}
619
620	/* Return true iff access to memory region starting at REF and of
621	length LEN has been instrumented. */
622
623	static bool
624	has_mem_ref_been_instrumented (const asan_mem_ref *ref, tree len)
625	{
626	HOST_WIDE_INT size_in_bytes
627	= tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
628
629	return size_in_bytes != -1
630	&& has_mem_ref_been_instrumented (ref: ref->start, access_size: size_in_bytes);
631	}
632
633	/* Set REF to the memory reference present in a gimple assignment
634	ASSIGNMENT. Return true upon successful completion, false
635	otherwise. */
636
637	static bool
638	get_mem_ref_of_assignment (const gassign *assignment,
639	asan_mem_ref *ref,
640	bool *ref_is_store)
641	{
642	gcc_assert (gimple_assign_single_p (assignment));
643
644	if (gimple_store_p (gs: assignment)
645	&& !gimple_clobber_p (s: assignment))
646	{
647	ref->start = gimple_assign_lhs (gs: assignment);
648	*ref_is_store = true;
649	}
650	else if (gimple_assign_load_p (assignment))
651	{
652	ref->start = gimple_assign_rhs1 (gs: assignment);
653	*ref_is_store = false;
654	}
655	else
656	return false;
657
658	ref->access_size = int_size_in_bytes (TREE_TYPE (ref->start));
659	return true;
660	}
661
662	/* Return address of last allocated dynamic alloca. */
663
664	static tree
665	get_last_alloca_addr ()
666	{
667	if (last_alloca_addr)
668	return last_alloca_addr;
669
670	last_alloca_addr = create_tmp_reg (ptr_type_node, "last_alloca_addr");
671	gassign *g = gimple_build_assign (last_alloca_addr, null_pointer_node);
672	edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
673	gsi_insert_on_edge_immediate (e, g);
674	return last_alloca_addr;
675	}
676
677	/* Insert __asan_allocas_unpoison (top, bottom) call before
678	__builtin_stack_restore (new_sp) call.
679	The pseudocode of this routine should look like this:
680	top = last_alloca_addr;
681	bot = new_sp;
682	__asan_allocas_unpoison (top, bot);
683	last_alloca_addr = new_sp;
684	__builtin_stack_restore (new_sp);
685	In general, we can't use new_sp as bot parameter because on some
686	architectures SP has non zero offset from dynamic stack area. Moreover, on
687	some architectures this offset (STACK_DYNAMIC_OFFSET) becomes known for each
688	particular function only after all callees were expanded to rtl.
689	The most noticeable example is PowerPC{,64}, see
690	http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi.html#DYNAM-STACK.
691	To overcome the issue we use following trick: pass new_sp as a second
692	parameter to __asan_allocas_unpoison and rewrite it during expansion with
693	new_sp + (virtual_dynamic_stack_rtx - sp) later in
694	expand_asan_emit_allocas_unpoison function.
695
696	HWASAN needs to do very similar, the eventual pseudocode should be:
697	__hwasan_tag_memory (virtual_stack_dynamic_rtx,
698	0,
699	new_sp - sp);
700	__builtin_stack_restore (new_sp)
701
702	Need to use the same trick to handle STACK_DYNAMIC_OFFSET as described
703	above. */
704
705	static void
706	handle_builtin_stack_restore (gcall *call, gimple_stmt_iterator *iter)
707	{
708	if (!iter
709	|| !(asan_sanitize_allocas_p () || hwasan_sanitize_allocas_p ()))
710	return;
711
712	tree restored_stack = gimple_call_arg (gs: call, index: 0);
713
714	gimple *g;
715
716	if (hwasan_sanitize_allocas_p ())
717	{
718	enum internal_fn fn = IFN_HWASAN_ALLOCA_UNPOISON;
719	/* There is only one piece of information `expand_HWASAN_ALLOCA_UNPOISON`
720	needs to work. This is the length of the area that we're
721	deallocating. Since the stack pointer is known at expand time, the
722	position of the new stack pointer after deallocation is enough
723	information to calculate this length. */
724	g = gimple_build_call_internal (fn, 1, restored_stack);
725	}
726	else
727	{
728	tree last_alloca = get_last_alloca_addr ();
729	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_ALLOCAS_UNPOISON);
730	g = gimple_build_call (fn, 2, last_alloca, restored_stack);
731	gsi_insert_before (iter, g, GSI_SAME_STMT);
732	g = gimple_build_assign (last_alloca, restored_stack);
733	}
734
735	gsi_insert_before (iter, g, GSI_SAME_STMT);
736	}
737
738	/* Deploy and poison redzones around __builtin_alloca call. To do this, we
739	should replace this call with another one with changed parameters and
740	replace all its uses with new address, so
741	addr = __builtin_alloca (old_size, align);
742	is replaced by
743	left_redzone_size = max (align, ASAN_RED_ZONE_SIZE);
744	Following two statements are optimized out if we know that
745	old_size & (ASAN_RED_ZONE_SIZE - 1) == 0, i.e. alloca doesn't need partial
746	redzone.
747	misalign = old_size & (ASAN_RED_ZONE_SIZE - 1);
748	partial_redzone_size = ASAN_RED_ZONE_SIZE - misalign;
749	right_redzone_size = ASAN_RED_ZONE_SIZE;
750	additional_size = left_redzone_size + partial_redzone_size +
751	right_redzone_size;
752	new_size = old_size + additional_size;
753	new_alloca = __builtin_alloca (new_size, max (align, 32))
754	__asan_alloca_poison (new_alloca, old_size)
755	addr = new_alloca + max (align, ASAN_RED_ZONE_SIZE);
756	last_alloca_addr = new_alloca;
757	ADDITIONAL_SIZE is added to make new memory allocation contain not only
758	requested memory, but also left, partial and right redzones as well as some
759	additional space, required by alignment. */
760
761	static void
762	handle_builtin_alloca (gcall *call, gimple_stmt_iterator *iter)
763	{
764	if (!iter
765	|| !(asan_sanitize_allocas_p () || hwasan_sanitize_allocas_p ()))
766	return;
767
768	gassign *g;
769	gcall *gg;
770	tree callee = gimple_call_fndecl (gs: call);
771	tree lhs = gimple_call_lhs (gs: call);
772	tree old_size = gimple_call_arg (gs: call, index: 0);
773	tree ptr_type = lhs ? TREE_TYPE (lhs) : ptr_type_node;
774	tree partial_size = NULL_TREE;
775	unsigned int align
776	= DECL_FUNCTION_CODE (decl: callee) == BUILT_IN_ALLOCA
777	? 0 : tree_to_uhwi (gimple_call_arg (gs: call, index: 1));
778
779	bool throws = false;
780	edge e = NULL;
781	if (stmt_can_throw_internal (cfun, call))
782	{
783	if (!lhs)
784	return;
785	throws = true;
786	e = find_fallthru_edge (edges: gsi_bb (i: *iter)->succs);
787	}
788
789	if (hwasan_sanitize_allocas_p ())
790	{
791	gimple_seq stmts = NULL;
792	location_t loc = gimple_location (g: gsi_stmt (i: *iter));
793	/*
794	HWASAN needs a different expansion.
795
796	addr = __builtin_alloca (size, align);
797
798	should be replaced by
799
800	new_size = size rounded up to HWASAN_TAG_GRANULE_SIZE byte alignment;
801	untagged_addr = __builtin_alloca (new_size, align);
802	tag = __hwasan_choose_alloca_tag ();
803	addr = ifn_HWASAN_SET_TAG (untagged_addr, tag);
804	__hwasan_tag_memory (untagged_addr, tag, new_size);
805	*/
806	/* Ensure alignment at least HWASAN_TAG_GRANULE_SIZE bytes so we start on
807	a tag granule. */
808	align = align > HWASAN_TAG_GRANULE_SIZE ? align : HWASAN_TAG_GRANULE_SIZE;
809
810	tree old_size = gimple_call_arg (gs: call, index: 0);
811	tree new_size = gimple_build_round_up (seq: &stmts, loc, size_type_node,
812	old_size,
813	HWASAN_TAG_GRANULE_SIZE);
814
815	/* Make the alloca call */
816	tree untagged_addr
817	= gimple_build (seq: &stmts, loc,
818	fn: as_combined_fn (fn: BUILT_IN_ALLOCA_WITH_ALIGN), type: ptr_type,
819	args: new_size, args: build_int_cst (size_type_node, align));
820
821	/* Choose the tag.
822	Here we use an internal function so we can choose the tag at expand
823	time. We need the decision to be made after stack variables have been
824	assigned their tag (i.e. once the hwasan_frame_tag_offset variable has
825	been set to one after the last stack variables tag). */
826	tree tag = gimple_build (seq: &stmts, loc, fn: CFN_HWASAN_CHOOSE_TAG,
827	unsigned_char_type_node);
828
829	/* Add tag to pointer. */
830	tree addr
831	= gimple_build (seq: &stmts, loc, fn: CFN_HWASAN_SET_TAG, type: ptr_type,
832	args: untagged_addr, args: tag);
833
834	/* Tag shadow memory.
835	NOTE: require using `untagged_addr` here for libhwasan API. */
836	gimple_build (seq: &stmts, loc, fn: as_combined_fn (fn: BUILT_IN_HWASAN_TAG_MEM),
837	void_type_node, args: untagged_addr, args: tag, args: new_size);
838
839	/* Insert the built up code sequence into the original instruction stream
840	the iterator points to. */
841	gsi_insert_seq_before (iter, stmts, GSI_SAME_STMT);
842
843	/* Finally, replace old alloca ptr with NEW_ALLOCA. */
844	replace_call_with_value (iter, addr);
845	return;
846	}
847
848	tree last_alloca = get_last_alloca_addr ();
849	const HOST_WIDE_INT redzone_mask = ASAN_RED_ZONE_SIZE - 1;
850
851	/* If ALIGN > ASAN_RED_ZONE_SIZE, we embed left redzone into first ALIGN
852	bytes of allocated space. Otherwise, align alloca to ASAN_RED_ZONE_SIZE
853	manually. */
854	align = MAX (align, ASAN_RED_ZONE_SIZE * BITS_PER_UNIT);
855
856	tree alloca_rz_mask = build_int_cst (size_type_node, redzone_mask);
857	tree redzone_size = build_int_cst (size_type_node, ASAN_RED_ZONE_SIZE);
858
859	/* Extract lower bits from old_size. */
860	wide_int size_nonzero_bits = get_nonzero_bits (old_size);
861	wide_int rz_mask
862	= wi::uhwi (val: redzone_mask, precision: wi::get_precision (x: size_nonzero_bits));
863	wide_int old_size_lower_bits = wi::bit_and (x: size_nonzero_bits, y: rz_mask);
864
865	/* If alloca size is aligned to ASAN_RED_ZONE_SIZE, we don't need partial
866	redzone. Otherwise, compute its size here. */
867	if (wi::ne_p (x: old_size_lower_bits, y: 0))
868	{
869	/* misalign = size & (ASAN_RED_ZONE_SIZE - 1)
870	partial_size = ASAN_RED_ZONE_SIZE - misalign. */
871	g = gimple_build_assign (make_ssa_name (size_type_node, NULL),
872	BIT_AND_EXPR, old_size, alloca_rz_mask);
873	gsi_insert_before (iter, g, GSI_SAME_STMT);
874	tree misalign = gimple_assign_lhs (gs: g);
875	g = gimple_build_assign (make_ssa_name (size_type_node, NULL), MINUS_EXPR,
876	redzone_size, misalign);
877	gsi_insert_before (iter, g, GSI_SAME_STMT);
878	partial_size = gimple_assign_lhs (gs: g);
879	}
880
881	/* additional_size = align + ASAN_RED_ZONE_SIZE. */
882	tree additional_size = build_int_cst (size_type_node, align / BITS_PER_UNIT
883	+ ASAN_RED_ZONE_SIZE);
884	/* If alloca has partial redzone, include it to additional_size too. */
885	if (partial_size)
886	{
887	/* additional_size += partial_size. */
888	g = gimple_build_assign (make_ssa_name (size_type_node), PLUS_EXPR,
889	partial_size, additional_size);
890	gsi_insert_before (iter, g, GSI_SAME_STMT);
891	additional_size = gimple_assign_lhs (gs: g);
892	}
893
894	/* new_size = old_size + additional_size. */
895	g = gimple_build_assign (make_ssa_name (size_type_node), PLUS_EXPR, old_size,
896	additional_size);
897	gsi_insert_before (iter, g, GSI_SAME_STMT);
898	tree new_size = gimple_assign_lhs (gs: g);
899
900	/* Build new __builtin_alloca call:
901	new_alloca_with_rz = __builtin_alloca (new_size, align). */
902	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ALLOCA_WITH_ALIGN);
903	gg = gimple_build_call (fn, 2, new_size,
904	build_int_cst (size_type_node, align));
905	tree new_alloca_with_rz = make_ssa_name (var: ptr_type, stmt: gg);
906	gimple_call_set_lhs (gs: gg, lhs: new_alloca_with_rz);
907	if (throws)
908	{
909	gimple_call_set_lhs (gs: call, NULL);
910	gsi_replace (iter, gg, true);
911	}
912	else
913	gsi_insert_before (iter, gg, GSI_SAME_STMT);
914
915	/* new_alloca = new_alloca_with_rz + align. */
916	g = gimple_build_assign (make_ssa_name (var: ptr_type), POINTER_PLUS_EXPR,
917	new_alloca_with_rz,
918	build_int_cst (size_type_node,
919	align / BITS_PER_UNIT));
920	gimple_stmt_iterator gsi = gsi_none ();
921	if (throws)
922	{
923	gsi_insert_on_edge_immediate (e, g);
924	gsi = gsi_for_stmt (g);
925	}
926	else
927	gsi_insert_before (iter, g, GSI_SAME_STMT);
928	tree new_alloca = gimple_assign_lhs (gs: g);
929
930	/* Poison newly created alloca redzones:
931	__asan_alloca_poison (new_alloca, old_size). */
932	fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_ALLOCA_POISON);
933	gg = gimple_build_call (fn, 2, new_alloca, old_size);
934	if (throws)
935	gsi_insert_after (&gsi, gg, GSI_NEW_STMT);
936	else
937	gsi_insert_before (iter, gg, GSI_SAME_STMT);
938
939	/* Save new_alloca_with_rz value into last_alloca to use it during
940	allocas unpoisoning. */
941	g = gimple_build_assign (last_alloca, new_alloca_with_rz);
942	if (throws)
943	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
944	else
945	gsi_insert_before (iter, g, GSI_SAME_STMT);
946
947	/* Finally, replace old alloca ptr with NEW_ALLOCA. */
948	if (throws)
949	{
950	g = gimple_build_assign (lhs, new_alloca);
951	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
952	}
953	else
954	replace_call_with_value (iter, new_alloca);
955	}
956
957	/* Return the memory references contained in a gimple statement
958	representing a builtin call that has to do with memory access. */
959
960	static bool
961	get_mem_refs_of_builtin_call (gcall *call,
962	asan_mem_ref *src0,
963	tree *src0_len,
964	bool *src0_is_store,
965	asan_mem_ref *src1,
966	tree *src1_len,
967	bool *src1_is_store,
968	asan_mem_ref *dst,
969	tree *dst_len,
970	bool *dst_is_store,
971	bool *dest_is_deref,
972	bool *intercepted_p,
973	gimple_stmt_iterator *iter = NULL)
974	{
975	gcc_checking_assert (gimple_call_builtin_p (call, BUILT_IN_NORMAL));
976
977	tree callee = gimple_call_fndecl (gs: call);
978	tree source0 = NULL_TREE, source1 = NULL_TREE,
979	dest = NULL_TREE, len = NULL_TREE;
980	bool is_store = true, got_reference_p = false;
981	HOST_WIDE_INT access_size = 1;
982
983	*intercepted_p = asan_intercepted_p (fcode: (DECL_FUNCTION_CODE (decl: callee)));
984
985	switch (DECL_FUNCTION_CODE (decl: callee))
986	{
987	/* (s, s, n) style memops. */
988	case BUILT_IN_BCMP:
989	case BUILT_IN_MEMCMP:
990	source0 = gimple_call_arg (gs: call, index: 0);
991	source1 = gimple_call_arg (gs: call, index: 1);
992	len = gimple_call_arg (gs: call, index: 2);
993	break;
994
995	/* (src, dest, n) style memops. */
996	case BUILT_IN_BCOPY:
997	source0 = gimple_call_arg (gs: call, index: 0);
998	dest = gimple_call_arg (gs: call, index: 1);
999	len = gimple_call_arg (gs: call, index: 2);
1000	break;
1001
1002	/* (dest, src, n) style memops. */
1003	case BUILT_IN_MEMCPY:
1004	case BUILT_IN_MEMCPY_CHK:
1005	case BUILT_IN_MEMMOVE:
1006	case BUILT_IN_MEMMOVE_CHK:
1007	case BUILT_IN_MEMPCPY:
1008	case BUILT_IN_MEMPCPY_CHK:
1009	dest = gimple_call_arg (gs: call, index: 0);
1010	source0 = gimple_call_arg (gs: call, index: 1);
1011	len = gimple_call_arg (gs: call, index: 2);
1012	break;
1013
1014	/* (dest, n) style memops. */
1015	case BUILT_IN_BZERO:
1016	dest = gimple_call_arg (gs: call, index: 0);
1017	len = gimple_call_arg (gs: call, index: 1);
1018	break;
1019
1020	/* (dest, x, n) style memops*/
1021	case BUILT_IN_MEMSET:
1022	case BUILT_IN_MEMSET_CHK:
1023	dest = gimple_call_arg (gs: call, index: 0);
1024	len = gimple_call_arg (gs: call, index: 2);
1025	break;
1026
1027	case BUILT_IN_STRLEN:
1028	/* Special case strlen here since its length is taken from its return
1029	value.
1030
1031	The approach taken by the sanitizers is to check a memory access
1032	before it's taken. For ASAN strlen is intercepted by libasan, so no
1033	check is inserted by the compiler.
1034
1035	This function still returns `true` and provides a length to the rest
1036	of the ASAN pass in order to record what areas have been checked,
1037	avoiding superfluous checks later on.
1038
1039	HWASAN does not intercept any of these internal functions.
1040	This means that checks for memory accesses must be inserted by the
1041	compiler.
1042	strlen is a special case, because we can tell the length from the
1043	return of the function, but that is not known until after the function
1044	has returned.
1045
1046	Hence we can't check the memory access before it happens.
1047	We could check the memory access after it has already happened, but
1048	for now we choose to just ignore `strlen` calls.
1049	This decision was simply made because that means the special case is
1050	limited to this one case of this one function. */
1051	if (hwasan_sanitize_p ())
1052	return false;
1053	source0 = gimple_call_arg (gs: call, index: 0);
1054	len = gimple_call_lhs (gs: call);
1055	break;
1056
1057	case BUILT_IN_STACK_RESTORE:
1058	handle_builtin_stack_restore (call, iter);
1059	break;
1060
1061	CASE_BUILT_IN_ALLOCA:
1062	handle_builtin_alloca (call, iter);
1063	break;
1064	/* And now the __atomic* and __sync builtins.
1065	These are handled differently from the classical memory
1066	access builtins above. */
1067
1068	case BUILT_IN_ATOMIC_LOAD_1:
1069	is_store = false;
1070	/* FALLTHRU */
1071	case BUILT_IN_SYNC_FETCH_AND_ADD_1:
1072	case BUILT_IN_SYNC_FETCH_AND_SUB_1:
1073	case BUILT_IN_SYNC_FETCH_AND_OR_1:
1074	case BUILT_IN_SYNC_FETCH_AND_AND_1:
1075	case BUILT_IN_SYNC_FETCH_AND_XOR_1:
1076	case BUILT_IN_SYNC_FETCH_AND_NAND_1:
1077	case BUILT_IN_SYNC_ADD_AND_FETCH_1:
1078	case BUILT_IN_SYNC_SUB_AND_FETCH_1:
1079	case BUILT_IN_SYNC_OR_AND_FETCH_1:
1080	case BUILT_IN_SYNC_AND_AND_FETCH_1:
1081	case BUILT_IN_SYNC_XOR_AND_FETCH_1:
1082	case BUILT_IN_SYNC_NAND_AND_FETCH_1:
1083	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1:
1084	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1:
1085	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_1:
1086	case BUILT_IN_SYNC_LOCK_RELEASE_1:
1087	case BUILT_IN_ATOMIC_EXCHANGE_1:
1088	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
1089	case BUILT_IN_ATOMIC_STORE_1:
1090	case BUILT_IN_ATOMIC_ADD_FETCH_1:
1091	case BUILT_IN_ATOMIC_SUB_FETCH_1:
1092	case BUILT_IN_ATOMIC_AND_FETCH_1:
1093	case BUILT_IN_ATOMIC_NAND_FETCH_1:
1094	case BUILT_IN_ATOMIC_XOR_FETCH_1:
1095	case BUILT_IN_ATOMIC_OR_FETCH_1:
1096	case BUILT_IN_ATOMIC_FETCH_ADD_1:
1097	case BUILT_IN_ATOMIC_FETCH_SUB_1:
1098	case BUILT_IN_ATOMIC_FETCH_AND_1:
1099	case BUILT_IN_ATOMIC_FETCH_NAND_1:
1100	case BUILT_IN_ATOMIC_FETCH_XOR_1:
1101	case BUILT_IN_ATOMIC_FETCH_OR_1:
1102	access_size = 1;
1103	goto do_atomic;
1104
1105	case BUILT_IN_ATOMIC_LOAD_2:
1106	is_store = false;
1107	/* FALLTHRU */
1108	case BUILT_IN_SYNC_FETCH_AND_ADD_2:
1109	case BUILT_IN_SYNC_FETCH_AND_SUB_2:
1110	case BUILT_IN_SYNC_FETCH_AND_OR_2:
1111	case BUILT_IN_SYNC_FETCH_AND_AND_2:
1112	case BUILT_IN_SYNC_FETCH_AND_XOR_2:
1113	case BUILT_IN_SYNC_FETCH_AND_NAND_2:
1114	case BUILT_IN_SYNC_ADD_AND_FETCH_2:
1115	case BUILT_IN_SYNC_SUB_AND_FETCH_2:
1116	case BUILT_IN_SYNC_OR_AND_FETCH_2:
1117	case BUILT_IN_SYNC_AND_AND_FETCH_2:
1118	case BUILT_IN_SYNC_XOR_AND_FETCH_2:
1119	case BUILT_IN_SYNC_NAND_AND_FETCH_2:
1120	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_2:
1121	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_2:
1122	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_2:
1123	case BUILT_IN_SYNC_LOCK_RELEASE_2:
1124	case BUILT_IN_ATOMIC_EXCHANGE_2:
1125	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
1126	case BUILT_IN_ATOMIC_STORE_2:
1127	case BUILT_IN_ATOMIC_ADD_FETCH_2:
1128	case BUILT_IN_ATOMIC_SUB_FETCH_2:
1129	case BUILT_IN_ATOMIC_AND_FETCH_2:
1130	case BUILT_IN_ATOMIC_NAND_FETCH_2:
1131	case BUILT_IN_ATOMIC_XOR_FETCH_2:
1132	case BUILT_IN_ATOMIC_OR_FETCH_2:
1133	case BUILT_IN_ATOMIC_FETCH_ADD_2:
1134	case BUILT_IN_ATOMIC_FETCH_SUB_2:
1135	case BUILT_IN_ATOMIC_FETCH_AND_2:
1136	case BUILT_IN_ATOMIC_FETCH_NAND_2:
1137	case BUILT_IN_ATOMIC_FETCH_XOR_2:
1138	case BUILT_IN_ATOMIC_FETCH_OR_2:
1139	access_size = 2;
1140	goto do_atomic;
1141
1142	case BUILT_IN_ATOMIC_LOAD_4:
1143	is_store = false;
1144	/* FALLTHRU */
1145	case BUILT_IN_SYNC_FETCH_AND_ADD_4:
1146	case BUILT_IN_SYNC_FETCH_AND_SUB_4:
1147	case BUILT_IN_SYNC_FETCH_AND_OR_4:
1148	case BUILT_IN_SYNC_FETCH_AND_AND_4:
1149	case BUILT_IN_SYNC_FETCH_AND_XOR_4:
1150	case BUILT_IN_SYNC_FETCH_AND_NAND_4:
1151	case BUILT_IN_SYNC_ADD_AND_FETCH_4:
1152	case BUILT_IN_SYNC_SUB_AND_FETCH_4:
1153	case BUILT_IN_SYNC_OR_AND_FETCH_4:
1154	case BUILT_IN_SYNC_AND_AND_FETCH_4:
1155	case BUILT_IN_SYNC_XOR_AND_FETCH_4:
1156	case BUILT_IN_SYNC_NAND_AND_FETCH_4:
1157	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_4:
1158	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_4:
1159	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_4:
1160	case BUILT_IN_SYNC_LOCK_RELEASE_4:
1161	case BUILT_IN_ATOMIC_EXCHANGE_4:
1162	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
1163	case BUILT_IN_ATOMIC_STORE_4:
1164	case BUILT_IN_ATOMIC_ADD_FETCH_4:
1165	case BUILT_IN_ATOMIC_SUB_FETCH_4:
1166	case BUILT_IN_ATOMIC_AND_FETCH_4:
1167	case BUILT_IN_ATOMIC_NAND_FETCH_4:
1168	case BUILT_IN_ATOMIC_XOR_FETCH_4:
1169	case BUILT_IN_ATOMIC_OR_FETCH_4:
1170	case BUILT_IN_ATOMIC_FETCH_ADD_4:
1171	case BUILT_IN_ATOMIC_FETCH_SUB_4:
1172	case BUILT_IN_ATOMIC_FETCH_AND_4:
1173	case BUILT_IN_ATOMIC_FETCH_NAND_4:
1174	case BUILT_IN_ATOMIC_FETCH_XOR_4:
1175	case BUILT_IN_ATOMIC_FETCH_OR_4:
1176	access_size = 4;
1177	goto do_atomic;
1178
1179	case BUILT_IN_ATOMIC_LOAD_8:
1180	is_store = false;
1181	/* FALLTHRU */
1182	case BUILT_IN_SYNC_FETCH_AND_ADD_8:
1183	case BUILT_IN_SYNC_FETCH_AND_SUB_8:
1184	case BUILT_IN_SYNC_FETCH_AND_OR_8:
1185	case BUILT_IN_SYNC_FETCH_AND_AND_8:
1186	case BUILT_IN_SYNC_FETCH_AND_XOR_8:
1187	case BUILT_IN_SYNC_FETCH_AND_NAND_8:
1188	case BUILT_IN_SYNC_ADD_AND_FETCH_8:
1189	case BUILT_IN_SYNC_SUB_AND_FETCH_8:
1190	case BUILT_IN_SYNC_OR_AND_FETCH_8:
1191	case BUILT_IN_SYNC_AND_AND_FETCH_8:
1192	case BUILT_IN_SYNC_XOR_AND_FETCH_8:
1193	case BUILT_IN_SYNC_NAND_AND_FETCH_8:
1194	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_8:
1195	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_8:
1196	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_8:
1197	case BUILT_IN_SYNC_LOCK_RELEASE_8:
1198	case BUILT_IN_ATOMIC_EXCHANGE_8:
1199	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
1200	case BUILT_IN_ATOMIC_STORE_8:
1201	case BUILT_IN_ATOMIC_ADD_FETCH_8:
1202	case BUILT_IN_ATOMIC_SUB_FETCH_8:
1203	case BUILT_IN_ATOMIC_AND_FETCH_8:
1204	case BUILT_IN_ATOMIC_NAND_FETCH_8:
1205	case BUILT_IN_ATOMIC_XOR_FETCH_8:
1206	case BUILT_IN_ATOMIC_OR_FETCH_8:
1207	case BUILT_IN_ATOMIC_FETCH_ADD_8:
1208	case BUILT_IN_ATOMIC_FETCH_SUB_8:
1209	case BUILT_IN_ATOMIC_FETCH_AND_8:
1210	case BUILT_IN_ATOMIC_FETCH_NAND_8:
1211	case BUILT_IN_ATOMIC_FETCH_XOR_8:
1212	case BUILT_IN_ATOMIC_FETCH_OR_8:
1213	access_size = 8;
1214	goto do_atomic;
1215
1216	case BUILT_IN_ATOMIC_LOAD_16:
1217	is_store = false;
1218	/* FALLTHRU */
1219	case BUILT_IN_SYNC_FETCH_AND_ADD_16:
1220	case BUILT_IN_SYNC_FETCH_AND_SUB_16:
1221	case BUILT_IN_SYNC_FETCH_AND_OR_16:
1222	case BUILT_IN_SYNC_FETCH_AND_AND_16:
1223	case BUILT_IN_SYNC_FETCH_AND_XOR_16:
1224	case BUILT_IN_SYNC_FETCH_AND_NAND_16:
1225	case BUILT_IN_SYNC_ADD_AND_FETCH_16:
1226	case BUILT_IN_SYNC_SUB_AND_FETCH_16:
1227	case BUILT_IN_SYNC_OR_AND_FETCH_16:
1228	case BUILT_IN_SYNC_AND_AND_FETCH_16:
1229	case BUILT_IN_SYNC_XOR_AND_FETCH_16:
1230	case BUILT_IN_SYNC_NAND_AND_FETCH_16:
1231	case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_16:
1232	case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_16:
1233	case BUILT_IN_SYNC_LOCK_TEST_AND_SET_16:
1234	case BUILT_IN_SYNC_LOCK_RELEASE_16:
1235	case BUILT_IN_ATOMIC_EXCHANGE_16:
1236	case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
1237	case BUILT_IN_ATOMIC_STORE_16:
1238	case BUILT_IN_ATOMIC_ADD_FETCH_16:
1239	case BUILT_IN_ATOMIC_SUB_FETCH_16:
1240	case BUILT_IN_ATOMIC_AND_FETCH_16:
1241	case BUILT_IN_ATOMIC_NAND_FETCH_16:
1242	case BUILT_IN_ATOMIC_XOR_FETCH_16:
1243	case BUILT_IN_ATOMIC_OR_FETCH_16:
1244	case BUILT_IN_ATOMIC_FETCH_ADD_16:
1245	case BUILT_IN_ATOMIC_FETCH_SUB_16:
1246	case BUILT_IN_ATOMIC_FETCH_AND_16:
1247	case BUILT_IN_ATOMIC_FETCH_NAND_16:
1248	case BUILT_IN_ATOMIC_FETCH_XOR_16:
1249	case BUILT_IN_ATOMIC_FETCH_OR_16:
1250	access_size = 16;
1251	/* FALLTHRU */
1252	do_atomic:
1253	{
1254	dest = gimple_call_arg (gs: call, index: 0);
1255	/* DEST represents the address of a memory location.
1256	instrument_derefs wants the memory location, so lets
1257	dereference the address DEST before handing it to
1258	instrument_derefs. */
1259	tree type = build_nonstandard_integer_type (access_size
1260	* BITS_PER_UNIT, 1);
1261	dest = build2 (MEM_REF, type, dest,
1262	build_int_cst (build_pointer_type (char_type_node), 0));
1263	break;
1264	}
1265
1266	default:
1267	/* The other builtins memory access are not instrumented in this
1268	function because they either don't have any length parameter,
1269	or their length parameter is just a limit. */
1270	break;
1271	}
1272
1273	if (len != NULL_TREE)
1274	{
1275	if (source0 != NULL_TREE)
1276	{
1277	src0->start = source0;
1278	src0->access_size = access_size;
1279	*src0_len = len;
1280	*src0_is_store = false;
1281	}
1282
1283	if (source1 != NULL_TREE)
1284	{
1285	src1->start = source1;
1286	src1->access_size = access_size;
1287	*src1_len = len;
1288	*src1_is_store = false;
1289	}
1290
1291	if (dest != NULL_TREE)
1292	{
1293	dst->start = dest;
1294	dst->access_size = access_size;
1295	*dst_len = len;
1296	*dst_is_store = true;
1297	}
1298
1299	got_reference_p = true;
1300	}
1301	else if (dest)
1302	{
1303	dst->start = dest;
1304	dst->access_size = access_size;
1305	*dst_len = NULL_TREE;
1306	*dst_is_store = is_store;
1307	*dest_is_deref = true;
1308	got_reference_p = true;
1309	}
1310
1311	return got_reference_p;
1312	}
1313
1314	/* Return true iff a given gimple statement has been instrumented.
1315	Note that the statement is "defined" by the memory references it
1316	contains. */
1317
1318	static bool
1319	has_stmt_been_instrumented_p (gimple *stmt)
1320	{
1321	if (gimple_assign_single_p (gs: stmt))
1322	{
1323	bool r_is_store;
1324	asan_mem_ref r;
1325	asan_mem_ref_init (ref: &r, NULL, access_size: 1);
1326
1327	if (get_mem_ref_of_assignment (assignment: as_a <gassign *> (p: stmt), ref: &r,
1328	ref_is_store: &r_is_store))
1329	{
1330	if (!has_mem_ref_been_instrumented (ref: &r))
1331	return false;
1332	if (r_is_store && gimple_assign_load_p (stmt))
1333	{
1334	asan_mem_ref src;
1335	asan_mem_ref_init (ref: &src, NULL, access_size: 1);
1336	src.start = gimple_assign_rhs1 (gs: stmt);
1337	src.access_size = int_size_in_bytes (TREE_TYPE (src.start));
1338	if (!has_mem_ref_been_instrumented (ref: &src))
1339	return false;
1340	}
1341	return true;
1342	}
1343	}
1344	else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
1345	{
1346	asan_mem_ref src0, src1, dest;
1347	asan_mem_ref_init (ref: &src0, NULL, access_size: 1);
1348	asan_mem_ref_init (ref: &src1, NULL, access_size: 1);
1349	asan_mem_ref_init (ref: &dest, NULL, access_size: 1);
1350
1351	tree src0_len = NULL_TREE, src1_len = NULL_TREE, dest_len = NULL_TREE;
1352	bool src0_is_store = false, src1_is_store = false,
1353	dest_is_store = false, dest_is_deref = false, intercepted_p = true;
1354	if (get_mem_refs_of_builtin_call (call: as_a <gcall *> (p: stmt),
1355	src0: &src0, src0_len: &src0_len, src0_is_store: &src0_is_store,
1356	src1: &src1, src1_len: &src1_len, src1_is_store: &src1_is_store,
1357	dst: &dest, dst_len: &dest_len, dst_is_store: &dest_is_store,
1358	dest_is_deref: &dest_is_deref, intercepted_p: &intercepted_p))
1359	{
1360	if (src0.start != NULL_TREE
1361	&& !has_mem_ref_been_instrumented (ref: &src0, len: src0_len))
1362	return false;
1363
1364	if (src1.start != NULL_TREE
1365	&& !has_mem_ref_been_instrumented (ref: &src1, len: src1_len))
1366	return false;
1367
1368	if (dest.start != NULL_TREE
1369	&& !has_mem_ref_been_instrumented (ref: &dest, len: dest_len))
1370	return false;
1371
1372	return true;
1373	}
1374	}
1375	else if (is_gimple_call (gs: stmt) && gimple_store_p (gs: stmt))
1376	{
1377	asan_mem_ref r;
1378	asan_mem_ref_init (ref: &r, NULL, access_size: 1);
1379
1380	r.start = gimple_call_lhs (gs: stmt);
1381	r.access_size = int_size_in_bytes (TREE_TYPE (r.start));
1382	return has_mem_ref_been_instrumented (ref: &r);
1383	}
1384
1385	return false;
1386	}
1387
1388	/* Insert a memory reference into the hash table. */
1389
1390	static void
1391	update_mem_ref_hash_table (tree ref, HOST_WIDE_INT access_size)
1392	{
1393	hash_table<asan_mem_ref_hasher> *ht = get_mem_ref_hash_table ();
1394
1395	asan_mem_ref r;
1396	asan_mem_ref_init (ref: &r, start: ref, access_size);
1397
1398	asan_mem_ref **slot = ht->find_slot (value: &r, insert: INSERT);
1399	if (*slot == NULL || (*slot)->access_size < access_size)
1400	*slot = asan_mem_ref_new (start: ref, access_size);
1401	}
1402
1403	/* Initialize shadow_ptr_types array. */
1404
1405	static void
1406	asan_init_shadow_ptr_types (void)
1407	{
1408	asan_shadow_set = new_alias_set ();
1409	tree types[3] = { signed_char_type_node, short_integer_type_node,
1410	integer_type_node };
1411
1412	for (unsigned i = 0; i < 3; i++)
1413	{
1414	shadow_ptr_types[i] = build_distinct_type_copy (types[i]);
1415	TYPE_ALIAS_SET (shadow_ptr_types[i]) = asan_shadow_set;
1416	shadow_ptr_types[i] = build_pointer_type (shadow_ptr_types[i]);
1417	}
1418
1419	initialize_sanitizer_builtins ();
1420	}
1421
1422	/* Create ADDR_EXPR of STRING_CST with the PP pretty printer text. */
1423
1424	static tree
1425	asan_pp_string (pretty_printer *pp)
1426	{
1427	const char *buf = pp_formatted_text (pp);
1428	size_t len = strlen (s: buf);
1429	tree ret = build_string (len + 1, buf);
1430	TREE_TYPE (ret)
1431	= build_array_type (TREE_TYPE (shadow_ptr_types[0]),
1432	build_index_type (size_int (len)));
1433	TREE_READONLY (ret) = 1;
1434	TREE_STATIC (ret) = 1;
1435	return build1 (ADDR_EXPR, shadow_ptr_types[0], ret);
1436	}
1437
1438	/* Clear shadow memory at SHADOW_MEM, LEN bytes. Can't call a library call here
1439	though. */
1440
1441	static void
1442	asan_clear_shadow (rtx shadow_mem, HOST_WIDE_INT len)
1443	{
1444	rtx_insn *insn, *insns, *jump;
1445	rtx_code_label *top_label;
1446	rtx end, addr, tmp;
1447
1448	gcc_assert ((len & 3) == 0);
1449	start_sequence ();
1450	clear_storage (shadow_mem, GEN_INT (len), BLOCK_OP_NORMAL);
1451	insns = get_insns ();
1452	end_sequence ();
1453	for (insn = insns; insn; insn = NEXT_INSN (insn))
1454	if (CALL_P (insn))
1455	break;
1456	if (insn == NULL_RTX)
1457	{
1458	emit_insn (insns);
1459	return;
1460	}
1461
1462	top_label = gen_label_rtx ();
1463	addr = copy_to_mode_reg (Pmode, XEXP (shadow_mem, 0));
1464	shadow_mem = adjust_automodify_address (shadow_mem, SImode, addr, 0);
1465	end = force_reg (Pmode, plus_constant (Pmode, addr, len));
1466	emit_label (top_label);
1467
1468	emit_move_insn (shadow_mem, const0_rtx);
1469	tmp = expand_simple_binop (Pmode, PLUS, addr, gen_int_mode (4, Pmode), addr,
1470	true, OPTAB_LIB_WIDEN);
1471	if (tmp != addr)
1472	emit_move_insn (addr, tmp);
1473	emit_cmp_and_jump_insns (addr, end, LT, NULL_RTX, Pmode, true, top_label);
1474	jump = get_last_insn ();
1475	gcc_assert (JUMP_P (jump));
1476	add_reg_br_prob_note (jump,
1477	profile_probability::guessed_always ()
1478	.apply_scale (num: 80, den: 100));
1479	}
1480
1481	void
1482	asan_function_start (void)
1483	{
1484	section *fnsec = function_section (current_function_decl);
1485	switch_to_section (fnsec);
1486	ASM_OUTPUT_DEBUG_LABEL (asm_out_file, "LASANPC",
1487	current_function_funcdef_no);
1488	}
1489
1490	/* Return number of shadow bytes that are occupied by a local variable
1491	of SIZE bytes. */
1492
1493	static unsigned HOST_WIDE_INT
1494	shadow_mem_size (unsigned HOST_WIDE_INT size)
1495	{
1496	/* It must be possible to align stack variables to granularity
1497	of shadow memory. */
1498	gcc_assert (BITS_PER_UNIT
1499	* ASAN_SHADOW_GRANULARITY <= MAX_SUPPORTED_STACK_ALIGNMENT);
1500
1501	return ROUND_UP (size, ASAN_SHADOW_GRANULARITY) / ASAN_SHADOW_GRANULARITY;
1502	}
1503
1504	/* Always emit 4 bytes at a time. */
1505	#define RZ_BUFFER_SIZE 4
1506
1507	/* ASAN redzone buffer container that handles emission of shadow bytes. */
1508	class asan_redzone_buffer
1509	{
1510	public:
1511	/* Constructor. */
1512	asan_redzone_buffer (rtx shadow_mem, HOST_WIDE_INT prev_offset):
1513	m_shadow_mem (shadow_mem), m_prev_offset (prev_offset),
1514	m_original_offset (prev_offset), m_shadow_bytes (RZ_BUFFER_SIZE)
1515	{}
1516
1517	/* Emit VALUE shadow byte at a given OFFSET. */
1518	void emit_redzone_byte (HOST_WIDE_INT offset, unsigned char value);
1519
1520	/* Emit RTX emission of the content of the buffer. */
1521	void flush_redzone_payload (void);
1522
1523	private:
1524	/* Flush if the content of the buffer is full
1525	(equal to RZ_BUFFER_SIZE). */
1526	void flush_if_full (void);
1527
1528	/* Memory where we last emitted a redzone payload. */
1529	rtx m_shadow_mem;
1530
1531	/* Relative offset where we last emitted a redzone payload. */
1532	HOST_WIDE_INT m_prev_offset;
1533
1534	/* Relative original offset. Used for checking only. */
1535	HOST_WIDE_INT m_original_offset;
1536
1537	public:
1538	/* Buffer with redzone payload. */
1539	auto_vec<unsigned char> m_shadow_bytes;
1540	};
1541
1542	/* Emit VALUE shadow byte at a given OFFSET. */
1543
1544	void
1545	asan_redzone_buffer::emit_redzone_byte (HOST_WIDE_INT offset,
1546	unsigned char value)
1547	{
1548	gcc_assert ((offset & (ASAN_SHADOW_GRANULARITY - 1)) == 0);
1549	gcc_assert (offset >= m_prev_offset);
1550
1551	HOST_WIDE_INT off
1552	= m_prev_offset + ASAN_SHADOW_GRANULARITY * m_shadow_bytes.length ();
1553	if (off == offset)
1554	/* Consecutive shadow memory byte. */;
1555	else if (offset < m_prev_offset + (HOST_WIDE_INT) (ASAN_SHADOW_GRANULARITY
1556	* RZ_BUFFER_SIZE)
1557	&& !m_shadow_bytes.is_empty ())
1558	{
1559	/* Shadow memory byte with a small gap. */
1560	for (; off < offset; off += ASAN_SHADOW_GRANULARITY)
1561	m_shadow_bytes.safe_push (obj: 0);
1562	}
1563	else
1564	{
1565	if (!m_shadow_bytes.is_empty ())
1566	flush_redzone_payload ();
1567
1568	/* Maybe start earlier in order to use aligned store. */
1569	HOST_WIDE_INT align = (offset - m_prev_offset) % ASAN_RED_ZONE_SIZE;
1570	if (align)
1571	{
1572	offset -= align;
1573	for (unsigned i = 0; i < align / BITS_PER_UNIT; i++)
1574	m_shadow_bytes.safe_push (obj: 0);
1575	}
1576
1577	/* Adjust m_prev_offset and m_shadow_mem. */
1578	HOST_WIDE_INT diff = offset - m_prev_offset;
1579	m_shadow_mem = adjust_address (m_shadow_mem, VOIDmode,
1580	diff >> ASAN_SHADOW_SHIFT);
1581	m_prev_offset = offset;
1582	}
1583	m_shadow_bytes.safe_push (obj: value);
1584	flush_if_full ();
1585	}
1586
1587	/* Emit RTX emission of the content of the buffer. */
1588
1589	void
1590	asan_redzone_buffer::flush_redzone_payload (void)
1591	{
1592	gcc_assert (WORDS_BIG_ENDIAN == BYTES_BIG_ENDIAN);
1593
1594	if (m_shadow_bytes.is_empty ())
1595	return;
1596
1597	/* Be sure we always emit to an aligned address. */
1598	gcc_assert (((m_prev_offset - m_original_offset)
1599	& (ASAN_RED_ZONE_SIZE - 1)) == 0);
1600
1601	/* Fill it to RZ_BUFFER_SIZE bytes with zeros if needed. */
1602	unsigned l = m_shadow_bytes.length ();
1603	for (unsigned i = 0; i <= RZ_BUFFER_SIZE - l; i++)
1604	m_shadow_bytes.safe_push (obj: 0);
1605
1606	if (dump_file && (dump_flags & TDF_DETAILS))
1607	fprintf (stream: dump_file,
1608	format: "Flushing rzbuffer at offset %" PRId64 " with: ", m_prev_offset);
1609
1610	unsigned HOST_WIDE_INT val = 0;
1611	for (unsigned i = 0; i < RZ_BUFFER_SIZE; i++)
1612	{
1613	unsigned char v
1614	= m_shadow_bytes[BYTES_BIG_ENDIAN ? RZ_BUFFER_SIZE - i - 1 : i];
1615	val |= (unsigned HOST_WIDE_INT)v << (BITS_PER_UNIT * i);
1616	if (dump_file && (dump_flags & TDF_DETAILS))
1617	fprintf (stream: dump_file, format: "%02x ", v);
1618	}
1619
1620	if (dump_file && (dump_flags & TDF_DETAILS))
1621	fprintf (stream: dump_file, format: "\n");
1622
1623	rtx c = gen_int_mode (val, SImode);
1624	m_shadow_mem = adjust_address (m_shadow_mem, SImode, 0);
1625	emit_move_insn (m_shadow_mem, c);
1626	m_shadow_bytes.truncate (size: 0);
1627	}
1628
1629	/* Flush if the content of the buffer is full
1630	(equal to RZ_BUFFER_SIZE). */
1631
1632	void
1633	asan_redzone_buffer::flush_if_full (void)
1634	{
1635	if (m_shadow_bytes.length () == RZ_BUFFER_SIZE)
1636	flush_redzone_payload ();
1637	}
1638
1639
1640	/* HWAddressSanitizer (hwasan) is a probabilistic method for detecting
1641	out-of-bounds and use-after-free bugs.
1642	Read more:
1643	http://code.google.com/p/address-sanitizer/
1644
1645	Similar to AddressSanitizer (asan) it consists of two parts: the
1646	instrumentation module in this file, and a run-time library.
1647
1648	The instrumentation module adds a run-time check before every memory insn in
1649	the same manner as asan (see the block comment for AddressSanitizer above).
1650	Currently, hwasan only adds out-of-line instrumentation, where each check is
1651	implemented as a function call to the run-time library. Hence a check for a
1652	load of N bytes from address X would be implemented with a function call to
1653	__hwasan_loadN(X), and checking a store of N bytes from address X would be
1654	implemented with a function call to __hwasan_storeN(X).
1655
1656	The main difference between hwasan and asan is in the information stored to
1657	help this checking. Both sanitizers use a shadow memory area which stores
1658	data recording the state of main memory at a corresponding address.
1659
1660	For hwasan, each 16 byte granule in main memory has a corresponding 1 byte
1661	in shadow memory. This shadow address can be calculated with equation:
1662	(addr >> log_2(HWASAN_TAG_GRANULE_SIZE))
1663	+ __hwasan_shadow_memory_dynamic_address;
1664	The conversion between real and shadow memory for asan is given in the block
1665	comment at the top of this file.
1666	The description of how this shadow memory is laid out for asan is in the
1667	block comment at the top of this file, here we describe how this shadow
1668	memory is used for hwasan.
1669
1670	For hwasan, each variable is assigned a byte-sized 'tag'. The extent of
1671	the shadow memory for that variable is filled with the assigned tag, and
1672	every pointer referencing that variable has its top byte set to the same
1673	tag. The run-time library redefines malloc so that every allocation returns
1674	a tagged pointer and tags the corresponding shadow memory with the same tag.
1675
1676	On each pointer dereference the tag found in the pointer is compared to the
1677	tag found in the shadow memory corresponding to the accessed memory address.
1678	If these tags are found to differ then this memory access is judged to be
1679	invalid and a report is generated.
1680
1681	This method of bug detection is not perfect -- it can not catch every bad
1682	access -- but catches them probabilistically instead. There is always the
1683	possibility that an invalid memory access will happen to access memory
1684	tagged with the same tag as the pointer that this access used.
1685	The chances of this are approx. 0.4% for any two uncorrelated objects.
1686
1687	Random tag generation can mitigate this problem by decreasing the
1688	probability that an invalid access will be missed in the same manner over
1689	multiple runs. i.e. if two objects are tagged the same in one run of the
1690	binary they are unlikely to be tagged the same in the next run.
1691	Both heap and stack allocated objects have random tags by default.
1692
1693	[16 byte granule implications]
1694	Since the shadow memory only has a resolution on real memory of 16 bytes,
1695	invalid accesses that are within the same 16 byte granule as a valid
1696	address will not be caught.
1697
1698	There is a "short-granule" feature in the runtime library which does catch
1699	such accesses, but this feature is not implemented for stack objects (since
1700	stack objects are allocated and tagged by compiler instrumentation, and
1701	this feature has not yet been implemented in GCC instrumentation).
1702
1703	Another outcome of this 16 byte resolution is that each tagged object must
1704	be 16 byte aligned. If two objects were to share any 16 byte granule in
1705	memory, then they both would have to be given the same tag, and invalid
1706	accesses to one using a pointer to the other would be undetectable.
1707
1708	[Compiler instrumentation]
1709	Compiler instrumentation ensures that two adjacent buffers on the stack are
1710	given different tags, this means an access to one buffer using a pointer
1711	generated from the other (e.g. through buffer overrun) will have mismatched
1712	tags and be caught by hwasan.
1713
1714	We don't randomly tag every object on the stack, since that would require
1715	keeping many registers to record each tag. Instead we randomly generate a
1716	tag for each function frame, and each new stack object uses a tag offset
1717	from that frame tag.
1718	i.e. each object is tagged as RFT + offset, where RFT is the "random frame
1719	tag" generated for this frame.
1720	This means that randomisation does not peturb the difference between tags
1721	on tagged stack objects within a frame, but this is mitigated by the fact
1722	that objects with the same tag within a frame are very far apart
1723	(approx. 2^HWASAN_TAG_SIZE objects apart).
1724
1725	As a demonstration, using the same example program as in the asan block
1726	comment above:
1727
1728	int
1729	foo ()
1730	{
1731	char a[24] = {0};
1732	int b[2] = {0};
1733
1734	a[5] = 1;
1735	b[1] = 2;
1736
1737	return a[5] + b[1];
1738	}
1739
1740	On AArch64 the stack will be ordered as follows for the above function:
1741
1742	Slot 1/ [24 bytes for variable 'a']
1743	Slot 2/ [8 bytes padding for alignment]
1744	Slot 3/ [8 bytes for variable 'b']
1745	Slot 4/ [8 bytes padding for alignment]
1746
1747	(The padding is there to ensure 16 byte alignment as described in the 16
1748	byte granule implications).
1749
1750	While the shadow memory will be ordered as follows:
1751
1752	- 2 bytes (representing 32 bytes in real memory) tagged with RFT + 1.
1753	- 1 byte (representing 16 bytes in real memory) tagged with RFT + 2.
1754
1755	And any pointer to "a" will have the tag RFT + 1, and any pointer to "b"
1756	will have the tag RFT + 2.
1757
1758	[Top Byte Ignore requirements]
1759	Hwasan requires the ability to store an 8 bit tag in every pointer. There
1760	is no instrumentation done to remove this tag from pointers before
1761	dereferencing, which means the hardware must ignore this tag during memory
1762	accesses.
1763
1764	Architectures where this feature is available should indicate this using
1765	the TARGET_MEMTAG_CAN_TAG_ADDRESSES hook.
1766
1767	[Stack requires cleanup on unwinding]
1768	During normal operation of a hwasan sanitized program more space in the
1769	shadow memory becomes tagged as the stack grows. As the stack shrinks this
1770	shadow memory space must become untagged. If it is not untagged then when
1771	the stack grows again (during other function calls later on in the program)
1772	objects on the stack that are usually not tagged (e.g. parameters passed on
1773	the stack) can be placed in memory whose shadow space is tagged with
1774	something else, and accesses can cause false positive reports.
1775
1776	Hence we place untagging code on every epilogue of functions which tag some
1777	stack objects.
1778
1779	Moreover, the run-time library intercepts longjmp & setjmp to untag when
1780	the stack is unwound this way.
1781
1782	C++ exceptions are not yet handled, which means this sanitizer can not
1783	handle C++ code that throws exceptions -- it will give false positives
1784	after an exception has been thrown. The implementation that the hwasan
1785	library has for handling these relies on the frame pointer being after any
1786	local variables. This is not generally the case for GCC. */
1787
1788
1789	/* Returns whether we are tagging pointers and checking those tags on memory
1790	access. */
1791	bool
1792	hwasan_sanitize_p ()
1793	{
1794	return sanitize_flags_p (flag: SANITIZE_HWADDRESS);
1795	}
1796
1797	/* Are we tagging the stack? */
1798	bool
1799	hwasan_sanitize_stack_p ()
1800	{
1801	return (hwasan_sanitize_p () && param_hwasan_instrument_stack);
1802	}
1803
1804	/* Are we tagging alloca objects? */
1805	bool
1806	hwasan_sanitize_allocas_p (void)
1807	{
1808	return (hwasan_sanitize_stack_p () && param_hwasan_instrument_allocas);
1809	}
1810
1811	/* Should we instrument reads? */
1812	bool
1813	hwasan_instrument_reads (void)
1814	{
1815	return (hwasan_sanitize_p () && param_hwasan_instrument_reads);
1816	}
1817
1818	/* Should we instrument writes? */
1819	bool
1820	hwasan_instrument_writes (void)
1821	{
1822	return (hwasan_sanitize_p () && param_hwasan_instrument_writes);
1823	}
1824
1825	/* Should we instrument builtin calls? */
1826	bool
1827	hwasan_memintrin (void)
1828	{
1829	return (hwasan_sanitize_p () && param_hwasan_instrument_mem_intrinsics);
1830	}
1831
1832	/* Insert code to protect stack vars. The prologue sequence should be emitted
1833	directly, epilogue sequence returned. BASE is the register holding the
1834	stack base, against which OFFSETS array offsets are relative to, OFFSETS
1835	array contains pairs of offsets in reverse order, always the end offset
1836	of some gap that needs protection followed by starting offset,
1837	and DECLS is an array of representative decls for each var partition.
1838	LENGTH is the length of the OFFSETS array, DECLS array is LENGTH / 2 - 1
1839	elements long (OFFSETS include gap before the first variable as well
1840	as gaps after each stack variable). PBASE is, if non-NULL, some pseudo
1841	register which stack vars DECL_RTLs are based on. Either BASE should be
1842	assigned to PBASE, when not doing use after return protection, or
1843	corresponding address based on __asan_stack_malloc* return value. */
1844
1845	rtx_insn *
1846	asan_emit_stack_protection (rtx base, rtx pbase, unsigned int alignb,
1847	HOST_WIDE_INT *offsets, tree *decls, int length)
1848	{
1849	rtx shadow_base, shadow_mem, ret, mem, orig_base;
1850	rtx_code_label *lab;
1851	rtx_insn *insns;
1852	char buf[32];
1853	HOST_WIDE_INT base_offset = offsets[length - 1];
1854	HOST_WIDE_INT base_align_bias = 0, offset, prev_offset;
1855	HOST_WIDE_INT asan_frame_size = offsets[0] - base_offset;
1856	HOST_WIDE_INT last_offset, last_size, last_size_aligned;
1857	int l;
1858	unsigned char cur_shadow_byte = ASAN_STACK_MAGIC_LEFT;
1859	tree str_cst, decl, id;
1860	int use_after_return_class = -1;
1861
1862	/* Don't emit anything when doing error recovery, the assertions
1863	might fail e.g. if a function had a frame offset overflow. */
1864	if (seen_error ())
1865	return NULL;
1866
1867	if (shadow_ptr_types[0] == NULL_TREE)
1868	asan_init_shadow_ptr_types ();
1869
1870	expanded_location cfun_xloc
1871	= expand_location (DECL_SOURCE_LOCATION (current_function_decl));
1872
1873	/* First of all, prepare the description string. */
1874	pretty_printer asan_pp;
1875
1876	pp_decimal_int (&asan_pp, length / 2 - 1);
1877	pp_space (&asan_pp);
1878	for (l = length - 2; l; l -= 2)
1879	{
1880	tree decl = decls[l / 2 - 1];
1881	pp_wide_integer (pp: &asan_pp, i: offsets[l] - base_offset);
1882	pp_space (&asan_pp);
1883	pp_wide_integer (pp: &asan_pp, i: offsets[l - 1] - offsets[l]);
1884	pp_space (&asan_pp);
1885
1886	expanded_location xloc
1887	= expand_location (DECL_SOURCE_LOCATION (decl));
1888	char location[32];
1889
1890	if (xloc.file == cfun_xloc.file)
1891	sprintf (s: location, format: ":%d", xloc.line);
1892	else
1893	location[0] = '\0';
1894
1895	if (DECL_P (decl) && DECL_NAME (decl))
1896	{
1897	unsigned idlen
1898	= IDENTIFIER_LENGTH (DECL_NAME (decl)) + strlen (s: location);
1899	pp_decimal_int (&asan_pp, idlen);
1900	pp_space (&asan_pp);
1901	pp_tree_identifier (&asan_pp, DECL_NAME (decl));
1902	pp_string (&asan_pp, location);
1903	}
1904	else
1905	pp_string (&asan_pp, "9 <unknown>");
1906
1907	if (l > 2)
1908	pp_space (&asan_pp);
1909	}
1910	str_cst = asan_pp_string (pp: &asan_pp);
1911
1912	/* Emit the prologue sequence. */
1913	if (asan_frame_size > 32 && asan_frame_size <= 65536 && pbase
1914	&& param_asan_use_after_return)
1915	{
1916	use_after_return_class = floor_log2 (x: asan_frame_size - 1) - 5;
1917	/* __asan_stack_malloc_N guarantees alignment
1918	N < 6 ? (64 << N) : 4096 bytes. */
1919	if (alignb > (use_after_return_class < 6
1920	? (64U << use_after_return_class) : 4096U))
1921	use_after_return_class = -1;
1922	else if (alignb > ASAN_RED_ZONE_SIZE && (asan_frame_size & (alignb - 1)))
1923	base_align_bias = ((asan_frame_size + alignb - 1)
1924	& ~(alignb - HOST_WIDE_INT_1)) - asan_frame_size;
1925	}
1926
1927	/* Align base if target is STRICT_ALIGNMENT. */
1928	if (STRICT_ALIGNMENT)
1929	{
1930	const HOST_WIDE_INT align
1931	= (GET_MODE_ALIGNMENT (SImode) / BITS_PER_UNIT) << ASAN_SHADOW_SHIFT;
1932	base = expand_binop (Pmode, and_optab, base, gen_int_mode (-align, Pmode),
1933	NULL_RTX, 1, OPTAB_DIRECT);
1934	}
1935
1936	if (use_after_return_class == -1 && pbase)
1937	emit_move_insn (pbase, base);
1938
1939	base = expand_binop (Pmode, add_optab, base,
1940	gen_int_mode (base_offset - base_align_bias, Pmode),
1941	NULL_RTX, 1, OPTAB_DIRECT);
1942	orig_base = NULL_RTX;
1943	if (use_after_return_class != -1)
1944	{
1945	if (asan_detect_stack_use_after_return == NULL_TREE)
1946	{
1947	id = get_identifier ("__asan_option_detect_stack_use_after_return");
1948	decl = build_decl (BUILTINS_LOCATION, VAR_DECL, id,
1949	integer_type_node);
1950	SET_DECL_ASSEMBLER_NAME (decl, id);
1951	TREE_ADDRESSABLE (decl) = 1;
1952	DECL_ARTIFICIAL (decl) = 1;
1953	DECL_IGNORED_P (decl) = 1;
1954	DECL_EXTERNAL (decl) = 1;
1955	TREE_STATIC (decl) = 1;
1956	TREE_PUBLIC (decl) = 1;
1957	TREE_USED (decl) = 1;
1958	asan_detect_stack_use_after_return = decl;
1959	}
1960	orig_base = gen_reg_rtx (Pmode);
1961	emit_move_insn (orig_base, base);
1962	ret = expand_normal (exp: asan_detect_stack_use_after_return);
1963	lab = gen_label_rtx ();
1964	emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
1965	VOIDmode, 0, lab,
1966	prob: profile_probability::very_likely ());
1967	snprintf (s: buf, maxlen: sizeof buf, format: "__asan_stack_malloc_%d",
1968	use_after_return_class);
1969	ret = init_one_libfunc (buf);
1970	ret = emit_library_call_value (fun: ret, NULL_RTX, fn_type: LCT_NORMAL, outmode: ptr_mode,
1971	GEN_INT (asan_frame_size
1972	+ base_align_bias),
1973	TYPE_MODE (pointer_sized_int_node));
1974	/* __asan_stack_malloc_[n] returns a pointer to fake stack if succeeded
1975	and NULL otherwise. Check RET value is NULL here and jump over the
1976	BASE reassignment in this case. Otherwise, reassign BASE to RET. */
1977	emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
1978	VOIDmode, 0, lab,
1979	prob: profile_probability:: very_unlikely ());
1980	ret = convert_memory_address (Pmode, ret);
1981	emit_move_insn (base, ret);
1982	emit_label (lab);
1983	emit_move_insn (pbase, expand_binop (Pmode, add_optab, base,
1984	gen_int_mode (base_align_bias
1985	- base_offset, Pmode),
1986	NULL_RTX, 1, OPTAB_DIRECT));
1987	}
1988	mem = gen_rtx_MEM (ptr_mode, base);
1989	mem = adjust_address (mem, VOIDmode, base_align_bias);
1990	emit_move_insn (mem, gen_int_mode (ASAN_STACK_FRAME_MAGIC, ptr_mode));
1991	mem = adjust_address (mem, VOIDmode, GET_MODE_SIZE (ptr_mode));
1992	emit_move_insn (mem, expand_normal (exp: str_cst));
1993	mem = adjust_address (mem, VOIDmode, GET_MODE_SIZE (ptr_mode));
1994	ASM_GENERATE_INTERNAL_LABEL (buf, "LASANPC", current_function_funcdef_no);
1995	id = get_identifier (buf);
1996	decl = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
1997	VAR_DECL, id, char_type_node);
1998	SET_DECL_ASSEMBLER_NAME (decl, id);
1999	TREE_ADDRESSABLE (decl) = 1;
2000	TREE_READONLY (decl) = 1;
2001	DECL_ARTIFICIAL (decl) = 1;
2002	DECL_IGNORED_P (decl) = 1;
2003	TREE_STATIC (decl) = 1;
2004	TREE_PUBLIC (decl) = 0;
2005	TREE_USED (decl) = 1;
2006	DECL_INITIAL (decl) = decl;
2007	TREE_ASM_WRITTEN (decl) = 1;
2008	TREE_ASM_WRITTEN (id) = 1;
2009	emit_move_insn (mem, expand_normal (build_fold_addr_expr (decl)));
2010	shadow_base = expand_binop (Pmode, lshr_optab, base,
2011	gen_int_shift_amount (Pmode, ASAN_SHADOW_SHIFT),
2012	NULL_RTX, 1, OPTAB_DIRECT);
2013	shadow_base
2014	= plus_constant (Pmode, shadow_base,
2015	asan_shadow_offset ()
2016	+ (base_align_bias >> ASAN_SHADOW_SHIFT));
2017	gcc_assert (asan_shadow_set != -1
2018	&& (ASAN_RED_ZONE_SIZE >> ASAN_SHADOW_SHIFT) == 4);
2019	shadow_mem = gen_rtx_MEM (SImode, shadow_base);
2020	set_mem_alias_set (shadow_mem, asan_shadow_set);
2021	if (STRICT_ALIGNMENT)
2022	set_mem_align (shadow_mem, (GET_MODE_ALIGNMENT (SImode)));
2023	prev_offset = base_offset;
2024
2025	asan_redzone_buffer rz_buffer (shadow_mem, prev_offset);
2026	for (l = length; l; l -= 2)
2027	{
2028	if (l == 2)
2029	cur_shadow_byte = ASAN_STACK_MAGIC_RIGHT;
2030	offset = offsets[l - 1];
2031
2032	bool extra_byte = (offset - base_offset) & (ASAN_SHADOW_GRANULARITY - 1);
2033	/* If a red-zone is not aligned to ASAN_SHADOW_GRANULARITY then
2034	the previous stack variable has size % ASAN_SHADOW_GRANULARITY != 0.
2035	In that case we have to emit one extra byte that will describe
2036	how many bytes (our of ASAN_SHADOW_GRANULARITY) can be accessed. */
2037	if (extra_byte)
2038	{
2039	HOST_WIDE_INT aoff
2040	= base_offset + ((offset - base_offset)
2041	& ~(ASAN_SHADOW_GRANULARITY - HOST_WIDE_INT_1));
2042	rz_buffer.emit_redzone_byte (offset: aoff, value: offset - aoff);
2043	offset = aoff + ASAN_SHADOW_GRANULARITY;
2044	}
2045
2046	/* Calculate size of red zone payload. */
2047	while (offset < offsets[l - 2])
2048	{
2049	rz_buffer.emit_redzone_byte (offset, value: cur_shadow_byte);
2050	offset += ASAN_SHADOW_GRANULARITY;
2051	}
2052
2053	cur_shadow_byte = ASAN_STACK_MAGIC_MIDDLE;
2054	}
2055
2056	/* As the automatic variables are aligned to
2057	ASAN_RED_ZONE_SIZE / ASAN_SHADOW_GRANULARITY, the buffer should be
2058	flushed here. */
2059	gcc_assert (rz_buffer.m_shadow_bytes.is_empty ());
2060
2061	do_pending_stack_adjust ();
2062
2063	/* Construct epilogue sequence. */
2064	start_sequence ();
2065
2066	lab = NULL;
2067	if (use_after_return_class != -1)
2068	{
2069	rtx_code_label *lab2 = gen_label_rtx ();
2070	char c = (char) ASAN_STACK_MAGIC_USE_AFTER_RET;
2071	emit_cmp_and_jump_insns (orig_base, base, EQ, NULL_RTX,
2072	VOIDmode, 0, lab2,
2073	prob: profile_probability::very_likely ());
2074	shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
2075	set_mem_alias_set (shadow_mem, asan_shadow_set);
2076	mem = gen_rtx_MEM (ptr_mode, base);
2077	mem = adjust_address (mem, VOIDmode, base_align_bias);
2078	emit_move_insn (mem, gen_int_mode (ASAN_STACK_RETIRED_MAGIC, ptr_mode));
2079	unsigned HOST_WIDE_INT sz = asan_frame_size >> ASAN_SHADOW_SHIFT;
2080	if (use_after_return_class < 5
2081	&& can_store_by_pieces (sz, builtin_memset_read_str, &c,
2082	BITS_PER_UNIT, true))
2083	{
2084	/* Emit:
2085	memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize);
2086	**SavedFlagPtr(FakeStack, class_id) = 0
2087	*/
2088	store_by_pieces (shadow_mem, sz, builtin_memset_read_str, &c,
2089	BITS_PER_UNIT, true, RETURN_BEGIN);
2090
2091	unsigned HOST_WIDE_INT offset
2092	= (1 << (use_after_return_class + 6));
2093	offset -= GET_MODE_SIZE (mode: ptr_mode);
2094	mem = gen_rtx_MEM (ptr_mode, base);
2095	mem = adjust_address (mem, ptr_mode, offset);
2096	rtx addr = gen_reg_rtx (ptr_mode);
2097	emit_move_insn (addr, mem);
2098	addr = convert_memory_address (Pmode, addr);
2099	mem = gen_rtx_MEM (QImode, addr);
2100	emit_move_insn (mem, const0_rtx);
2101	}
2102	else if (use_after_return_class >= 5
2103	|| !set_storage_via_setmem (shadow_mem,
2104	GEN_INT (sz),
2105	gen_int_mode (c, QImode),
2106	BITS_PER_UNIT, BITS_PER_UNIT,
2107	-1, sz, sz, sz))
2108	{
2109	snprintf (s: buf, maxlen: sizeof buf, format: "__asan_stack_free_%d",
2110	use_after_return_class);
2111	ret = init_one_libfunc (buf);
2112	rtx addr = convert_memory_address (ptr_mode, base);
2113	rtx orig_addr = convert_memory_address (ptr_mode, orig_base);
2114	emit_library_call (fun: ret, fn_type: LCT_NORMAL, outmode: ptr_mode, arg1: addr, arg1_mode: ptr_mode,
2115	GEN_INT (asan_frame_size + base_align_bias),
2116	TYPE_MODE (pointer_sized_int_node),
2117	arg3: orig_addr, arg3_mode: ptr_mode);
2118	}
2119	lab = gen_label_rtx ();
2120	emit_jump (lab);
2121	emit_label (lab2);
2122	}
2123
2124	shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
2125	set_mem_alias_set (shadow_mem, asan_shadow_set);
2126
2127	if (STRICT_ALIGNMENT)
2128	set_mem_align (shadow_mem, (GET_MODE_ALIGNMENT (SImode)));
2129
2130	prev_offset = base_offset;
2131	last_offset = base_offset;
2132	last_size = 0;
2133	last_size_aligned = 0;
2134	for (l = length; l; l -= 2)
2135	{
2136	offset = base_offset + ((offsets[l - 1] - base_offset)
2137	& ~(ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1));
2138	if (last_offset + last_size_aligned < offset)
2139	{
2140	shadow_mem = adjust_address (shadow_mem, VOIDmode,
2141	(last_offset - prev_offset)
2142	>> ASAN_SHADOW_SHIFT);
2143	prev_offset = last_offset;
2144	asan_clear_shadow (shadow_mem, len: last_size_aligned >> ASAN_SHADOW_SHIFT);
2145	last_offset = offset;
2146	last_size = 0;
2147	}
2148	else
2149	last_size = offset - last_offset;
2150	last_size += base_offset + ((offsets[l - 2] - base_offset)
2151	& ~(ASAN_MIN_RED_ZONE_SIZE - HOST_WIDE_INT_1))
2152	- offset;
2153
2154	/* Unpoison shadow memory that corresponds to a variable that is
2155	is subject of use-after-return sanitization. */
2156	if (l > 2)
2157	{
2158	decl = decls[l / 2 - 2];
2159	if (asan_handled_variables != NULL
2160	&& asan_handled_variables->contains (k: decl))
2161	{
2162	HOST_WIDE_INT size = offsets[l - 3] - offsets[l - 2];
2163	if (dump_file && (dump_flags & TDF_DETAILS))
2164	{
2165	const char *n = (DECL_NAME (decl)
2166	? IDENTIFIER_POINTER (DECL_NAME (decl))
2167	: "<unknown>");
2168	fprintf (stream: dump_file, format: "Unpoisoning shadow stack for variable: "
2169	"%s (%" PRId64 " B)\n", n, size);
2170	}
2171
2172	last_size += size & ~(ASAN_MIN_RED_ZONE_SIZE - HOST_WIDE_INT_1);
2173	}
2174	}
2175	last_size_aligned
2176	= ((last_size + (ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1))
2177	& ~(ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1));
2178	}
2179	if (last_size_aligned)
2180	{
2181	shadow_mem = adjust_address (shadow_mem, VOIDmode,
2182	(last_offset - prev_offset)
2183	>> ASAN_SHADOW_SHIFT);
2184	asan_clear_shadow (shadow_mem, len: last_size_aligned >> ASAN_SHADOW_SHIFT);
2185	}
2186
2187	/* Clean-up set with instrumented stack variables. */
2188	delete asan_handled_variables;
2189	asan_handled_variables = NULL;
2190	delete asan_used_labels;
2191	asan_used_labels = NULL;
2192
2193	do_pending_stack_adjust ();
2194	if (lab)
2195	emit_label (lab);
2196
2197	insns = get_insns ();
2198	end_sequence ();
2199	return insns;
2200	}
2201
2202	/* Emit __asan_allocas_unpoison (top, bot) call. The BASE parameter corresponds
2203	to BOT argument, for TOP virtual_stack_dynamic_rtx is used. NEW_SEQUENCE
2204	indicates whether we're emitting new instructions sequence or not. */
2205
2206	rtx_insn *
2207	asan_emit_allocas_unpoison (rtx top, rtx bot, rtx_insn *before)
2208	{
2209	if (before)
2210	push_to_sequence (before);
2211	else
2212	start_sequence ();
2213	rtx ret = init_one_libfunc ("__asan_allocas_unpoison");
2214	top = convert_memory_address (ptr_mode, top);
2215	bot = convert_memory_address (ptr_mode, bot);
2216	emit_library_call (fun: ret, fn_type: LCT_NORMAL, outmode: ptr_mode,
2217	arg1: top, arg1_mode: ptr_mode, arg2: bot, arg2_mode: ptr_mode);
2218
2219	do_pending_stack_adjust ();
2220	rtx_insn *insns = get_insns ();
2221	end_sequence ();
2222	return insns;
2223	}
2224
2225	/* Return true if DECL, a global var, might be overridden and needs
2226	therefore a local alias. */
2227
2228	static bool
2229	asan_needs_local_alias (tree decl)
2230	{
2231	return DECL_WEAK (decl) || !targetm.binds_local_p (decl);
2232	}
2233
2234	/* Return true if DECL, a global var, is an artificial ODR indicator symbol
2235	therefore doesn't need protection. */
2236
2237	static bool
2238	is_odr_indicator (tree decl)
2239	{
2240	return (DECL_ARTIFICIAL (decl)
2241	&& lookup_attribute (attr_name: "asan odr indicator", DECL_ATTRIBUTES (decl)));
2242	}
2243
2244	/* Return true if DECL is a VAR_DECL that should be protected
2245	by Address Sanitizer, by appending a red zone with protected
2246	shadow memory after it and aligning it to at least
2247	ASAN_RED_ZONE_SIZE bytes. */
2248
2249	bool
2250	asan_protect_global (tree decl, bool ignore_decl_rtl_set_p)
2251	{
2252	if (!param_asan_globals)
2253	return false;
2254
2255	rtx rtl, symbol;
2256
2257	if (TREE_CODE (decl) == STRING_CST)
2258	{
2259	/* Instrument all STRING_CSTs except those created
2260	by asan_pp_string here. */
2261	if (shadow_ptr_types[0] != NULL_TREE
2262	&& TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
2263	&& TREE_TYPE (TREE_TYPE (decl)) == TREE_TYPE (shadow_ptr_types[0]))
2264	return false;
2265	return true;
2266	}
2267	if (!VAR_P (decl)
2268	/* TLS vars aren't statically protectable. */
2269	|| DECL_THREAD_LOCAL_P (decl)
2270	/* Externs will be protected elsewhere. */
2271	|| DECL_EXTERNAL (decl)
2272	/* PR sanitizer/81697: For architectures that use section anchors first
2273	call to asan_protect_global may occur before DECL_RTL (decl) is set.
2274	We should ignore DECL_RTL_SET_P then, because otherwise the first call
2275	to asan_protect_global will return FALSE and the following calls on the
2276	same decl after setting DECL_RTL (decl) will return TRUE and we'll end
2277	up with inconsistency at runtime. */
2278	|| (!DECL_RTL_SET_P (decl) && !ignore_decl_rtl_set_p)
2279	/* Comdat vars pose an ABI problem, we can't know if
2280	the var that is selected by the linker will have
2281	padding or not. */
2282	|| DECL_ONE_ONLY (decl)
2283	/* Similarly for common vars. People can use -fno-common.
2284	Note: Linux kernel is built with -fno-common, so we do instrument
2285	globals there even if it is C. */
2286	|| (DECL_COMMON (decl) && TREE_PUBLIC (decl))
2287	/* Don't protect if using user section, often vars placed
2288	into user section from multiple TUs are then assumed
2289	to be an array of such vars, putting padding in there
2290	breaks this assumption. */
2291	|| (DECL_SECTION_NAME (decl) != NULL
2292	&& !symtab_node::get (decl)->implicit_section
2293	&& !section_sanitized_p (DECL_SECTION_NAME (decl)))
2294	|| DECL_SIZE (decl) == 0
2295	|| ASAN_RED_ZONE_SIZE * BITS_PER_UNIT > MAX_OFILE_ALIGNMENT
2296	|| TREE_CODE (DECL_SIZE_UNIT (decl)) != INTEGER_CST
2297	|| !valid_constant_size_p (DECL_SIZE_UNIT (decl))
2298	|| DECL_ALIGN_UNIT (decl) > 2 * ASAN_RED_ZONE_SIZE
2299	|| TREE_TYPE (decl) == ubsan_get_source_location_type ()
2300	|| is_odr_indicator (decl))
2301	return false;
2302
2303	if (!ignore_decl_rtl_set_p || DECL_RTL_SET_P (decl))
2304	{
2305
2306	rtl = DECL_RTL (decl);
2307	if (!MEM_P (rtl) || GET_CODE (XEXP (rtl, 0)) != SYMBOL_REF)
2308	return false;
2309	symbol = XEXP (rtl, 0);
2310
2311	if (CONSTANT_POOL_ADDRESS_P (symbol)
2312	|| TREE_CONSTANT_POOL_ADDRESS_P (symbol))
2313	return false;
2314	}
2315
2316	if (lookup_attribute (attr_name: "weakref", DECL_ATTRIBUTES (decl)))
2317	return false;
2318
2319	if (!TARGET_SUPPORTS_ALIASES && asan_needs_local_alias (decl))
2320	return false;
2321
2322	return true;
2323	}
2324
2325	/* Construct a function tree for __asan_report_{load,store}{1,2,4,8,16,_n}.
2326	IS_STORE is either 1 (for a store) or 0 (for a load). */
2327
2328	static tree
2329	report_error_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
2330	int *nargs)
2331	{
2332	gcc_assert (!hwasan_sanitize_p ());
2333
2334	static enum built_in_function report[2][2][6]
2335	= { { { BUILT_IN_ASAN_REPORT_LOAD1, BUILT_IN_ASAN_REPORT_LOAD2,
2336	BUILT_IN_ASAN_REPORT_LOAD4, BUILT_IN_ASAN_REPORT_LOAD8,
2337	BUILT_IN_ASAN_REPORT_LOAD16, BUILT_IN_ASAN_REPORT_LOAD_N },
2338	{ BUILT_IN_ASAN_REPORT_STORE1, BUILT_IN_ASAN_REPORT_STORE2,
2339	BUILT_IN_ASAN_REPORT_STORE4, BUILT_IN_ASAN_REPORT_STORE8,
2340	BUILT_IN_ASAN_REPORT_STORE16, BUILT_IN_ASAN_REPORT_STORE_N } },
2341	{ { BUILT_IN_ASAN_REPORT_LOAD1_NOABORT,
2342	BUILT_IN_ASAN_REPORT_LOAD2_NOABORT,
2343	BUILT_IN_ASAN_REPORT_LOAD4_NOABORT,
2344	BUILT_IN_ASAN_REPORT_LOAD8_NOABORT,
2345	BUILT_IN_ASAN_REPORT_LOAD16_NOABORT,
2346	BUILT_IN_ASAN_REPORT_LOAD_N_NOABORT },
2347	{ BUILT_IN_ASAN_REPORT_STORE1_NOABORT,
2348	BUILT_IN_ASAN_REPORT_STORE2_NOABORT,
2349	BUILT_IN_ASAN_REPORT_STORE4_NOABORT,
2350	BUILT_IN_ASAN_REPORT_STORE8_NOABORT,
2351	BUILT_IN_ASAN_REPORT_STORE16_NOABORT,
2352	BUILT_IN_ASAN_REPORT_STORE_N_NOABORT } } };
2353	if (size_in_bytes == -1)
2354	{
2355	*nargs = 2;
2356	return builtin_decl_implicit (fncode: report[recover_p][is_store][5]);
2357	}
2358	*nargs = 1;
2359	int size_log2 = exact_log2 (x: size_in_bytes);
2360	return builtin_decl_implicit (fncode: report[recover_p][is_store][size_log2]);
2361	}
2362
2363	/* Construct a function tree for __asan_{load,store}{1,2,4,8,16,_n}.
2364	IS_STORE is either 1 (for a store) or 0 (for a load). */
2365
2366	static tree
2367	check_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
2368	int *nargs)
2369	{
2370	static enum built_in_function check[2][2][6]
2371	= { { { BUILT_IN_ASAN_LOAD1, BUILT_IN_ASAN_LOAD2,
2372	BUILT_IN_ASAN_LOAD4, BUILT_IN_ASAN_LOAD8,
2373	BUILT_IN_ASAN_LOAD16, BUILT_IN_ASAN_LOADN },
2374	{ BUILT_IN_ASAN_STORE1, BUILT_IN_ASAN_STORE2,
2375	BUILT_IN_ASAN_STORE4, BUILT_IN_ASAN_STORE8,
2376	BUILT_IN_ASAN_STORE16, BUILT_IN_ASAN_STOREN } },
2377	{ { BUILT_IN_ASAN_LOAD1_NOABORT,
2378	BUILT_IN_ASAN_LOAD2_NOABORT,
2379	BUILT_IN_ASAN_LOAD4_NOABORT,
2380	BUILT_IN_ASAN_LOAD8_NOABORT,
2381	BUILT_IN_ASAN_LOAD16_NOABORT,
2382	BUILT_IN_ASAN_LOADN_NOABORT },
2383	{ BUILT_IN_ASAN_STORE1_NOABORT,
2384	BUILT_IN_ASAN_STORE2_NOABORT,
2385	BUILT_IN_ASAN_STORE4_NOABORT,
2386	BUILT_IN_ASAN_STORE8_NOABORT,
2387	BUILT_IN_ASAN_STORE16_NOABORT,
2388	BUILT_IN_ASAN_STOREN_NOABORT } } };
2389	if (size_in_bytes == -1)
2390	{
2391	*nargs = 2;
2392	return builtin_decl_implicit (fncode: check[recover_p][is_store][5]);
2393	}
2394	*nargs = 1;
2395	int size_log2 = exact_log2 (x: size_in_bytes);
2396	return builtin_decl_implicit (fncode: check[recover_p][is_store][size_log2]);
2397	}
2398
2399	/* Split the current basic block and create a condition statement
2400	insertion point right before or after the statement pointed to by
2401	ITER. Return an iterator to the point at which the caller might
2402	safely insert the condition statement.
2403
2404	THEN_BLOCK must be set to the address of an uninitialized instance
2405	of basic_block. The function will then set *THEN_BLOCK to the
2406	'then block' of the condition statement to be inserted by the
2407	caller.
2408
2409	If CREATE_THEN_FALLTHRU_EDGE is false, no edge will be created from
2410	*THEN_BLOCK to *FALLTHROUGH_BLOCK.
2411
2412	Similarly, the function will set *FALLTRHOUGH_BLOCK to the 'else
2413	block' of the condition statement to be inserted by the caller.
2414
2415	Note that *FALLTHROUGH_BLOCK is a new block that contains the
2416	statements starting from *ITER, and *THEN_BLOCK is a new empty
2417	block.
2418
2419	*ITER is adjusted to point to always point to the first statement
2420	of the basic block * FALLTHROUGH_BLOCK. That statement is the
2421	same as what ITER was pointing to prior to calling this function,
2422	if BEFORE_P is true; otherwise, it is its following statement. */
2423
2424	gimple_stmt_iterator
2425	create_cond_insert_point (gimple_stmt_iterator *iter,
2426	bool before_p,
2427	bool then_more_likely_p,
2428	bool create_then_fallthru_edge,
2429	basic_block *then_block,
2430	basic_block *fallthrough_block)
2431	{
2432	gimple_stmt_iterator gsi = *iter;
2433
2434	if (!gsi_end_p (i: gsi) && before_p)
2435	gsi_prev (i: &gsi);
2436
2437	basic_block cur_bb = gsi_bb (i: *iter);
2438
2439	edge e = split_block (cur_bb, gsi_stmt (i: gsi));
2440
2441	/* Get a hold on the 'condition block', the 'then block' and the
2442	'else block'. */
2443	basic_block cond_bb = e->src;
2444	basic_block fallthru_bb = e->dest;
2445	basic_block then_bb = create_empty_bb (cond_bb);
2446	if (current_loops)
2447	{
2448	add_bb_to_loop (then_bb, cond_bb->loop_father);
2449	loops_state_set (flags: LOOPS_NEED_FIXUP);
2450	}
2451
2452	/* Set up the newly created 'then block'. */
2453	e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
2454	profile_probability fallthrough_probability
2455	= then_more_likely_p
2456	? profile_probability::very_unlikely ()
2457	: profile_probability::very_likely ();
2458	e->probability = fallthrough_probability.invert ();
2459	then_bb->count = e->count ();
2460	if (create_then_fallthru_edge)
2461	make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
2462
2463	/* Set up the fallthrough basic block. */
2464	e = find_edge (cond_bb, fallthru_bb);
2465	e->flags = EDGE_FALSE_VALUE;
2466	e->probability = fallthrough_probability;
2467
2468	/* Update dominance info for the newly created then_bb; note that
2469	fallthru_bb's dominance info has already been updated by
2470	split_bock. */
2471	if (dom_info_available_p (CDI_DOMINATORS))
2472	set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
2473
2474	*then_block = then_bb;
2475	*fallthrough_block = fallthru_bb;
2476	*iter = gsi_start_bb (bb: fallthru_bb);
2477
2478	return gsi_last_bb (bb: cond_bb);
2479	}
2480
2481	/* Insert an if condition followed by a 'then block' right before the
2482	statement pointed to by ITER. The fallthrough block -- which is the
2483	else block of the condition as well as the destination of the
2484	outcoming edge of the 'then block' -- starts with the statement
2485	pointed to by ITER.
2486
2487	COND is the condition of the if.
2488
2489	If THEN_MORE_LIKELY_P is true, the probability of the edge to the
2490	'then block' is higher than the probability of the edge to the
2491	fallthrough block.
2492
2493	Upon completion of the function, *THEN_BB is set to the newly
2494	inserted 'then block' and similarly, *FALLTHROUGH_BB is set to the
2495	fallthrough block.
2496
2497	*ITER is adjusted to still point to the same statement it was
2498	pointing to initially. */
2499
2500	static void
2501	insert_if_then_before_iter (gcond *cond,
2502	gimple_stmt_iterator *iter,
2503	bool then_more_likely_p,
2504	basic_block *then_bb,
2505	basic_block *fallthrough_bb)
2506	{
2507	gimple_stmt_iterator cond_insert_point =
2508	create_cond_insert_point (iter,
2509	/*before_p=*/true,
2510	then_more_likely_p,
2511	/*create_then_fallthru_edge=*/true,
2512	then_block: then_bb,
2513	fallthrough_block: fallthrough_bb);
2514	gsi_insert_after (&cond_insert_point, cond, GSI_NEW_STMT);
2515	}
2516
2517	/* Build (base_addr >> ASAN_SHADOW_SHIFT) + asan_shadow_offset ().
2518	If RETURN_ADDRESS is set to true, return memory location instread
2519	of a value in the shadow memory. */
2520
2521	static tree
2522	build_shadow_mem_access (gimple_stmt_iterator *gsi, location_t location,
2523	tree base_addr, tree shadow_ptr_type,
2524	bool return_address = false)
2525	{
2526	tree t, uintptr_type = TREE_TYPE (base_addr);
2527	tree shadow_type = TREE_TYPE (shadow_ptr_type);
2528	gimple *g;
2529
2530	t = build_int_cst (uintptr_type, ASAN_SHADOW_SHIFT);
2531	g = gimple_build_assign (make_ssa_name (var: uintptr_type), RSHIFT_EXPR,
2532	base_addr, t);
2533	gimple_set_location (g, location);
2534	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2535
2536	t = build_int_cst (uintptr_type, asan_shadow_offset ());
2537	g = gimple_build_assign (make_ssa_name (var: uintptr_type), PLUS_EXPR,
2538	gimple_assign_lhs (gs: g), t);
2539	gimple_set_location (g, location);
2540	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2541
2542	g = gimple_build_assign (make_ssa_name (var: shadow_ptr_type), NOP_EXPR,
2543	gimple_assign_lhs (gs: g));
2544	gimple_set_location (g, location);
2545	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2546
2547	if (!return_address)
2548	{
2549	t = build2 (MEM_REF, shadow_type, gimple_assign_lhs (gs: g),
2550	build_int_cst (shadow_ptr_type, 0));
2551	g = gimple_build_assign (make_ssa_name (var: shadow_type), MEM_REF, t);
2552	gimple_set_location (g, location);
2553	gsi_insert_after (gsi, g, GSI_NEW_STMT);
2554	}
2555
2556	return gimple_assign_lhs (gs: g);
2557	}
2558
2559	/* BASE can already be an SSA_NAME; in that case, do not create a
2560	new SSA_NAME for it. */
2561
2562	static tree
2563	maybe_create_ssa_name (location_t loc, tree base, gimple_stmt_iterator *iter,
2564	bool before_p)
2565	{
2566	STRIP_USELESS_TYPE_CONVERSION (base);
2567	if (TREE_CODE (base) == SSA_NAME)
2568	return base;
2569	gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (base)), base);
2570	gimple_set_location (g, location: loc);
2571	if (before_p)
2572	gsi_insert_before (iter, g, GSI_SAME_STMT);
2573	else
2574	gsi_insert_after (iter, g, GSI_NEW_STMT);
2575	return gimple_assign_lhs (gs: g);
2576	}
2577
2578	/* LEN can already have necessary size and precision;
2579	in that case, do not create a new variable. */
2580
2581	tree
2582	maybe_cast_to_ptrmode (location_t loc, tree len, gimple_stmt_iterator *iter,
2583	bool before_p)
2584	{
2585	if (ptrofftype_p (type: len))
2586	return len;
2587	gimple *g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
2588	NOP_EXPR, len);
2589	gimple_set_location (g, location: loc);
2590	if (before_p)
2591	gsi_insert_before (iter, g, GSI_SAME_STMT);
2592	else
2593	gsi_insert_after (iter, g, GSI_NEW_STMT);
2594	return gimple_assign_lhs (gs: g);
2595	}
2596
2597	/* Instrument the memory access instruction BASE. Insert new
2598	statements before or after ITER.
2599
2600	Note that the memory access represented by BASE can be either an
2601	SSA_NAME, or a non-SSA expression. LOCATION is the source code
2602	location. IS_STORE is TRUE for a store, FALSE for a load.
2603	BEFORE_P is TRUE for inserting the instrumentation code before
2604	ITER, FALSE for inserting it after ITER. IS_SCALAR_ACCESS is TRUE
2605	for a scalar memory access and FALSE for memory region access.
2606	NON_ZERO_P is TRUE if memory region is guaranteed to have non-zero
2607	length. ALIGN tells alignment of accessed memory object.
2608
2609	START_INSTRUMENTED and END_INSTRUMENTED are TRUE if start/end of
2610	memory region have already been instrumented.
2611
2612	If BEFORE_P is TRUE, *ITER is arranged to still point to the
2613	statement it was pointing to prior to calling this function,
2614	otherwise, it points to the statement logically following it. */
2615
2616	static void
2617	build_check_stmt (location_t loc, tree base, tree len,
2618	HOST_WIDE_INT size_in_bytes, gimple_stmt_iterator *iter,
2619	bool is_non_zero_len, bool before_p, bool is_store,
2620	bool is_scalar_access, unsigned int align = 0)
2621	{
2622	gimple_stmt_iterator gsi = *iter;
2623	gimple *g;
2624
2625	gcc_assert (!(size_in_bytes > 0 && !is_non_zero_len));
2626	gcc_assert (size_in_bytes == -1 || size_in_bytes >= 1);
2627
2628	gsi = *iter;
2629
2630	base = unshare_expr (base);
2631	base = maybe_create_ssa_name (loc, base, iter: &gsi, before_p);
2632
2633	if (len)
2634	{
2635	len = unshare_expr (len);
2636	len = maybe_cast_to_ptrmode (loc, len, iter, before_p);
2637	}
2638	else
2639	{
2640	gcc_assert (size_in_bytes != -1);
2641	len = build_int_cst (pointer_sized_int_node, size_in_bytes);
2642	}
2643
2644	if (size_in_bytes > 1)
2645	{
2646	if ((size_in_bytes & (size_in_bytes - 1)) != 0
2647	|| size_in_bytes > 16)
2648	is_scalar_access = false;
2649	else if (align && align < size_in_bytes * BITS_PER_UNIT)
2650	{
2651	/* On non-strict alignment targets, if
2652	16-byte access is just 8-byte aligned,
2653	this will result in misaligned shadow
2654	memory 2 byte load, but otherwise can
2655	be handled using one read. */
2656	if (size_in_bytes != 16
2657	|| STRICT_ALIGNMENT
2658	|| align < 8 * BITS_PER_UNIT)
2659	is_scalar_access = false;
2660	}
2661	}
2662
2663	HOST_WIDE_INT flags = 0;
2664	if (is_store)
2665	flags |= ASAN_CHECK_STORE;
2666	if (is_non_zero_len)
2667	flags |= ASAN_CHECK_NON_ZERO_LEN;
2668	if (is_scalar_access)
2669	flags |= ASAN_CHECK_SCALAR_ACCESS;
2670
2671	enum internal_fn fn = hwasan_sanitize_p ()
2672	? IFN_HWASAN_CHECK
2673	: IFN_ASAN_CHECK;
2674
2675	g = gimple_build_call_internal (fn, 4,
2676	build_int_cst (integer_type_node, flags),
2677	base, len,
2678	build_int_cst (integer_type_node,
2679	align / BITS_PER_UNIT));
2680	gimple_set_location (g, location: loc);
2681	if (before_p)
2682	gsi_insert_before (&gsi, g, GSI_SAME_STMT);
2683	else
2684	{
2685	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
2686	gsi_next (i: &gsi);
2687	*iter = gsi;
2688	}
2689	}
2690
2691	/* If T represents a memory access, add instrumentation code before ITER.
2692	LOCATION is source code location.
2693	IS_STORE is either TRUE (for a store) or FALSE (for a load). */
2694
2695	static void
2696	instrument_derefs (gimple_stmt_iterator *iter, tree t,
2697	location_t location, bool is_store)
2698	{
2699	if (is_store && !(asan_instrument_writes () || hwasan_instrument_writes ()))
2700	return;
2701	if (!is_store && !(asan_instrument_reads () || hwasan_instrument_reads ()))
2702	return;
2703
2704	tree type, base;
2705	HOST_WIDE_INT size_in_bytes;
2706	if (location == UNKNOWN_LOCATION)
2707	location = EXPR_LOCATION (t);
2708
2709	type = TREE_TYPE (t);
2710	switch (TREE_CODE (t))
2711	{
2712	case ARRAY_REF:
2713	case COMPONENT_REF:
2714	case INDIRECT_REF:
2715	case MEM_REF:
2716	case VAR_DECL:
2717	case BIT_FIELD_REF:
2718	break;
2719	/* FALLTHRU */
2720	default:
2721	return;
2722	}
2723
2724	size_in_bytes = int_size_in_bytes (type);
2725	if (size_in_bytes <= 0)
2726	return;
2727
2728	poly_int64 bitsize, bitpos;
2729	tree offset;
2730	machine_mode mode;
2731	int unsignedp, reversep, volatilep = 0;
2732	tree inner = get_inner_reference (t, &bitsize, &bitpos, &offset, &mode,
2733	&unsignedp, &reversep, &volatilep);
2734
2735	if (TREE_CODE (t) == COMPONENT_REF
2736	&& DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1)) != NULL_TREE)
2737	{
2738	tree repr = DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1));
2739	instrument_derefs (iter, t: build3 (COMPONENT_REF, TREE_TYPE (repr),
2740	TREE_OPERAND (t, 0), repr,
2741	TREE_OPERAND (t, 2)),
2742	location, is_store);
2743	return;
2744	}
2745
2746	if (!multiple_p (a: bitpos, BITS_PER_UNIT)
2747	|| maybe_ne (a: bitsize, b: size_in_bytes * BITS_PER_UNIT))
2748	return;
2749
2750	if (VAR_P (inner) && DECL_HARD_REGISTER (inner))
2751	return;
2752
2753	poly_int64 decl_size;
2754	if ((VAR_P (inner) || TREE_CODE (inner) == RESULT_DECL)
2755	&& offset == NULL_TREE
2756	&& DECL_SIZE (inner)
2757	&& poly_int_tree_p (DECL_SIZE (inner), value: &decl_size)
2758	&& known_subrange_p (pos1: bitpos, size1: bitsize, pos2: 0, size2: decl_size))
2759	{
2760	if (VAR_P (inner) && DECL_THREAD_LOCAL_P (inner))
2761	return;
2762	/* If we're not sanitizing globals and we can tell statically that this
2763	access is inside a global variable, then there's no point adding
2764	instrumentation to check the access. N.b. hwasan currently never
2765	sanitizes globals. */
2766	if ((hwasan_sanitize_p () || !param_asan_globals)
2767	&& is_global_var (t: inner))
2768	return;
2769	if (!TREE_STATIC (inner))
2770	{
2771	/* Automatic vars in the current function will be always
2772	accessible. */
2773	if (decl_function_context (inner) == current_function_decl
2774	&& (!asan_sanitize_use_after_scope ()
2775	|| !TREE_ADDRESSABLE (inner)))
2776	return;
2777	}
2778	/* Always instrument external vars, they might be dynamically
2779	initialized. */
2780	else if (!DECL_EXTERNAL (inner))
2781	{
2782	/* For static vars if they are known not to be dynamically
2783	initialized, they will be always accessible. */
2784	varpool_node *vnode = varpool_node::get (decl: inner);
2785	if (vnode && !vnode->dynamically_initialized)
2786	return;
2787	}
2788	}
2789
2790	if (DECL_P (inner)
2791	&& decl_function_context (inner) == current_function_decl
2792	&& !TREE_ADDRESSABLE (inner))
2793	mark_addressable (inner);
2794
2795	base = build_fold_addr_expr (t);
2796	if (!has_mem_ref_been_instrumented (ref: base, access_size: size_in_bytes))
2797	{
2798	unsigned int align = get_object_alignment (t);
2799	build_check_stmt (loc: location, base, NULL_TREE, size_in_bytes, iter,
2800	/*is_non_zero_len*/size_in_bytes > 0, /*before_p=*/true,
2801	is_store, /*is_scalar_access*/true, align);
2802	update_mem_ref_hash_table (ref: base, access_size: size_in_bytes);
2803	update_mem_ref_hash_table (ref: t, access_size: size_in_bytes);
2804	}
2805
2806	}
2807
2808	/* Insert a memory reference into the hash table if access length
2809	can be determined in compile time. */
2810
2811	static void
2812	maybe_update_mem_ref_hash_table (tree base, tree len)
2813	{
2814	if (!POINTER_TYPE_P (TREE_TYPE (base))
2815	|| !INTEGRAL_TYPE_P (TREE_TYPE (len)))
2816	return;
2817
2818	HOST_WIDE_INT size_in_bytes = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
2819
2820	if (size_in_bytes != -1)
2821	update_mem_ref_hash_table (ref: base, access_size: size_in_bytes);
2822	}
2823
2824	/* Instrument an access to a contiguous memory region that starts at
2825	the address pointed to by BASE, over a length of LEN (expressed in
2826	the sizeof (*BASE) bytes). ITER points to the instruction before
2827	which the instrumentation instructions must be inserted. LOCATION
2828	is the source location that the instrumentation instructions must
2829	have. If IS_STORE is true, then the memory access is a store;
2830	otherwise, it's a load. */
2831
2832	static void
2833	instrument_mem_region_access (tree base, tree len,
2834	gimple_stmt_iterator *iter,
2835	location_t location, bool is_store)
2836	{
2837	if (!POINTER_TYPE_P (TREE_TYPE (base))
2838	|| !INTEGRAL_TYPE_P (TREE_TYPE (len))
2839	|| integer_zerop (len))
2840	return;
2841
2842	HOST_WIDE_INT size_in_bytes = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
2843
2844	if ((size_in_bytes == -1)
2845	|| !has_mem_ref_been_instrumented (ref: base, access_size: size_in_bytes))
2846	{
2847	build_check_stmt (loc: location, base, len, size_in_bytes, iter,
2848	/*is_non_zero_len*/size_in_bytes > 0, /*before_p*/true,
2849	is_store, /*is_scalar_access*/false, /*align*/0);
2850	}
2851
2852	maybe_update_mem_ref_hash_table (base, len);
2853	*iter = gsi_for_stmt (gsi_stmt (i: *iter));
2854	}
2855
2856	/* Instrument the call to a built-in memory access function that is
2857	pointed to by the iterator ITER.
2858
2859	Upon completion, return TRUE iff *ITER has been advanced to the
2860	statement following the one it was originally pointing to. */
2861
2862	static bool
2863	instrument_builtin_call (gimple_stmt_iterator *iter)
2864	{
2865	if (!(asan_memintrin () || hwasan_memintrin ()))
2866	return false;
2867
2868	bool iter_advanced_p = false;
2869	gcall *call = as_a <gcall *> (p: gsi_stmt (i: *iter));
2870
2871	gcc_checking_assert (gimple_call_builtin_p (call, BUILT_IN_NORMAL));
2872
2873	location_t loc = gimple_location (g: call);
2874
2875	asan_mem_ref src0, src1, dest;
2876	asan_mem_ref_init (ref: &src0, NULL, access_size: 1);
2877	asan_mem_ref_init (ref: &src1, NULL, access_size: 1);
2878	asan_mem_ref_init (ref: &dest, NULL, access_size: 1);
2879
2880	tree src0_len = NULL_TREE, src1_len = NULL_TREE, dest_len = NULL_TREE;
2881	bool src0_is_store = false, src1_is_store = false, dest_is_store = false,
2882	dest_is_deref = false, intercepted_p = true;
2883
2884	if (get_mem_refs_of_builtin_call (call,
2885	src0: &src0, src0_len: &src0_len, src0_is_store: &src0_is_store,
2886	src1: &src1, src1_len: &src1_len, src1_is_store: &src1_is_store,
2887	dst: &dest, dst_len: &dest_len, dst_is_store: &dest_is_store,
2888	dest_is_deref: &dest_is_deref, intercepted_p: &intercepted_p, iter))
2889	{
2890	if (dest_is_deref)
2891	{
2892	instrument_derefs (iter, t: dest.start, location: loc, is_store: dest_is_store);
2893	gsi_next (i: iter);
2894	iter_advanced_p = true;
2895	}
2896	else if (!intercepted_p
2897	&& (src0_len || src1_len || dest_len))
2898	{
2899	if (src0.start != NULL_TREE)
2900	instrument_mem_region_access (base: src0.start, len: src0_len,
2901	iter, location: loc, /*is_store=*/false);
2902	if (src1.start != NULL_TREE)
2903	instrument_mem_region_access (base: src1.start, len: src1_len,
2904	iter, location: loc, /*is_store=*/false);
2905	if (dest.start != NULL_TREE)
2906	instrument_mem_region_access (base: dest.start, len: dest_len,
2907	iter, location: loc, /*is_store=*/true);
2908
2909	*iter = gsi_for_stmt (call);
2910	gsi_next (i: iter);
2911	iter_advanced_p = true;
2912	}
2913	else
2914	{
2915	if (src0.start != NULL_TREE)
2916	maybe_update_mem_ref_hash_table (base: src0.start, len: src0_len);
2917	if (src1.start != NULL_TREE)
2918	maybe_update_mem_ref_hash_table (base: src1.start, len: src1_len);
2919	if (dest.start != NULL_TREE)
2920	maybe_update_mem_ref_hash_table (base: dest.start, len: dest_len);
2921	}
2922	}
2923	return iter_advanced_p;
2924	}
2925
2926	/* Instrument the assignment statement ITER if it is subject to
2927	instrumentation. Return TRUE iff instrumentation actually
2928	happened. In that case, the iterator ITER is advanced to the next
2929	logical expression following the one initially pointed to by ITER,
2930	and the relevant memory reference that which access has been
2931	instrumented is added to the memory references hash table. */
2932
2933	static bool
2934	maybe_instrument_assignment (gimple_stmt_iterator *iter)
2935	{
2936	gimple *s = gsi_stmt (i: *iter);
2937
2938	gcc_assert (gimple_assign_single_p (s));
2939
2940	tree ref_expr = NULL_TREE;
2941	bool is_store, is_instrumented = false;
2942
2943	if (gimple_store_p (gs: s))
2944	{
2945	ref_expr = gimple_assign_lhs (gs: s);
2946	is_store = true;
2947	instrument_derefs (iter, t: ref_expr,
2948	location: gimple_location (g: s),
2949	is_store);
2950	is_instrumented = true;
2951	}
2952
2953	if (gimple_assign_load_p (s))
2954	{
2955	ref_expr = gimple_assign_rhs1 (gs: s);
2956	is_store = false;
2957	instrument_derefs (iter, t: ref_expr,
2958	location: gimple_location (g: s),
2959	is_store);
2960	is_instrumented = true;
2961	}
2962
2963	if (is_instrumented)
2964	gsi_next (i: iter);
2965
2966	return is_instrumented;
2967	}
2968
2969	/* Instrument the function call pointed to by the iterator ITER, if it
2970	is subject to instrumentation. At the moment, the only function
2971	calls that are instrumented are some built-in functions that access
2972	memory. Look at instrument_builtin_call to learn more.
2973
2974	Upon completion return TRUE iff *ITER was advanced to the statement
2975	following the one it was originally pointing to. */
2976
2977	static bool
2978	maybe_instrument_call (gimple_stmt_iterator *iter)
2979	{
2980	gimple *stmt = gsi_stmt (i: *iter);
2981	bool is_builtin = gimple_call_builtin_p (stmt, BUILT_IN_NORMAL);
2982
2983	if (is_builtin && instrument_builtin_call (iter))
2984	return true;
2985
2986	if (gimple_call_noreturn_p (s: stmt))
2987	{
2988	if (is_builtin)
2989	{
2990	tree callee = gimple_call_fndecl (gs: stmt);
2991	switch (DECL_FUNCTION_CODE (decl: callee))
2992	{
2993	case BUILT_IN_UNREACHABLE:
2994	case BUILT_IN_UNREACHABLE_TRAP:
2995	case BUILT_IN_TRAP:
2996	/* Don't instrument these. */
2997	return false;
2998	default:
2999	break;
3000	}
3001	}
3002	/* If a function does not return, then we must handle clearing up the
3003	shadow stack accordingly. For ASAN we can simply set the entire stack
3004	to "valid" for accesses by setting the shadow space to 0 and all
3005	accesses will pass checks. That means that some bad accesses may be
3006	missed, but we will not report any false positives.
3007
3008	This is not possible for HWASAN. Since there is no "always valid" tag
3009	we can not set any space to "always valid". If we were to clear the
3010	entire shadow stack then code resuming from `longjmp` or a caught
3011	exception would trigger false positives when correctly accessing
3012	variables on the stack. Hence we need to handle things like
3013	`longjmp`, thread exit, and exceptions in a different way. These
3014	problems must be handled externally to the compiler, e.g. in the
3015	language runtime. */
3016	if (! hwasan_sanitize_p ())
3017	{
3018	tree decl = builtin_decl_implicit (fncode: BUILT_IN_ASAN_HANDLE_NO_RETURN);
3019	gimple *g = gimple_build_call (decl, 0);
3020	gimple_set_location (g, location: gimple_location (g: stmt));
3021	gsi_insert_before (iter, g, GSI_SAME_STMT);
3022	}
3023	}
3024
3025	bool instrumented = false;
3026	if (gimple_store_p (gs: stmt))
3027	{
3028	tree ref_expr = gimple_call_lhs (gs: stmt);
3029	instrument_derefs (iter, t: ref_expr,
3030	location: gimple_location (g: stmt),
3031	/*is_store=*/true);
3032
3033	instrumented = true;
3034	}
3035
3036	/* Walk through gimple_call arguments and check them id needed. */
3037	unsigned args_num = gimple_call_num_args (gs: stmt);
3038	for (unsigned i = 0; i < args_num; ++i)
3039	{
3040	tree arg = gimple_call_arg (gs: stmt, index: i);
3041	/* If ARG is not a non-aggregate register variable, compiler in general
3042	creates temporary for it and pass it as argument to gimple call.
3043	But in some cases, e.g. when we pass by value a small structure that
3044	fits to register, compiler can avoid extra overhead by pulling out
3045	these temporaries. In this case, we should check the argument. */
3046	if (!is_gimple_reg (arg) && !is_gimple_min_invariant (arg))
3047	{
3048	instrument_derefs (iter, t: arg,
3049	location: gimple_location (g: stmt),
3050	/*is_store=*/false);
3051	instrumented = true;
3052	}
3053	}
3054	if (instrumented)
3055	gsi_next (i: iter);
3056	return instrumented;
3057	}
3058
3059	/* Walk each instruction of all basic block and instrument those that
3060	represent memory references: loads, stores, or function calls.
3061	In a given basic block, this function avoids instrumenting memory
3062	references that have already been instrumented. */
3063
3064	static void
3065	transform_statements (void)
3066	{
3067	basic_block bb, last_bb = NULL;
3068	gimple_stmt_iterator i;
3069	int saved_last_basic_block = last_basic_block_for_fn (cfun);
3070
3071	FOR_EACH_BB_FN (bb, cfun)
3072	{
3073	basic_block prev_bb = bb;
3074
3075	if (bb->index >= saved_last_basic_block) continue;
3076
3077	/* Flush the mem ref hash table, if current bb doesn't have
3078	exactly one predecessor, or if that predecessor (skipping
3079	over asan created basic blocks) isn't the last processed
3080	basic block. Thus we effectively flush on extended basic
3081	block boundaries. */
3082	while (single_pred_p (bb: prev_bb))
3083	{
3084	prev_bb = single_pred (bb: prev_bb);
3085	if (prev_bb->index < saved_last_basic_block)
3086	break;
3087	}
3088	if (prev_bb != last_bb)
3089	empty_mem_ref_hash_table ();
3090	last_bb = bb;
3091
3092	for (i = gsi_start_bb (bb); !gsi_end_p (i);)
3093	{
3094	gimple *s = gsi_stmt (i);
3095
3096	if (has_stmt_been_instrumented_p (stmt: s))
3097	gsi_next (i: &i);
3098	else if (gimple_assign_single_p (gs: s)
3099	&& !gimple_clobber_p (s)
3100	&& maybe_instrument_assignment (iter: &i))
3101	/* Nothing to do as maybe_instrument_assignment advanced
3102	the iterator I. */;
3103	else if (is_gimple_call (gs: s) && maybe_instrument_call (iter: &i))
3104	/* Nothing to do as maybe_instrument_call
3105	advanced the iterator I. */;
3106	else
3107	{
3108	/* No instrumentation happened.
3109
3110	If the current instruction is a function call that
3111	might free something, let's forget about the memory
3112	references that got instrumented. Otherwise we might
3113	miss some instrumentation opportunities. Do the same
3114	for a ASAN_MARK poisoning internal function. */
3115	if (is_gimple_call (gs: s)
3116	&& (!nonfreeing_call_p (s)
3117	|| asan_mark_p (stmt: s, flag: ASAN_MARK_POISON)))
3118	empty_mem_ref_hash_table ();
3119
3120	gsi_next (i: &i);
3121	}
3122	}
3123	}
3124	free_mem_ref_resources ();
3125	}
3126
3127	/* Build
3128	__asan_before_dynamic_init (module_name)
3129	or
3130	__asan_after_dynamic_init ()
3131	call. */
3132
3133	tree
3134	asan_dynamic_init_call (bool after_p)
3135	{
3136	if (shadow_ptr_types[0] == NULL_TREE)
3137	asan_init_shadow_ptr_types ();
3138
3139	tree fn = builtin_decl_implicit (fncode: after_p
3140	? BUILT_IN_ASAN_AFTER_DYNAMIC_INIT
3141	: BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT);
3142	tree module_name_cst = NULL_TREE;
3143	if (!after_p)
3144	{
3145	pretty_printer module_name_pp;
3146	pp_string (&module_name_pp, main_input_filename);
3147
3148	module_name_cst = asan_pp_string (pp: &module_name_pp);
3149	module_name_cst = fold_convert (const_ptr_type_node,
3150	module_name_cst);
3151	}
3152
3153	return build_call_expr (fn, after_p ? 0 : 1, module_name_cst);
3154	}
3155
3156	/* Build
3157	struct __asan_global
3158	{
3159	const void *__beg;
3160	uptr __size;
3161	uptr __size_with_redzone;
3162	const void *__name;
3163	const void *__module_name;
3164	uptr __has_dynamic_init;
3165	__asan_global_source_location *__location;
3166	char *__odr_indicator;
3167	} type. */
3168
3169	static tree
3170	asan_global_struct (void)
3171	{
3172	static const char *field_names[]
3173	= { "__beg", "__size", "__size_with_redzone",
3174	"__name", "__module_name", "__has_dynamic_init", "__location",
3175	"__odr_indicator" };
3176	tree fields[ARRAY_SIZE (field_names)], ret;
3177	unsigned i;
3178
3179	ret = make_node (RECORD_TYPE);
3180	for (i = 0; i < ARRAY_SIZE (field_names); i++)
3181	{
3182	fields[i]
3183	= build_decl (UNKNOWN_LOCATION, FIELD_DECL,
3184	get_identifier (field_names[i]),
3185	(i == 0 || i == 3) ? const_ptr_type_node
3186	: pointer_sized_int_node);
3187	DECL_CONTEXT (fields[i]) = ret;
3188	if (i)
3189	DECL_CHAIN (fields[i - 1]) = fields[i];
3190	}
3191	tree type_decl = build_decl (input_location, TYPE_DECL,
3192	get_identifier ("__asan_global"), ret);
3193	DECL_IGNORED_P (type_decl) = 1;
3194	DECL_ARTIFICIAL (type_decl) = 1;
3195	TYPE_FIELDS (ret) = fields[0];
3196	TYPE_NAME (ret) = type_decl;
3197	TYPE_STUB_DECL (ret) = type_decl;
3198	TYPE_ARTIFICIAL (ret) = 1;
3199	layout_type (ret);
3200	return ret;
3201	}
3202
3203	/* Create and return odr indicator symbol for DECL.
3204	TYPE is __asan_global struct type as returned by asan_global_struct. */
3205
3206	static tree
3207	create_odr_indicator (tree decl, tree type)
3208	{
3209	char *name;
3210	tree uptr = TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (type)));
3211	tree decl_name
3212	= (HAS_DECL_ASSEMBLER_NAME_P (decl) ? DECL_ASSEMBLER_NAME (decl)
3213	: DECL_NAME (decl));
3214	/* DECL_NAME theoretically might be NULL. Bail out with 0 in this case. */
3215	if (decl_name == NULL_TREE)
3216	return build_int_cst (uptr, 0);
3217	const char *dname = IDENTIFIER_POINTER (decl_name);
3218	if (HAS_DECL_ASSEMBLER_NAME_P (decl))
3219	dname = targetm.strip_name_encoding (dname);
3220	size_t len = strlen (s: dname) + sizeof ("__odr_asan_");
3221	name = XALLOCAVEC (char, len);
3222	snprintf (s: name, maxlen: len, format: "__odr_asan_%s", dname);
3223	#ifndef NO_DOT_IN_LABEL
3224	name[sizeof ("__odr_asan") - 1] = '.';
3225	#elif !defined(NO_DOLLAR_IN_LABEL)
3226	name[sizeof ("__odr_asan") - 1] = '$';
3227	#endif
3228	tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (name),
3229	char_type_node);
3230	TREE_ADDRESSABLE (var) = 1;
3231	TREE_READONLY (var) = 0;
3232	TREE_THIS_VOLATILE (var) = 1;
3233	DECL_ARTIFICIAL (var) = 1;
3234	DECL_IGNORED_P (var) = 1;
3235	TREE_STATIC (var) = 1;
3236	TREE_PUBLIC (var) = 1;
3237	DECL_VISIBILITY (var) = DECL_VISIBILITY (decl);
3238	DECL_VISIBILITY_SPECIFIED (var) = DECL_VISIBILITY_SPECIFIED (decl);
3239
3240	TREE_USED (var) = 1;
3241	tree ctor = build_constructor_va (TREE_TYPE (var), 1, NULL_TREE,
3242	build_int_cst (unsigned_type_node, 0));
3243	TREE_CONSTANT (ctor) = 1;
3244	TREE_STATIC (ctor) = 1;
3245	DECL_INITIAL (var) = ctor;
3246	DECL_ATTRIBUTES (var) = tree_cons (get_identifier ("asan odr indicator"),
3247	NULL, DECL_ATTRIBUTES (var));
3248	make_decl_rtl (var);
3249	varpool_node::finalize_decl (decl: var);
3250	return fold_convert (uptr, build_fold_addr_expr (var));
3251	}
3252
3253	/* Return true if DECL, a global var, might be overridden and needs
3254	an additional odr indicator symbol. */
3255
3256	static bool
3257	asan_needs_odr_indicator_p (tree decl)
3258	{
3259	/* Don't emit ODR indicators for kernel because:
3260	a) Kernel is written in C thus doesn't need ODR indicators.
3261	b) Some kernel code may have assumptions about symbols containing specific
3262	patterns in their names. Since ODR indicators contain original names
3263	of symbols they are emitted for, these assumptions would be broken for
3264	ODR indicator symbols. */
3265	return (!(flag_sanitize & SANITIZE_KERNEL_ADDRESS)
3266	&& !DECL_ARTIFICIAL (decl)
3267	&& !DECL_WEAK (decl)
3268	&& TREE_PUBLIC (decl));
3269	}
3270
3271	/* Append description of a single global DECL into vector V.
3272	TYPE is __asan_global struct type as returned by asan_global_struct. */
3273
3274	static void
3275	asan_add_global (tree decl, tree type, vec<constructor_elt, va_gc> *v)
3276	{
3277	tree init, uptr = TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (type)));
3278	unsigned HOST_WIDE_INT size;
3279	tree str_cst, module_name_cst, refdecl = decl;
3280	vec<constructor_elt, va_gc> *vinner = NULL;
3281
3282	pretty_printer asan_pp, module_name_pp;
3283
3284	if (DECL_NAME (decl))
3285	pp_tree_identifier (&asan_pp, DECL_NAME (decl));
3286	else
3287	pp_string (&asan_pp, "<unknown>");
3288	str_cst = asan_pp_string (pp: &asan_pp);
3289
3290	if (!in_lto_p)
3291	pp_string (&module_name_pp, main_input_filename);
3292	else
3293	{
3294	const_tree tu = get_ultimate_context ((const_tree)decl);
3295	if (tu != NULL_TREE)
3296	pp_string (&module_name_pp, IDENTIFIER_POINTER (DECL_NAME (tu)));
3297	else
3298	pp_string (&module_name_pp, aux_base_name);
3299	}
3300
3301	module_name_cst = asan_pp_string (pp: &module_name_pp);
3302
3303	if (asan_needs_local_alias (decl))
3304	{
3305	char buf[20];
3306	ASM_GENERATE_INTERNAL_LABEL (buf, "LASAN", vec_safe_length (v) + 1);
3307	refdecl = build_decl (DECL_SOURCE_LOCATION (decl),
3308	VAR_DECL, get_identifier (buf), TREE_TYPE (decl));
3309	TREE_ADDRESSABLE (refdecl) = TREE_ADDRESSABLE (decl);
3310	TREE_READONLY (refdecl) = TREE_READONLY (decl);
3311	TREE_THIS_VOLATILE (refdecl) = TREE_THIS_VOLATILE (decl);
3312	DECL_NOT_GIMPLE_REG_P (refdecl) = DECL_NOT_GIMPLE_REG_P (decl);
3313	DECL_ARTIFICIAL (refdecl) = DECL_ARTIFICIAL (decl);
3314	DECL_IGNORED_P (refdecl) = DECL_IGNORED_P (decl);
3315	TREE_STATIC (refdecl) = 1;
3316	TREE_PUBLIC (refdecl) = 0;
3317	TREE_USED (refdecl) = 1;
3318	assemble_alias (refdecl, DECL_ASSEMBLER_NAME (decl));
3319	}
3320
3321	tree odr_indicator_ptr
3322	= (asan_needs_odr_indicator_p (decl) ? create_odr_indicator (decl, type)
3323	: build_int_cst (uptr, 0));
3324	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3325	fold_convert (const_ptr_type_node,
3326	build_fold_addr_expr (refdecl)));
3327	size = tree_to_uhwi (DECL_SIZE_UNIT (decl));
3328	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, build_int_cst (uptr, size));
3329	size += asan_red_zone_size (size);
3330	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, build_int_cst (uptr, size));
3331	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3332	fold_convert (const_ptr_type_node, str_cst));
3333	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3334	fold_convert (const_ptr_type_node, module_name_cst));
3335	varpool_node *vnode = varpool_node::get (decl);
3336	int has_dynamic_init = 0;
3337	/* FIXME: Enable initialization order fiasco detection in LTO mode once
3338	proper fix for PR 79061 will be applied. */
3339	if (!in_lto_p)
3340	has_dynamic_init = vnode ? vnode->dynamically_initialized : 0;
3341	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
3342	build_int_cst (uptr, has_dynamic_init));
3343	tree locptr = NULL_TREE;
3344	location_t loc = DECL_SOURCE_LOCATION (decl);
3345	expanded_location xloc = expand_location (loc);
3346	if (xloc.file != NULL)
3347	{
3348	static int lasanloccnt = 0;
3349	char buf[25];
3350	ASM_GENERATE_INTERNAL_LABEL (buf, "LASANLOC", ++lasanloccnt);
3351	tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (buf),
3352	ubsan_get_source_location_type ());
3353	TREE_STATIC (var) = 1;
3354	TREE_PUBLIC (var) = 0;
3355	DECL_ARTIFICIAL (var) = 1;
3356	DECL_IGNORED_P (var) = 1;
3357	pretty_printer filename_pp;
3358	pp_string (&filename_pp, xloc.file);
3359	tree str = asan_pp_string (pp: &filename_pp);
3360	tree ctor = build_constructor_va (TREE_TYPE (var), 3,
3361	NULL_TREE, str, NULL_TREE,
3362	build_int_cst (unsigned_type_node,
3363	xloc.line), NULL_TREE,
3364	build_int_cst (unsigned_type_node,
3365	xloc.column));
3366	TREE_CONSTANT (ctor) = 1;
3367	TREE_STATIC (ctor) = 1;
3368	DECL_INITIAL (var) = ctor;
3369	varpool_node::finalize_decl (decl: var);
3370	locptr = fold_convert (uptr, build_fold_addr_expr (var));
3371	}
3372	else
3373	locptr = build_int_cst (uptr, 0);
3374	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, locptr);
3375	CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, odr_indicator_ptr);
3376	init = build_constructor (type, vinner);
3377	CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, init);
3378	}
3379
3380	/* Initialize sanitizer.def builtins if the FE hasn't initialized them. */
3381	void
3382	initialize_sanitizer_builtins (void)
3383	{
3384	tree decl;
3385
3386	if (builtin_decl_implicit_p (fncode: BUILT_IN_ASAN_INIT))
3387	return;
3388
3389	tree BT_FN_VOID = build_function_type_list (void_type_node, NULL_TREE);
3390	tree BT_FN_VOID_PTR
3391	= build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
3392	tree BT_FN_VOID_CONST_PTR
3393	= build_function_type_list (void_type_node, const_ptr_type_node, NULL_TREE);
3394	tree BT_FN_VOID_PTR_PTR
3395	= build_function_type_list (void_type_node, ptr_type_node,
3396	ptr_type_node, NULL_TREE);
3397	tree BT_FN_VOID_PTR_PTR_PTR
3398	= build_function_type_list (void_type_node, ptr_type_node,
3399	ptr_type_node, ptr_type_node, NULL_TREE);
3400	tree BT_FN_VOID_PTR_PTRMODE
3401	= build_function_type_list (void_type_node, ptr_type_node,
3402	pointer_sized_int_node, NULL_TREE);
3403	tree BT_FN_VOID_INT
3404	= build_function_type_list (void_type_node, integer_type_node, NULL_TREE);
3405	tree BT_FN_SIZE_CONST_PTR_INT
3406	= build_function_type_list (size_type_node, const_ptr_type_node,
3407	integer_type_node, NULL_TREE);
3408
3409	tree BT_FN_VOID_UINT8_UINT8
3410	= build_function_type_list (void_type_node, unsigned_char_type_node,
3411	unsigned_char_type_node, NULL_TREE);
3412	tree BT_FN_VOID_UINT16_UINT16
3413	= build_function_type_list (void_type_node, uint16_type_node,
3414	uint16_type_node, NULL_TREE);
3415	tree BT_FN_VOID_UINT32_UINT32
3416	= build_function_type_list (void_type_node, uint32_type_node,
3417	uint32_type_node, NULL_TREE);
3418	tree BT_FN_VOID_UINT64_UINT64
3419	= build_function_type_list (void_type_node, uint64_type_node,
3420	uint64_type_node, NULL_TREE);
3421	tree BT_FN_VOID_FLOAT_FLOAT
3422	= build_function_type_list (void_type_node, float_type_node,
3423	float_type_node, NULL_TREE);
3424	tree BT_FN_VOID_DOUBLE_DOUBLE
3425	= build_function_type_list (void_type_node, double_type_node,
3426	double_type_node, NULL_TREE);
3427	tree BT_FN_VOID_UINT64_PTR
3428	= build_function_type_list (void_type_node, uint64_type_node,
3429	ptr_type_node, NULL_TREE);
3430
3431	tree BT_FN_PTR_CONST_PTR_UINT8
3432	= build_function_type_list (ptr_type_node, const_ptr_type_node,
3433	unsigned_char_type_node, NULL_TREE);
3434	tree BT_FN_VOID_PTR_UINT8_PTRMODE
3435	= build_function_type_list (void_type_node, ptr_type_node,
3436	unsigned_char_type_node,
3437	pointer_sized_int_node, NULL_TREE);
3438
3439	tree BT_FN_BOOL_VPTR_PTR_IX_INT_INT[5];
3440	tree BT_FN_IX_CONST_VPTR_INT[5];
3441	tree BT_FN_IX_VPTR_IX_INT[5];
3442	tree BT_FN_VOID_VPTR_IX_INT[5];
3443	tree vptr
3444	= build_pointer_type (build_qualified_type (void_type_node,
3445	TYPE_QUAL_VOLATILE));
3446	tree cvptr
3447	= build_pointer_type (build_qualified_type (void_type_node,
3448	TYPE_QUAL_VOLATILE
3449	|TYPE_QUAL_CONST));
3450	tree boolt
3451	= lang_hooks.types.type_for_size (BOOL_TYPE_SIZE, 1);
3452	int i;
3453	for (i = 0; i < 5; i++)
3454	{
3455	tree ix = build_nonstandard_integer_type (BITS_PER_UNIT * (1 << i), 1);
3456	BT_FN_BOOL_VPTR_PTR_IX_INT_INT[i]
3457	= build_function_type_list (boolt, vptr, ptr_type_node, ix,
3458	integer_type_node, integer_type_node,
3459	NULL_TREE);
3460	BT_FN_IX_CONST_VPTR_INT[i]
3461	= build_function_type_list (ix, cvptr, integer_type_node, NULL_TREE);
3462	BT_FN_IX_VPTR_IX_INT[i]
3463	= build_function_type_list (ix, vptr, ix, integer_type_node,
3464	NULL_TREE);
3465	BT_FN_VOID_VPTR_IX_INT[i]
3466	= build_function_type_list (void_type_node, vptr, ix,
3467	integer_type_node, NULL_TREE);
3468	}
3469	#define BT_FN_BOOL_VPTR_PTR_I1_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[0]
3470	#define BT_FN_I1_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[0]
3471	#define BT_FN_I1_VPTR_I1_INT BT_FN_IX_VPTR_IX_INT[0]
3472	#define BT_FN_VOID_VPTR_I1_INT BT_FN_VOID_VPTR_IX_INT[0]
3473	#define BT_FN_BOOL_VPTR_PTR_I2_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[1]
3474	#define BT_FN_I2_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[1]
3475	#define BT_FN_I2_VPTR_I2_INT BT_FN_IX_VPTR_IX_INT[1]
3476	#define BT_FN_VOID_VPTR_I2_INT BT_FN_VOID_VPTR_IX_INT[1]
3477	#define BT_FN_BOOL_VPTR_PTR_I4_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[2]
3478	#define BT_FN_I4_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[2]
3479	#define BT_FN_I4_VPTR_I4_INT BT_FN_IX_VPTR_IX_INT[2]
3480	#define BT_FN_VOID_VPTR_I4_INT BT_FN_VOID_VPTR_IX_INT[2]
3481	#define BT_FN_BOOL_VPTR_PTR_I8_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[3]
3482	#define BT_FN_I8_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[3]
3483	#define BT_FN_I8_VPTR_I8_INT BT_FN_IX_VPTR_IX_INT[3]
3484	#define BT_FN_VOID_VPTR_I8_INT BT_FN_VOID_VPTR_IX_INT[3]
3485	#define BT_FN_BOOL_VPTR_PTR_I16_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[4]
3486	#define BT_FN_I16_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[4]
3487	#define BT_FN_I16_VPTR_I16_INT BT_FN_IX_VPTR_IX_INT[4]
3488	#define BT_FN_VOID_VPTR_I16_INT BT_FN_VOID_VPTR_IX_INT[4]
3489	#undef ATTR_NOTHROW_LIST
3490	#define ATTR_NOTHROW_LIST ECF_NOTHROW
3491	#undef ATTR_NOTHROW_LEAF_LIST
3492	#define ATTR_NOTHROW_LEAF_LIST ECF_NOTHROW | ECF_LEAF
3493	#undef ATTR_TMPURE_NOTHROW_LEAF_LIST
3494	#define ATTR_TMPURE_NOTHROW_LEAF_LIST ECF_TM_PURE | ATTR_NOTHROW_LEAF_LIST
3495	#undef ATTR_NORETURN_NOTHROW_LEAF_LIST
3496	#define ATTR_NORETURN_NOTHROW_LEAF_LIST ECF_NORETURN | ATTR_NOTHROW_LEAF_LIST
3497	#undef ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST
3498	#define ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST \
3499	ECF_CONST | ATTR_NORETURN_NOTHROW_LEAF_LIST
3500	#undef ATTR_TMPURE_NORETURN_NOTHROW_LEAF_LIST
3501	#define ATTR_TMPURE_NORETURN_NOTHROW_LEAF_LIST \
3502	ECF_TM_PURE | ATTR_NORETURN_NOTHROW_LEAF_LIST
3503	#undef ATTR_COLD_NOTHROW_LEAF_LIST
3504	#define ATTR_COLD_NOTHROW_LEAF_LIST \
3505	/* ECF_COLD missing */ ATTR_NOTHROW_LEAF_LIST
3506	#undef ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST
3507	#define ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST \
3508	/* ECF_COLD missing */ ATTR_NORETURN_NOTHROW_LEAF_LIST
3509	#undef ATTR_COLD_CONST_NORETURN_NOTHROW_LEAF_LIST
3510	#define ATTR_COLD_CONST_NORETURN_NOTHROW_LEAF_LIST \
3511	/* ECF_COLD missing */ ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST
3512	#undef ATTR_PURE_NOTHROW_LEAF_LIST
3513	#define ATTR_PURE_NOTHROW_LEAF_LIST ECF_PURE | ATTR_NOTHROW_LEAF_LIST
3514	#undef DEF_BUILTIN_STUB
3515	#define DEF_BUILTIN_STUB(ENUM, NAME)
3516	#undef DEF_SANITIZER_BUILTIN_1
3517	#define DEF_SANITIZER_BUILTIN_1(ENUM, NAME, TYPE, ATTRS) \
3518	do { \
3519	decl = add_builtin_function ("__builtin_" NAME, TYPE, ENUM, \
3520	BUILT_IN_NORMAL, NAME, NULL_TREE); \
3521	set_call_expr_flags (decl, ATTRS); \
3522	set_builtin_decl (ENUM, decl, true); \
3523	} while (0)
3524	#undef DEF_SANITIZER_BUILTIN
3525	#define DEF_SANITIZER_BUILTIN(ENUM, NAME, TYPE, ATTRS) \
3526	DEF_SANITIZER_BUILTIN_1 (ENUM, NAME, TYPE, ATTRS);
3527
3528	#include "sanitizer.def"
3529
3530	/* -fsanitize=object-size uses __builtin_dynamic_object_size and
3531	__builtin_object_size, but they might not be available for e.g. Fortran at
3532	this point. We use DEF_SANITIZER_BUILTIN here only as a convenience
3533	macro. */
3534	if (flag_sanitize & SANITIZE_OBJECT_SIZE)
3535	{
3536	if (!builtin_decl_implicit_p (fncode: BUILT_IN_OBJECT_SIZE))
3537	DEF_SANITIZER_BUILTIN_1 (BUILT_IN_OBJECT_SIZE, "object_size",
3538	BT_FN_SIZE_CONST_PTR_INT,
3539	ATTR_PURE_NOTHROW_LEAF_LIST);
3540	if (!builtin_decl_implicit_p (fncode: BUILT_IN_DYNAMIC_OBJECT_SIZE))
3541	DEF_SANITIZER_BUILTIN_1 (BUILT_IN_DYNAMIC_OBJECT_SIZE,
3542	"dynamic_object_size",
3543	BT_FN_SIZE_CONST_PTR_INT,
3544	ATTR_PURE_NOTHROW_LEAF_LIST);
3545	}
3546
3547	#undef DEF_SANITIZER_BUILTIN_1
3548	#undef DEF_SANITIZER_BUILTIN
3549	#undef DEF_BUILTIN_STUB
3550	}
3551
3552	/* Called via htab_traverse. Count number of emitted
3553	STRING_CSTs in the constant hash table. */
3554
3555	int
3556	count_string_csts (constant_descriptor_tree **slot,
3557	unsigned HOST_WIDE_INT *data)
3558	{
3559	struct constant_descriptor_tree *desc = *slot;
3560	if (TREE_CODE (desc->value) == STRING_CST
3561	&& TREE_ASM_WRITTEN (desc->value)
3562	&& asan_protect_global (decl: desc->value))
3563	++*data;
3564	return 1;
3565	}
3566
3567	/* Helper structure to pass two parameters to
3568	add_string_csts. */
3569
3570	struct asan_add_string_csts_data
3571	{
3572	tree type;
3573	vec<constructor_elt, va_gc> *v;
3574	};
3575
3576	/* Called via hash_table::traverse. Call asan_add_global
3577	on emitted STRING_CSTs from the constant hash table. */
3578
3579	int
3580	add_string_csts (constant_descriptor_tree **slot,
3581	asan_add_string_csts_data *aascd)
3582	{
3583	struct constant_descriptor_tree *desc = *slot;
3584	if (TREE_CODE (desc->value) == STRING_CST
3585	&& TREE_ASM_WRITTEN (desc->value)
3586	&& asan_protect_global (decl: desc->value))
3587	{
3588	asan_add_global (SYMBOL_REF_DECL (XEXP (desc->rtl, 0)),
3589	type: aascd->type, v: aascd->v);
3590	}
3591	return 1;
3592	}
3593
3594	/* Needs to be GTY(()), because cgraph_build_static_cdtor may
3595	invoke ggc_collect. */
3596	static GTY(()) tree asan_ctor_statements;
3597
3598	/* Module-level instrumentation.
3599	- Insert __asan_init_vN() into the list of CTORs.
3600	- TODO: insert redzones around globals.
3601	*/
3602
3603	void
3604	asan_finish_file (void)
3605	{
3606	varpool_node *vnode;
3607	unsigned HOST_WIDE_INT gcount = 0;
3608
3609	if (shadow_ptr_types[0] == NULL_TREE)
3610	asan_init_shadow_ptr_types ();
3611	/* Avoid instrumenting code in the asan ctors/dtors.
3612	We don't need to insert padding after the description strings,
3613	nor after .LASAN* array. */
3614	flag_sanitize &= ~SANITIZE_ADDRESS;
3615
3616	/* For user-space we want asan constructors to run first.
3617	Linux kernel does not support priorities other than default, and the only
3618	other user of constructors is coverage. So we run with the default
3619	priority. */
3620	int priority = flag_sanitize & SANITIZE_USER_ADDRESS
3621	? MAX_RESERVED_INIT_PRIORITY - 1 : DEFAULT_INIT_PRIORITY;
3622
3623	if (flag_sanitize & SANITIZE_USER_ADDRESS)
3624	{
3625	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_INIT);
3626	append_to_statement_list (build_call_expr (fn, 0), &asan_ctor_statements);
3627	fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_VERSION_MISMATCH_CHECK);
3628	append_to_statement_list (build_call_expr (fn, 0), &asan_ctor_statements);
3629	}
3630	FOR_EACH_DEFINED_VARIABLE (vnode)
3631	if (TREE_ASM_WRITTEN (vnode->decl)
3632	&& asan_protect_global (decl: vnode->decl))
3633	++gcount;
3634	hash_table<tree_descriptor_hasher> *const_desc_htab = constant_pool_htab ();
3635	const_desc_htab->traverse<unsigned HOST_WIDE_INT *, count_string_csts>
3636	(argument: &gcount);
3637	if (gcount)
3638	{
3639	tree type = asan_global_struct (), var, ctor;
3640	tree dtor_statements = NULL_TREE;
3641	vec<constructor_elt, va_gc> *v;
3642	char buf[20];
3643
3644	type = build_array_type_nelts (type, gcount);
3645	ASM_GENERATE_INTERNAL_LABEL (buf, "LASAN", 0);
3646	var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (buf),
3647	type);
3648	TREE_STATIC (var) = 1;
3649	TREE_PUBLIC (var) = 0;
3650	DECL_ARTIFICIAL (var) = 1;
3651	DECL_IGNORED_P (var) = 1;
3652	vec_alloc (v, nelems: gcount);
3653	FOR_EACH_DEFINED_VARIABLE (vnode)
3654	if (TREE_ASM_WRITTEN (vnode->decl)
3655	&& asan_protect_global (decl: vnode->decl))
3656	asan_add_global (decl: vnode->decl, TREE_TYPE (type), v);
3657	struct asan_add_string_csts_data aascd;
3658	aascd.type = TREE_TYPE (type);
3659	aascd.v = v;
3660	const_desc_htab->traverse<asan_add_string_csts_data *, add_string_csts>
3661	(argument: &aascd);
3662	ctor = build_constructor (type, v);
3663	TREE_CONSTANT (ctor) = 1;
3664	TREE_STATIC (ctor) = 1;
3665	DECL_INITIAL (var) = ctor;
3666	SET_DECL_ALIGN (var, MAX (DECL_ALIGN (var),
3667	ASAN_SHADOW_GRANULARITY * BITS_PER_UNIT));
3668
3669	varpool_node::finalize_decl (decl: var);
3670
3671	tree fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_REGISTER_GLOBALS);
3672	tree gcount_tree = build_int_cst (pointer_sized_int_node, gcount);
3673	append_to_statement_list (build_call_expr (fn, 2,
3674	build_fold_addr_expr (var),
3675	gcount_tree),
3676	&asan_ctor_statements);
3677
3678	fn = builtin_decl_implicit (fncode: BUILT_IN_ASAN_UNREGISTER_GLOBALS);
3679	append_to_statement_list (build_call_expr (fn, 2,
3680	build_fold_addr_expr (var),
3681	gcount_tree),
3682	&dtor_statements);
3683	cgraph_build_static_cdtor (which: 'D', body: dtor_statements, priority);
3684	}
3685	if (asan_ctor_statements)
3686	cgraph_build_static_cdtor (which: 'I', body: asan_ctor_statements, priority);
3687	flag_sanitize |= SANITIZE_ADDRESS;
3688	}
3689
3690	/* Poison or unpoison (depending on IS_CLOBBER variable) shadow memory based
3691	on SHADOW address. Newly added statements will be added to ITER with
3692	given location LOC. We mark SIZE bytes in shadow memory, where
3693	LAST_CHUNK_SIZE is greater than zero in situation where we are at the
3694	end of a variable. */
3695
3696	static void
3697	asan_store_shadow_bytes (gimple_stmt_iterator *iter, location_t loc,
3698	tree shadow,
3699	unsigned HOST_WIDE_INT base_addr_offset,
3700	bool is_clobber, unsigned size,
3701	unsigned last_chunk_size)
3702	{
3703	tree shadow_ptr_type;
3704
3705	switch (size)
3706	{
3707	case 1:
3708	shadow_ptr_type = shadow_ptr_types[0];
3709	break;
3710	case 2:
3711	shadow_ptr_type = shadow_ptr_types[1];
3712	break;
3713	case 4:
3714	shadow_ptr_type = shadow_ptr_types[2];
3715	break;
3716	default:
3717	gcc_unreachable ();
3718	}
3719
3720	unsigned char c = (char) is_clobber ? ASAN_STACK_MAGIC_USE_AFTER_SCOPE : 0;
3721	unsigned HOST_WIDE_INT val = 0;
3722	unsigned last_pos = size;
3723	if (last_chunk_size && !is_clobber)
3724	last_pos = BYTES_BIG_ENDIAN ? 0 : size - 1;
3725	for (unsigned i = 0; i < size; ++i)
3726	{
3727	unsigned char shadow_c = c;
3728	if (i == last_pos)
3729	shadow_c = last_chunk_size;
3730	val |= (unsigned HOST_WIDE_INT) shadow_c << (BITS_PER_UNIT * i);
3731	}
3732
3733	/* Handle last chunk in unpoisoning. */
3734	tree magic = build_int_cst (TREE_TYPE (shadow_ptr_type), val);
3735
3736	tree dest = build2 (MEM_REF, TREE_TYPE (shadow_ptr_type), shadow,
3737	build_int_cst (shadow_ptr_type, base_addr_offset));
3738
3739	gimple *g = gimple_build_assign (dest, magic);
3740	gimple_set_location (g, location: loc);
3741	gsi_insert_after (iter, g, GSI_NEW_STMT);
3742	}
3743
3744	/* Expand the ASAN_MARK builtins. */
3745
3746	bool
3747	asan_expand_mark_ifn (gimple_stmt_iterator *iter)
3748	{
3749	gimple *g = gsi_stmt (i: *iter);
3750	location_t loc = gimple_location (g);
3751	HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
3752	bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
3753
3754	tree base = gimple_call_arg (gs: g, index: 1);
3755	gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
3756	tree decl = TREE_OPERAND (base, 0);
3757
3758	/* For a nested function, we can have: ASAN_MARK (2, &FRAME.2.fp_input, 4) */
3759	if (TREE_CODE (decl) == COMPONENT_REF
3760	&& DECL_NONLOCAL_FRAME (TREE_OPERAND (decl, 0)))
3761	decl = TREE_OPERAND (decl, 0);
3762
3763	gcc_checking_assert (TREE_CODE (decl) == VAR_DECL);
3764
3765	if (hwasan_sanitize_p ())
3766	{
3767	gcc_assert (param_hwasan_instrument_stack);
3768	gimple_seq stmts = NULL;
3769	/* Here we swap ASAN_MARK calls for HWASAN_MARK.
3770	This is because we are using the approach of using ASAN_MARK as a
3771	synonym until here.
3772	That approach means we don't yet have to duplicate all the special
3773	cases for ASAN_MARK and ASAN_POISON with the exact same handling but
3774	called HWASAN_MARK etc.
3775
3776	N.b. __asan_poison_stack_memory (which implements ASAN_MARK for ASAN)
3777	rounds the size up to its shadow memory granularity, while
3778	__hwasan_tag_memory (which implements the same for HWASAN) does not.
3779	Hence we emit HWASAN_MARK with an aligned size unlike ASAN_MARK. */
3780	tree len = gimple_call_arg (gs: g, index: 2);
3781	tree new_len = gimple_build_round_up (seq: &stmts, loc, size_type_node, old_size: len,
3782	HWASAN_TAG_GRANULE_SIZE);
3783	gimple_build (seq: &stmts, loc, fn: CFN_HWASAN_MARK,
3784	void_type_node, args: gimple_call_arg (gs: g, index: 0),
3785	args: base, args: new_len);
3786	gsi_replace_with_seq (iter, stmts, true);
3787	return false;
3788	}
3789
3790	if (is_poison)
3791	{
3792	if (asan_handled_variables == NULL)
3793	asan_handled_variables = new hash_set<tree> (16);
3794	asan_handled_variables->add (k: decl);
3795	}
3796	tree len = gimple_call_arg (gs: g, index: 2);
3797
3798	gcc_assert (tree_fits_shwi_p (len));
3799	unsigned HOST_WIDE_INT size_in_bytes = tree_to_shwi (len);
3800	gcc_assert (size_in_bytes);
3801
3802	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3803	NOP_EXPR, base);
3804	gimple_set_location (g, location: loc);
3805	gsi_replace (iter, g, false);
3806	tree base_addr = gimple_assign_lhs (gs: g);
3807
3808	/* Generate direct emission if size_in_bytes is small. */
3809	if (size_in_bytes
3810	<= (unsigned)param_use_after_scope_direct_emission_threshold)
3811	{
3812	const unsigned HOST_WIDE_INT shadow_size
3813	= shadow_mem_size (size: size_in_bytes);
3814	const unsigned int shadow_align
3815	= (get_pointer_alignment (base) / BITS_PER_UNIT) >> ASAN_SHADOW_SHIFT;
3816
3817	tree shadow = build_shadow_mem_access (gsi: iter, location: loc, base_addr,
3818	shadow_ptr_type: shadow_ptr_types[0], return_address: true);
3819
3820	for (unsigned HOST_WIDE_INT offset = 0; offset < shadow_size;)
3821	{
3822	unsigned size = 1;
3823	if (shadow_size - offset >= 4
3824	&& (!STRICT_ALIGNMENT || shadow_align >= 4))
3825	size = 4;
3826	else if (shadow_size - offset >= 2
3827	&& (!STRICT_ALIGNMENT || shadow_align >= 2))
3828	size = 2;
3829
3830	unsigned HOST_WIDE_INT last_chunk_size = 0;
3831	unsigned HOST_WIDE_INT s = (offset + size) * ASAN_SHADOW_GRANULARITY;
3832	if (s > size_in_bytes)
3833	last_chunk_size = ASAN_SHADOW_GRANULARITY - (s - size_in_bytes);
3834
3835	asan_store_shadow_bytes (iter, loc, shadow, base_addr_offset: offset, is_clobber: is_poison,
3836	size, last_chunk_size);
3837	offset += size;
3838	}
3839	}
3840	else
3841	{
3842	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3843	NOP_EXPR, len);
3844	gimple_set_location (g, location: loc);
3845	gsi_insert_before (iter, g, GSI_SAME_STMT);
3846	tree sz_arg = gimple_assign_lhs (gs: g);
3847
3848	tree fun
3849	= builtin_decl_implicit (fncode: is_poison ? BUILT_IN_ASAN_POISON_STACK_MEMORY
3850	: BUILT_IN_ASAN_UNPOISON_STACK_MEMORY);
3851	g = gimple_build_call (fun, 2, base_addr, sz_arg);
3852	gimple_set_location (g, location: loc);
3853	gsi_insert_after (iter, g, GSI_NEW_STMT);
3854	}
3855
3856	return false;
3857	}
3858
3859	/* Expand the ASAN_{LOAD,STORE} builtins. */
3860
3861	bool
3862	asan_expand_check_ifn (gimple_stmt_iterator *iter, bool use_calls)
3863	{
3864	gcc_assert (!hwasan_sanitize_p ());
3865	gimple *g = gsi_stmt (i: *iter);
3866	location_t loc = gimple_location (g);
3867	bool recover_p;
3868	if (flag_sanitize & SANITIZE_USER_ADDRESS)
3869	recover_p = (flag_sanitize_recover & SANITIZE_USER_ADDRESS) != 0;
3870	else
3871	recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_ADDRESS) != 0;
3872
3873	HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
3874	gcc_assert (flags < ASAN_CHECK_LAST);
3875	bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
3876	bool is_store = (flags & ASAN_CHECK_STORE) != 0;
3877	bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
3878
3879	tree base = gimple_call_arg (gs: g, index: 1);
3880	tree len = gimple_call_arg (gs: g, index: 2);
3881	HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (gs: g, index: 3));
3882
3883	HOST_WIDE_INT size_in_bytes
3884	= is_scalar_access && tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
3885
3886	if (use_calls)
3887	{
3888	/* Instrument using callbacks. */
3889	gimple *g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3890	NOP_EXPR, base);
3891	gimple_set_location (g, location: loc);
3892	gsi_insert_before (iter, g, GSI_SAME_STMT);
3893	tree base_addr = gimple_assign_lhs (gs: g);
3894
3895	int nargs;
3896	tree fun = check_func (is_store, recover_p, size_in_bytes, nargs: &nargs);
3897	if (nargs == 1)
3898	g = gimple_build_call (fun, 1, base_addr);
3899	else
3900	{
3901	gcc_assert (nargs == 2);
3902	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3903	NOP_EXPR, len);
3904	gimple_set_location (g, location: loc);
3905	gsi_insert_before (iter, g, GSI_SAME_STMT);
3906	tree sz_arg = gimple_assign_lhs (gs: g);
3907	g = gimple_build_call (fun, nargs, base_addr, sz_arg);
3908	}
3909	gimple_set_location (g, location: loc);
3910	gsi_replace (iter, g, false);
3911	return false;
3912	}
3913
3914	HOST_WIDE_INT real_size_in_bytes = size_in_bytes == -1 ? 1 : size_in_bytes;
3915
3916	tree shadow_ptr_type = shadow_ptr_types[real_size_in_bytes == 16 ? 1 : 0];
3917	tree shadow_type = TREE_TYPE (shadow_ptr_type);
3918
3919	gimple_stmt_iterator gsi = *iter;
3920
3921	if (!is_non_zero_len)
3922	{
3923	/* So, the length of the memory area to asan-protect is
3924	non-constant. Let's guard the generated instrumentation code
3925	like:
3926
3927	if (len != 0)
3928	{
3929	//asan instrumentation code goes here.
3930	}
3931	// falltrough instructions, starting with *ITER. */
3932
3933	g = gimple_build_cond (NE_EXPR,
3934	len,
3935	build_int_cst (TREE_TYPE (len), 0),
3936	NULL_TREE, NULL_TREE);
3937	gimple_set_location (g, location: loc);
3938
3939	basic_block then_bb, fallthrough_bb;
3940	insert_if_then_before_iter (cond: as_a <gcond *> (p: g), iter,
3941	/*then_more_likely_p=*/true,
3942	then_bb: &then_bb, fallthrough_bb: &fallthrough_bb);
3943	/* Note that fallthrough_bb starts with the statement that was
3944	pointed to by ITER. */
3945
3946	/* The 'then block' of the 'if (len != 0) condition is where
3947	we'll generate the asan instrumentation code now. */
3948	gsi = gsi_last_bb (bb: then_bb);
3949	}
3950
3951	/* Get an iterator on the point where we can add the condition
3952	statement for the instrumentation. */
3953	basic_block then_bb, else_bb;
3954	gsi = create_cond_insert_point (iter: &gsi, /*before_p*/false,
3955	/*then_more_likely_p=*/false,
3956	/*create_then_fallthru_edge*/recover_p,
3957	then_block: &then_bb,
3958	fallthrough_block: &else_bb);
3959
3960	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
3961	NOP_EXPR, base);
3962	gimple_set_location (g, location: loc);
3963	gsi_insert_before (&gsi, g, GSI_NEW_STMT);
3964	tree base_addr = gimple_assign_lhs (gs: g);
3965
3966	tree t = NULL_TREE;
3967	if (real_size_in_bytes >= 8)
3968	{
3969	tree shadow = build_shadow_mem_access (gsi: &gsi, location: loc, base_addr,
3970	shadow_ptr_type);
3971	t = shadow;
3972	}
3973	else
3974	{
3975	/* Slow path for 1, 2 and 4 byte accesses. */
3976	/* Test (shadow != 0)
3977	& ((base_addr & 7) + (real_size_in_bytes - 1)) >= shadow). */
3978	tree shadow = build_shadow_mem_access (gsi: &gsi, location: loc, base_addr,
3979	shadow_ptr_type);
3980	gimple *shadow_test = build_assign (NE_EXPR, shadow, 0);
3981	gimple_seq seq = NULL;
3982	gimple_seq_add_stmt (&seq, shadow_test);
3983	/* Aligned (>= 8 bytes) can test just
3984	(real_size_in_bytes - 1 >= shadow), as base_addr & 7 is known
3985	to be 0. */
3986	if (align < 8)
3987	{
3988	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR,
3989	base_addr, 7));
3990	gimple_seq_add_stmt (&seq,
3991	build_type_cast (shadow_type,
3992	gimple_seq_last (s: seq)));
3993	if (real_size_in_bytes > 1)
3994	gimple_seq_add_stmt (&seq,
3995	build_assign (PLUS_EXPR,
3996	gimple_seq_last (s: seq),
3997	real_size_in_bytes - 1));
3998	t = gimple_assign_lhs (gs: gimple_seq_last_stmt (s: seq));
3999	}
4000	else
4001	t = build_int_cst (shadow_type, real_size_in_bytes - 1);
4002	gimple_seq_add_stmt (&seq, build_assign (GE_EXPR, t, shadow));
4003	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR, shadow_test,
4004	gimple_seq_last (s: seq)));
4005	t = gimple_assign_lhs (gs: gimple_seq_last (s: seq));
4006	gimple_seq_set_location (seq, loc);
4007	gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
4008
4009	/* For non-constant, misaligned or otherwise weird access sizes,
4010	check first and last byte. */
4011	if (size_in_bytes == -1)
4012	{
4013	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
4014	MINUS_EXPR, len,
4015	build_int_cst (pointer_sized_int_node, 1));
4016	gimple_set_location (g, location: loc);
4017	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4018	tree last = gimple_assign_lhs (gs: g);
4019	g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
4020	PLUS_EXPR, base_addr, last);
4021	gimple_set_location (g, location: loc);
4022	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4023	tree base_end_addr = gimple_assign_lhs (gs: g);
4024
4025	tree shadow = build_shadow_mem_access (gsi: &gsi, location: loc, base_addr: base_end_addr,
4026	shadow_ptr_type);
4027	gimple *shadow_test = build_assign (NE_EXPR, shadow, 0);
4028	gimple_seq seq = NULL;
4029	gimple_seq_add_stmt (&seq, shadow_test);
4030	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR,
4031	base_end_addr, 7));
4032	gimple_seq_add_stmt (&seq, build_type_cast (shadow_type,
4033	gimple_seq_last (s: seq)));
4034	gimple_seq_add_stmt (&seq, build_assign (GE_EXPR,
4035	gimple_seq_last (s: seq),
4036	shadow));
4037	gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR, shadow_test,
4038	gimple_seq_last (s: seq)));
4039	gimple_seq_add_stmt (&seq, build_assign (BIT_IOR_EXPR, t,
4040	gimple_seq_last (s: seq)));
4041	t = gimple_assign_lhs (gs: gimple_seq_last (s: seq));
4042	gimple_seq_set_location (seq, loc);
4043	gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
4044	}
4045	}
4046
4047	g = gimple_build_cond (NE_EXPR, t, build_int_cst (TREE_TYPE (t), 0),
4048	NULL_TREE, NULL_TREE);
4049	gimple_set_location (g, location: loc);
4050	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4051
4052	/* Generate call to the run-time library (e.g. __asan_report_load8). */
4053	gsi = gsi_start_bb (bb: then_bb);
4054	int nargs;
4055	tree fun = report_error_func (is_store, recover_p, size_in_bytes, nargs: &nargs);
4056	g = gimple_build_call (fun, nargs, base_addr, len);
4057	gimple_set_location (g, location: loc);
4058	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
4059
4060	gsi_remove (iter, true);
4061	*iter = gsi_start_bb (bb: else_bb);
4062
4063	return true;
4064	}
4065
4066	/* Create ASAN shadow variable for a VAR_DECL which has been rewritten
4067	into SSA. Already seen VAR_DECLs are stored in SHADOW_VARS_MAPPING. */
4068
4069	static tree
4070	create_asan_shadow_var (tree var_decl,
4071	hash_map<tree, tree> &shadow_vars_mapping)
4072	{
4073	tree *slot = shadow_vars_mapping.get (k: var_decl);
4074	if (slot == NULL)
4075	{
4076	tree shadow_var = copy_node (var_decl);
4077
4078	copy_body_data id;
4079	memset (s: &id, c: 0, n: sizeof (copy_body_data));
4080	id.src_fn = id.dst_fn = current_function_decl;
4081	copy_decl_for_dup_finish (id: &id, decl: var_decl, copy: shadow_var);
4082
4083	DECL_ARTIFICIAL (shadow_var) = 1;
4084	DECL_IGNORED_P (shadow_var) = 1;
4085	DECL_SEEN_IN_BIND_EXPR_P (shadow_var) = 0;
4086	gimple_add_tmp_var (shadow_var);
4087
4088	shadow_vars_mapping.put (k: var_decl, v: shadow_var);
4089	return shadow_var;
4090	}
4091	else
4092	return *slot;
4093	}
4094
4095	/* Expand ASAN_POISON ifn. */
4096
4097	bool
4098	asan_expand_poison_ifn (gimple_stmt_iterator *iter,
4099	bool *need_commit_edge_insert,
4100	hash_map<tree, tree> &shadow_vars_mapping)
4101	{
4102	gimple *g = gsi_stmt (i: *iter);
4103	tree poisoned_var = gimple_call_lhs (gs: g);
4104	if (!poisoned_var || has_zero_uses (var: poisoned_var))
4105	{
4106	gsi_remove (iter, true);
4107	return true;
4108	}
4109
4110	if (SSA_NAME_VAR (poisoned_var) == NULL_TREE)
4111	SET_SSA_NAME_VAR_OR_IDENTIFIER (poisoned_var,
4112	create_tmp_var (TREE_TYPE (poisoned_var)));
4113
4114	tree shadow_var = create_asan_shadow_var (SSA_NAME_VAR (poisoned_var),
4115	shadow_vars_mapping);
4116
4117	bool recover_p;
4118	if (flag_sanitize & SANITIZE_USER_ADDRESS)
4119	recover_p = (flag_sanitize_recover & SANITIZE_USER_ADDRESS) != 0;
4120	else
4121	recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_ADDRESS) != 0;
4122	tree size = DECL_SIZE_UNIT (shadow_var);
4123	gimple *poison_call
4124	= gimple_build_call_internal (IFN_ASAN_MARK, 3,
4125	build_int_cst (integer_type_node,
4126	ASAN_MARK_POISON),
4127	build_fold_addr_expr (shadow_var), size);
4128
4129	gimple *use;
4130	imm_use_iterator imm_iter;
4131	FOR_EACH_IMM_USE_STMT (use, imm_iter, poisoned_var)
4132	{
4133	if (is_gimple_debug (gs: use))
4134	continue;
4135
4136	int nargs;
4137	bool store_p = gimple_call_internal_p (gs: use, fn: IFN_ASAN_POISON_USE);
4138	gcall *call;
4139	if (hwasan_sanitize_p ())
4140	{
4141	tree fun = builtin_decl_implicit (fncode: BUILT_IN_HWASAN_TAG_MISMATCH4);
4142	/* NOTE: hwasan has no __hwasan_report_* functions like asan does.
4143	We use __hwasan_tag_mismatch4 with arguments that tell it the
4144	size of access and load to report all tag mismatches.
4145
4146	The arguments to this function are:
4147	Address of invalid access.
4148	Bitfield containing information about the access
4149	(access_info)
4150	Pointer to a frame of registers
4151	(for use in printing the contents of registers in a dump)
4152	Not used yet -- to be used by inline instrumentation.
4153	Size of access.
4154
4155	The access_info bitfield encodes the following pieces of
4156	information:
4157	- Is this a store or load?
4158	access_info & 0x10 => store
4159	- Should the program continue after reporting the error?
4160	access_info & 0x20 => recover
4161	- What size access is this (not used here since we can always
4162	pass the size in the last argument)
4163
4164	if (access_info & 0xf == 0xf)
4165	size is taken from last argument.
4166	else
4167	size == 1 << (access_info & 0xf)
4168
4169	The last argument contains the size of the access iff the
4170	access_info size indicator is 0xf (we always use this argument
4171	rather than storing the size in the access_info bitfield).
4172
4173	See the function definition `__hwasan_tag_mismatch4` in
4174	libsanitizer/hwasan for the full definition.
4175	*/
4176	unsigned access_info = (0x20 * recover_p)
4177	+ (0x10 * store_p)
4178	+ (0xf);
4179	call = gimple_build_call (fun, 4,
4180	build_fold_addr_expr (shadow_var),
4181	build_int_cst (pointer_sized_int_node,
4182	access_info),
4183	build_int_cst (pointer_sized_int_node, 0),
4184	size);
4185	}
4186	else
4187	{
4188	tree fun = report_error_func (is_store: store_p, recover_p, size_in_bytes: tree_to_uhwi (size),
4189	nargs: &nargs);
4190	call = gimple_build_call (fun, 1,
4191	build_fold_addr_expr (shadow_var));
4192	}
4193	gimple_set_location (g: call, location: gimple_location (g: use));
4194	gimple *call_to_insert = call;
4195
4196	/* The USE can be a gimple PHI node. If so, insert the call on
4197	all edges leading to the PHI node. */
4198	if (is_a <gphi *> (p: use))
4199	{
4200	gphi *phi = dyn_cast<gphi *> (p: use);
4201	for (unsigned i = 0; i < gimple_phi_num_args (gs: phi); ++i)
4202	if (gimple_phi_arg_def (gs: phi, index: i) == poisoned_var)
4203	{
4204	edge e = gimple_phi_arg_edge (phi, i);
4205
4206	/* Do not insert on an edge we can't split. */
4207	if (e->flags & EDGE_ABNORMAL)
4208	continue;
4209
4210	if (call_to_insert == NULL)
4211	call_to_insert = gimple_copy (call);
4212
4213	gsi_insert_seq_on_edge (e, call_to_insert);
4214	*need_commit_edge_insert = true;
4215	call_to_insert = NULL;
4216	}
4217	}
4218	else
4219	{
4220	gimple_stmt_iterator gsi = gsi_for_stmt (use);
4221	if (store_p)
4222	gsi_replace (&gsi, call, true);
4223	else
4224	gsi_insert_before (&gsi, call, GSI_NEW_STMT);
4225	}
4226	}
4227
4228	SSA_NAME_IS_DEFAULT_DEF (poisoned_var) = true;
4229	SSA_NAME_DEF_STMT (poisoned_var) = gimple_build_nop ();
4230	gsi_replace (iter, poison_call, false);
4231
4232	return true;
4233	}
4234
4235	/* Instrument the current function. */
4236
4237	static unsigned int
4238	asan_instrument (void)
4239	{
4240	if (hwasan_sanitize_p ())
4241	{
4242	transform_statements ();
4243	return 0;
4244	}
4245
4246	if (shadow_ptr_types[0] == NULL_TREE)
4247	asan_init_shadow_ptr_types ();
4248	transform_statements ();
4249	last_alloca_addr = NULL_TREE;
4250	return 0;
4251	}
4252
4253	static bool
4254	gate_asan (void)
4255	{
4256	return sanitize_flags_p (flag: SANITIZE_ADDRESS);
4257	}
4258
4259	namespace {
4260
4261	const pass_data pass_data_asan =
4262	{
4263	.type: GIMPLE_PASS, /* type */
4264	.name: "asan", /* name */
4265	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
4266	.tv_id: TV_NONE, /* tv_id */
4267	.properties_required: ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
4268	.properties_provided: 0, /* properties_provided */
4269	.properties_destroyed: 0, /* properties_destroyed */
4270	.todo_flags_start: 0, /* todo_flags_start */
4271	TODO_update_ssa, /* todo_flags_finish */
4272	};
4273
4274	class pass_asan : public gimple_opt_pass
4275	{
4276	public:
4277	pass_asan (gcc::context *ctxt)
4278	: gimple_opt_pass (pass_data_asan, ctxt)
4279	{}
4280
4281	/* opt_pass methods: */
4282	opt_pass * clone () final override { return new pass_asan (m_ctxt); }
4283	bool gate (function *) final override
4284	{
4285	return gate_asan () || gate_hwasan ();
4286	}
4287	unsigned int execute (function *) final override
4288	{
4289	return asan_instrument ();
4290	}
4291
4292	}; // class pass_asan
4293
4294	} // anon namespace
4295
4296	gimple_opt_pass *
4297	make_pass_asan (gcc::context *ctxt)
4298	{
4299	return new pass_asan (ctxt);
4300	}
4301
4302	namespace {
4303
4304	const pass_data pass_data_asan_O0 =
4305	{
4306	.type: GIMPLE_PASS, /* type */
4307	.name: "asan0", /* name */
4308	.optinfo_flags: OPTGROUP_NONE, /* optinfo_flags */
4309	.tv_id: TV_NONE, /* tv_id */
4310	.properties_required: ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
4311	.properties_provided: 0, /* properties_provided */
4312	.properties_destroyed: 0, /* properties_destroyed */
4313	.todo_flags_start: 0, /* todo_flags_start */
4314	TODO_update_ssa, /* todo_flags_finish */
4315	};
4316
4317	class pass_asan_O0 : public gimple_opt_pass
4318	{
4319	public:
4320	pass_asan_O0 (gcc::context *ctxt)
4321	: gimple_opt_pass (pass_data_asan_O0, ctxt)
4322	{}
4323
4324	/* opt_pass methods: */
4325	bool gate (function *) final override
4326	{
4327	return !optimize && (gate_asan () || gate_hwasan ());
4328	}
4329	unsigned int execute (function *) final override
4330	{
4331	return asan_instrument ();
4332	}
4333
4334	}; // class pass_asan_O0
4335
4336	} // anon namespace
4337
4338	gimple_opt_pass *
4339	make_pass_asan_O0 (gcc::context *ctxt)
4340	{
4341	return new pass_asan_O0 (ctxt);
4342	}
4343
4344	/* HWASAN */
4345
4346	/* For stack tagging:
4347
4348	Return the offset from the frame base tag that the "next" expanded object
4349	should have. */
4350	uint8_t
4351	hwasan_current_frame_tag ()
4352	{
4353	return hwasan_frame_tag_offset;
4354	}
4355
4356	/* For stack tagging:
4357
4358	Return the 'base pointer' for this function. If that base pointer has not
4359	yet been created then we create a register to hold it and record the insns
4360	to initialize the register in `hwasan_frame_base_init_seq` for later
4361	emission. */
4362	rtx
4363	hwasan_frame_base ()
4364	{
4365	if (! hwasan_frame_base_ptr)
4366	{
4367	start_sequence ();
4368	hwasan_frame_base_ptr
4369	= force_reg (Pmode,
4370	targetm.memtag.insert_random_tag (virtual_stack_vars_rtx,
4371	NULL_RTX));
4372	hwasan_frame_base_init_seq = get_insns ();
4373	end_sequence ();
4374	}
4375
4376	return hwasan_frame_base_ptr;
4377	}
4378
4379	/* For stack tagging:
4380
4381	Check whether this RTX is a standard pointer addressing the base of the
4382	stack variables for this frame. Returns true if the RTX is either
4383	virtual_stack_vars_rtx or hwasan_frame_base_ptr. */
4384	bool
4385	stack_vars_base_reg_p (rtx base)
4386	{
4387	return base == virtual_stack_vars_rtx || base == hwasan_frame_base_ptr;
4388	}
4389
4390	/* For stack tagging:
4391
4392	Emit frame base initialisation.
4393	If hwasan_frame_base has been used before here then
4394	hwasan_frame_base_init_seq contains the sequence of instructions to
4395	initialize it. This must be put just before the hwasan prologue, so we emit
4396	the insns before parm_birth_insn (which will point to the first instruction
4397	of the hwasan prologue if it exists).
4398
4399	We update `parm_birth_insn` to point to the start of this initialisation
4400	since that represents the end of the initialisation done by
4401	expand_function_{start,end} functions and we want to maintain that. */
4402	void
4403	hwasan_maybe_emit_frame_base_init ()
4404	{
4405	if (! hwasan_frame_base_init_seq)
4406	return;
4407	emit_insn_before (hwasan_frame_base_init_seq, parm_birth_insn);
4408	parm_birth_insn = hwasan_frame_base_init_seq;
4409	}
4410
4411	/* Record a compile-time constant size stack variable that HWASAN will need to
4412	tag. This record of the range of a stack variable will be used by
4413	`hwasan_emit_prologue` to emit the RTL at the start of each frame which will
4414	set tags in the shadow memory according to the assigned tag for each object.
4415
4416	The range that the object spans in stack space should be described by the
4417	bounds `untagged_base + nearest_offset` and
4418	`untagged_base + farthest_offset`.
4419	`tagged_base` is the base address which contains the "base frame tag" for
4420	this frame, and from which the value to address this object with will be
4421	calculated.
4422
4423	We record the `untagged_base` since the functions in the hwasan library we
4424	use to tag memory take pointers without a tag. */
4425	void
4426	hwasan_record_stack_var (rtx untagged_base, rtx tagged_base,
4427	poly_int64 nearest_offset, poly_int64 farthest_offset)
4428	{
4429	hwasan_stack_var cur_var;
4430	cur_var.untagged_base = untagged_base;
4431	cur_var.tagged_base = tagged_base;
4432	cur_var.nearest_offset = nearest_offset;
4433	cur_var.farthest_offset = farthest_offset;
4434	cur_var.tag_offset = hwasan_current_frame_tag ();
4435
4436	hwasan_tagged_stack_vars.safe_push (obj: cur_var);
4437	}
4438
4439	/* Return the RTX representing the farthest extent of the statically allocated
4440	stack objects for this frame. If hwasan_frame_base_ptr has not been
4441	initialized then we are not storing any static variables on the stack in
4442	this frame. In this case we return NULL_RTX to represent that.
4443
4444	Otherwise simply return virtual_stack_vars_rtx + frame_offset. */
4445	rtx
4446	hwasan_get_frame_extent ()
4447	{
4448	return (hwasan_frame_base_ptr
4449	? plus_constant (Pmode, virtual_stack_vars_rtx, frame_offset)
4450	: NULL_RTX);
4451	}
4452
4453	/* For stack tagging:
4454
4455	Increment the frame tag offset modulo the size a tag can represent. */
4456	void
4457	hwasan_increment_frame_tag ()
4458	{
4459	uint8_t tag_bits = HWASAN_TAG_SIZE;
4460	gcc_assert (HWASAN_TAG_SIZE
4461	<= sizeof (hwasan_frame_tag_offset) * CHAR_BIT);
4462	hwasan_frame_tag_offset = (hwasan_frame_tag_offset + 1) % (1 << tag_bits);
4463	/* The "background tag" of the stack is zero by definition.
4464	This is the tag that objects like parameters passed on the stack and
4465	spilled registers are given. It is handy to avoid this tag for objects
4466	whose tags we decide ourselves, partly to ensure that buffer overruns
4467	can't affect these important variables (e.g. saved link register, saved
4468	stack pointer etc) and partly to make debugging easier (everything with a
4469	tag of zero is space allocated automatically by the compiler).
4470
4471	This is not feasible when using random frame tags (the default
4472	configuration for hwasan) since the tag for the given frame is randomly
4473	chosen at runtime. In order to avoid any tags matching the stack
4474	background we would need to decide tag offsets at runtime instead of
4475	compile time (and pay the resulting performance cost).
4476
4477	When not using random base tags for each frame (i.e. when compiled with
4478	`--param hwasan-random-frame-tag=0`) the base tag for each frame is zero.
4479	This means the tag that each object gets is equal to the
4480	hwasan_frame_tag_offset used in determining it.
4481	When this is the case we *can* ensure no object gets the tag of zero by
4482	simply ensuring no object has the hwasan_frame_tag_offset of zero.
4483
4484	There is the extra complication that we only record the
4485	hwasan_frame_tag_offset here (which is the offset from the tag stored in
4486	the stack pointer). In the kernel, the tag in the stack pointer is 0xff
4487	rather than zero. This does not cause problems since tags of 0xff are
4488	never checked in the kernel. As mentioned at the beginning of this
4489	comment the background tag of the stack is zero by definition, which means
4490	that for the kernel we should skip offsets of both 0 and 1 from the stack
4491	pointer. Avoiding the offset of 0 ensures we use a tag which will be
4492	checked, avoiding the offset of 1 ensures we use a tag that is not the
4493	same as the background. */
4494	if (hwasan_frame_tag_offset == 0 && ! param_hwasan_random_frame_tag)
4495	hwasan_frame_tag_offset += 1;
4496	if (hwasan_frame_tag_offset == 1 && ! param_hwasan_random_frame_tag
4497	&& sanitize_flags_p (flag: SANITIZE_KERNEL_HWADDRESS))
4498	hwasan_frame_tag_offset += 1;
4499	}
4500
4501	/* Clear internal state for the next function.
4502	This function is called before variables on the stack get expanded, in
4503	`init_vars_expansion`. */
4504	void
4505	hwasan_record_frame_init ()
4506	{
4507	delete asan_used_labels;
4508	asan_used_labels = NULL;
4509
4510	/* If this isn't the case then some stack variable was recorded *before*
4511	hwasan_record_frame_init is called, yet *after* the hwasan prologue for
4512	the previous frame was emitted. Such stack variables would not have
4513	their shadow stack filled in. */
4514	gcc_assert (hwasan_tagged_stack_vars.is_empty ());
4515	hwasan_frame_base_ptr = NULL_RTX;
4516	hwasan_frame_base_init_seq = NULL;
4517
4518	/* When not using a random frame tag we can avoid the background stack
4519	color which gives the user a little better debug output upon a crash.
4520	Meanwhile, when using a random frame tag it will be nice to avoid adding
4521	tags for the first object since that is unnecessary extra work.
4522	Hence set the initial hwasan_frame_tag_offset to be 0 if using a random
4523	frame tag and 1 otherwise.
4524
4525	As described in hwasan_increment_frame_tag, in the kernel the stack
4526	pointer has the tag 0xff. That means that to avoid 0xff and 0 (the tag
4527	which the kernel does not check and the background tag respectively) we
4528	start with a tag offset of 2. */
4529	hwasan_frame_tag_offset = param_hwasan_random_frame_tag
4530	? 0
4531	: sanitize_flags_p (flag: SANITIZE_KERNEL_HWADDRESS) ? 2 : 1;
4532	}
4533
4534	/* For stack tagging:
4535	(Emits HWASAN equivalent of what is emitted by
4536	`asan_emit_stack_protection`).
4537
4538	Emits the extra prologue code to set the shadow stack as required for HWASAN
4539	stack instrumentation.
4540
4541	Uses the vector of recorded stack variables hwasan_tagged_stack_vars. When
4542	this function has completed hwasan_tagged_stack_vars is empty and all
4543	objects it had pointed to are deallocated. */
4544	void
4545	hwasan_emit_prologue ()
4546	{
4547	/* We need untagged base pointers since libhwasan only accepts untagged
4548	pointers in __hwasan_tag_memory. We need the tagged base pointer to obtain
4549	the base tag for an offset. */
4550
4551	if (hwasan_tagged_stack_vars.is_empty ())
4552	return;
4553
4554	poly_int64 bot = 0, top = 0;
4555	for (hwasan_stack_var &cur : hwasan_tagged_stack_vars)
4556	{
4557	poly_int64 nearest = cur.nearest_offset;
4558	poly_int64 farthest = cur.farthest_offset;
4559
4560	if (known_ge (nearest, farthest))
4561	{
4562	top = nearest;
4563	bot = farthest;
4564	}
4565	else
4566	{
4567	/* Given how these values are calculated, one must be known greater
4568	than the other. */
4569	gcc_assert (known_le (nearest, farthest));
4570	top = farthest;
4571	bot = nearest;
4572	}
4573	poly_int64 size = (top - bot);
4574
4575	/* Assert the edge of each variable is aligned to the HWASAN tag granule
4576	size. */
4577	gcc_assert (multiple_p (top, HWASAN_TAG_GRANULE_SIZE));
4578	gcc_assert (multiple_p (bot, HWASAN_TAG_GRANULE_SIZE));
4579	gcc_assert (multiple_p (size, HWASAN_TAG_GRANULE_SIZE));
4580
4581	rtx fn = init_one_libfunc ("__hwasan_tag_memory");
4582	rtx base_tag = targetm.memtag.extract_tag (cur.tagged_base, NULL_RTX);
4583	rtx tag = plus_constant (QImode, base_tag, cur.tag_offset);
4584	tag = hwasan_truncate_to_tag_size (tag, NULL_RTX);
4585
4586	rtx bottom = convert_memory_address (ptr_mode,
4587	plus_constant (Pmode,
4588	cur.untagged_base,
4589	bot));
4590	emit_library_call (fun: fn, fn_type: LCT_NORMAL, VOIDmode,
4591	arg1: bottom, arg1_mode: ptr_mode,
4592	arg2: tag, QImode,
4593	arg3: gen_int_mode (size, ptr_mode), arg3_mode: ptr_mode);
4594	}
4595	/* Clear the stack vars, we've emitted the prologue for them all now. */
4596	hwasan_tagged_stack_vars.truncate (size: 0);
4597	}
4598
4599	/* For stack tagging:
4600
4601	Return RTL insns to clear the tags between DYNAMIC and VARS pointers
4602	into the stack. These instructions should be emitted at the end of
4603	every function.
4604
4605	If `dynamic` is NULL_RTX then no insns are returned. */
4606	rtx_insn *
4607	hwasan_emit_untag_frame (rtx dynamic, rtx vars)
4608	{
4609	if (! dynamic)
4610	return NULL;
4611
4612	start_sequence ();
4613
4614	dynamic = convert_memory_address (ptr_mode, dynamic);
4615	vars = convert_memory_address (ptr_mode, vars);
4616
4617	rtx top_rtx;
4618	rtx bot_rtx;
4619	if (FRAME_GROWS_DOWNWARD)
4620	{
4621	top_rtx = vars;
4622	bot_rtx = dynamic;
4623	}
4624	else
4625	{
4626	top_rtx = dynamic;
4627	bot_rtx = vars;
4628	}
4629
4630	rtx size_rtx = expand_simple_binop (ptr_mode, MINUS, top_rtx, bot_rtx,
4631	NULL_RTX, /* unsignedp = */0,
4632	OPTAB_DIRECT);
4633
4634	rtx fn = init_one_libfunc ("__hwasan_tag_memory");
4635	emit_library_call (fun: fn, fn_type: LCT_NORMAL, VOIDmode,
4636	arg1: bot_rtx, arg1_mode: ptr_mode,
4637	HWASAN_STACK_BACKGROUND, QImode,
4638	arg3: size_rtx, arg3_mode: ptr_mode);
4639
4640	do_pending_stack_adjust ();
4641	rtx_insn *insns = get_insns ();
4642	end_sequence ();
4643	return insns;
4644	}
4645
4646	/* Needs to be GTY(()), because cgraph_build_static_cdtor may
4647	invoke ggc_collect. */
4648	static GTY(()) tree hwasan_ctor_statements;
4649
4650	/* Insert module initialization into this TU. This initialization calls the
4651	initialization code for libhwasan. */
4652	void
4653	hwasan_finish_file (void)
4654	{
4655	/* Do not emit constructor initialization for the kernel.
4656	(the kernel has its own initialization already). */
4657	if (flag_sanitize & SANITIZE_KERNEL_HWADDRESS)
4658	return;
4659
4660	/* Avoid instrumenting code in the hwasan constructors/destructors. */
4661	flag_sanitize &= ~SANITIZE_HWADDRESS;
4662	int priority = MAX_RESERVED_INIT_PRIORITY - 1;
4663	tree fn = builtin_decl_implicit (fncode: BUILT_IN_HWASAN_INIT);
4664	append_to_statement_list (build_call_expr (fn, 0), &hwasan_ctor_statements);
4665	cgraph_build_static_cdtor (which: 'I', body: hwasan_ctor_statements, priority);
4666	flag_sanitize |= SANITIZE_HWADDRESS;
4667	}
4668
4669	/* For stack tagging:
4670
4671	Truncate `tag` to the number of bits that a tag uses (i.e. to
4672	HWASAN_TAG_SIZE). Store the result in `target` if it's convenient. */
4673	rtx
4674	hwasan_truncate_to_tag_size (rtx tag, rtx target)
4675	{
4676	gcc_assert (GET_MODE (tag) == QImode);
4677	if (HWASAN_TAG_SIZE != GET_MODE_PRECISION (QImode))
4678	{
4679	gcc_assert (GET_MODE_PRECISION (QImode) > HWASAN_TAG_SIZE);
4680	rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << HWASAN_TAG_SIZE) - 1,
4681	QImode);
4682	tag = expand_simple_binop (QImode, AND, tag, mask, target,
4683	/* unsignedp = */1, OPTAB_WIDEN);
4684	gcc_assert (tag);
4685	}
4686	return tag;
4687	}
4688
4689	/* Construct a function tree for __hwasan_{load,store}{1,2,4,8,16,_n}.
4690	IS_STORE is either 1 (for a store) or 0 (for a load). */
4691	static combined_fn
4692	hwasan_check_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
4693	int *nargs)
4694	{
4695	static enum built_in_function check[2][2][6]
4696	= { { { BUILT_IN_HWASAN_LOAD1, BUILT_IN_HWASAN_LOAD2,
4697	BUILT_IN_HWASAN_LOAD4, BUILT_IN_HWASAN_LOAD8,
4698	BUILT_IN_HWASAN_LOAD16, BUILT_IN_HWASAN_LOADN },
4699	{ BUILT_IN_HWASAN_STORE1, BUILT_IN_HWASAN_STORE2,
4700	BUILT_IN_HWASAN_STORE4, BUILT_IN_HWASAN_STORE8,
4701	BUILT_IN_HWASAN_STORE16, BUILT_IN_HWASAN_STOREN } },
4702	{ { BUILT_IN_HWASAN_LOAD1_NOABORT,
4703	BUILT_IN_HWASAN_LOAD2_NOABORT,
4704	BUILT_IN_HWASAN_LOAD4_NOABORT,
4705	BUILT_IN_HWASAN_LOAD8_NOABORT,
4706	BUILT_IN_HWASAN_LOAD16_NOABORT,
4707	BUILT_IN_HWASAN_LOADN_NOABORT },
4708	{ BUILT_IN_HWASAN_STORE1_NOABORT,
4709	BUILT_IN_HWASAN_STORE2_NOABORT,
4710	BUILT_IN_HWASAN_STORE4_NOABORT,
4711	BUILT_IN_HWASAN_STORE8_NOABORT,
4712	BUILT_IN_HWASAN_STORE16_NOABORT,
4713	BUILT_IN_HWASAN_STOREN_NOABORT } } };
4714	if (size_in_bytes == -1)
4715	{
4716	*nargs = 2;
4717	return as_combined_fn (fn: check[recover_p][is_store][5]);
4718	}
4719	*nargs = 1;
4720	int size_log2 = exact_log2 (x: size_in_bytes);
4721	gcc_assert (size_log2 >= 0 && size_log2 <= 5);
4722	return as_combined_fn (fn: check[recover_p][is_store][size_log2]);
4723	}
4724
4725	/* Expand the HWASAN_{LOAD,STORE} builtins. */
4726	bool
4727	hwasan_expand_check_ifn (gimple_stmt_iterator *iter, bool)
4728	{
4729	gimple *g = gsi_stmt (i: *iter);
4730	location_t loc = gimple_location (g);
4731	bool recover_p;
4732	if (flag_sanitize & SANITIZE_USER_HWADDRESS)
4733	recover_p = (flag_sanitize_recover & SANITIZE_USER_HWADDRESS) != 0;
4734	else
4735	recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_HWADDRESS) != 0;
4736
4737	HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (gs: g, index: 0));
4738	gcc_assert (flags < ASAN_CHECK_LAST);
4739	bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
4740	bool is_store = (flags & ASAN_CHECK_STORE) != 0;
4741	bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
4742
4743	tree base = gimple_call_arg (gs: g, index: 1);
4744	tree len = gimple_call_arg (gs: g, index: 2);
4745
4746	/* `align` is unused for HWASAN_CHECK, but we pass the argument anyway
4747	since that way the arguments match ASAN_CHECK. */
4748	/* HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (g, 3)); */
4749
4750	unsigned HOST_WIDE_INT size_in_bytes
4751	= is_scalar_access ? tree_to_shwi (len) : -1;
4752
4753	gimple_stmt_iterator gsi = *iter;
4754
4755	if (!is_non_zero_len)
4756	{
4757	/* So, the length of the memory area to hwasan-protect is
4758	non-constant. Let's guard the generated instrumentation code
4759	like:
4760
4761	if (len != 0)
4762	{
4763	// hwasan instrumentation code goes here.
4764	}
4765	// falltrough instructions, starting with *ITER. */
4766
4767	g = gimple_build_cond (NE_EXPR,
4768	len,
4769	build_int_cst (TREE_TYPE (len), 0),
4770	NULL_TREE, NULL_TREE);
4771	gimple_set_location (g, location: loc);
4772
4773	basic_block then_bb, fallthrough_bb;
4774	insert_if_then_before_iter (cond: as_a <gcond *> (p: g), iter,
4775	/*then_more_likely_p=*/true,
4776	then_bb: &then_bb, fallthrough_bb: &fallthrough_bb);
4777	/* Note that fallthrough_bb starts with the statement that was
4778	pointed to by ITER. */
4779
4780	/* The 'then block' of the 'if (len != 0) condition is where
4781	we'll generate the hwasan instrumentation code now. */
4782	gsi = gsi_last_bb (bb: then_bb);
4783	}
4784
4785	gimple_seq stmts = NULL;
4786	tree base_addr = gimple_build (seq: &stmts, loc, code: NOP_EXPR,
4787	pointer_sized_int_node, ops: base);
4788
4789	int nargs = 0;
4790	combined_fn fn
4791	= hwasan_check_func (is_store, recover_p, size_in_bytes, nargs: &nargs);
4792	if (nargs == 1)
4793	gimple_build (seq: &stmts, loc, fn, void_type_node, args: base_addr);
4794	else
4795	{
4796	gcc_assert (nargs == 2);
4797	tree sz_arg = gimple_build (seq: &stmts, loc, code: NOP_EXPR,
4798	pointer_sized_int_node, ops: len);
4799	gimple_build (seq: &stmts, loc, fn, void_type_node, args: base_addr, args: sz_arg);
4800	}
4801
4802	gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
4803	gsi_remove (iter, true);
4804	*iter = gsi;
4805	return false;
4806	}
4807
4808	/* For stack tagging:
4809
4810	Dummy: the HWASAN_MARK internal function should only ever be in the code
4811	after the sanopt pass. */
4812	bool
4813	hwasan_expand_mark_ifn (gimple_stmt_iterator *)
4814	{
4815	gcc_unreachable ();
4816	}
4817
4818	bool
4819	gate_hwasan ()
4820	{
4821	return hwasan_sanitize_p ();
4822	}
4823
4824	#include "gt-asan.h"
4825